James Robinson | 646469d | 2014-10-03 15:33:28 -0700 | [diff] [blame] | 1 | # -*- coding: utf-8 -*- |
| 2 | """ |
| 3 | jinja2.lexer |
| 4 | ~~~~~~~~~~~~ |
| 5 | |
| 6 | This module implements a Jinja / Python combination lexer. The |
| 7 | `Lexer` class provided by this module is used to do some preprocessing |
| 8 | for Jinja. |
| 9 | |
| 10 | On the one hand it filters out invalid operators like the bitshift |
| 11 | operators we don't allow in templates. On the other hand it separates |
| 12 | template code and python code in expressions. |
| 13 | |
| 14 | :copyright: (c) 2010 by the Jinja Team. |
| 15 | :license: BSD, see LICENSE for more details. |
| 16 | """ |
| 17 | import re |
| 18 | |
| 19 | from operator import itemgetter |
| 20 | from collections import deque |
| 21 | from jinja2.exceptions import TemplateSyntaxError |
| 22 | from jinja2.utils import LRUCache |
| 23 | from jinja2._compat import next, iteritems, implements_iterator, text_type, \ |
| 24 | intern |
| 25 | |
| 26 | |
| 27 | # cache for the lexers. Exists in order to be able to have multiple |
| 28 | # environments with the same lexer |
| 29 | _lexer_cache = LRUCache(50) |
| 30 | |
| 31 | # static regular expressions |
| 32 | whitespace_re = re.compile(r'\s+', re.U) |
| 33 | string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'" |
| 34 | r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S) |
| 35 | integer_re = re.compile(r'\d+') |
| 36 | |
| 37 | # we use the unicode identifier rule if this python version is able |
| 38 | # to handle unicode identifiers, otherwise the standard ASCII one. |
| 39 | try: |
| 40 | compile('föö', '<unknown>', 'eval') |
| 41 | except SyntaxError: |
| 42 | name_re = re.compile(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b') |
| 43 | else: |
| 44 | from jinja2 import _stringdefs |
| 45 | name_re = re.compile(r'[%s][%s]*' % (_stringdefs.xid_start, |
| 46 | _stringdefs.xid_continue)) |
| 47 | |
| 48 | float_re = re.compile(r'(?<!\.)\d+\.\d+') |
| 49 | newline_re = re.compile(r'(\r\n|\r|\n)') |
| 50 | |
| 51 | # internal the tokens and keep references to them |
| 52 | TOKEN_ADD = intern('add') |
| 53 | TOKEN_ASSIGN = intern('assign') |
| 54 | TOKEN_COLON = intern('colon') |
| 55 | TOKEN_COMMA = intern('comma') |
| 56 | TOKEN_DIV = intern('div') |
| 57 | TOKEN_DOT = intern('dot') |
| 58 | TOKEN_EQ = intern('eq') |
| 59 | TOKEN_FLOORDIV = intern('floordiv') |
| 60 | TOKEN_GT = intern('gt') |
| 61 | TOKEN_GTEQ = intern('gteq') |
| 62 | TOKEN_LBRACE = intern('lbrace') |
| 63 | TOKEN_LBRACKET = intern('lbracket') |
| 64 | TOKEN_LPAREN = intern('lparen') |
| 65 | TOKEN_LT = intern('lt') |
| 66 | TOKEN_LTEQ = intern('lteq') |
| 67 | TOKEN_MOD = intern('mod') |
| 68 | TOKEN_MUL = intern('mul') |
| 69 | TOKEN_NE = intern('ne') |
| 70 | TOKEN_PIPE = intern('pipe') |
| 71 | TOKEN_POW = intern('pow') |
| 72 | TOKEN_RBRACE = intern('rbrace') |
| 73 | TOKEN_RBRACKET = intern('rbracket') |
| 74 | TOKEN_RPAREN = intern('rparen') |
| 75 | TOKEN_SEMICOLON = intern('semicolon') |
| 76 | TOKEN_SUB = intern('sub') |
| 77 | TOKEN_TILDE = intern('tilde') |
| 78 | TOKEN_WHITESPACE = intern('whitespace') |
| 79 | TOKEN_FLOAT = intern('float') |
| 80 | TOKEN_INTEGER = intern('integer') |
| 81 | TOKEN_NAME = intern('name') |
| 82 | TOKEN_STRING = intern('string') |
| 83 | TOKEN_OPERATOR = intern('operator') |
| 84 | TOKEN_BLOCK_BEGIN = intern('block_begin') |
| 85 | TOKEN_BLOCK_END = intern('block_end') |
| 86 | TOKEN_VARIABLE_BEGIN = intern('variable_begin') |
| 87 | TOKEN_VARIABLE_END = intern('variable_end') |
| 88 | TOKEN_RAW_BEGIN = intern('raw_begin') |
| 89 | TOKEN_RAW_END = intern('raw_end') |
| 90 | TOKEN_COMMENT_BEGIN = intern('comment_begin') |
| 91 | TOKEN_COMMENT_END = intern('comment_end') |
| 92 | TOKEN_COMMENT = intern('comment') |
| 93 | TOKEN_LINESTATEMENT_BEGIN = intern('linestatement_begin') |
| 94 | TOKEN_LINESTATEMENT_END = intern('linestatement_end') |
| 95 | TOKEN_LINECOMMENT_BEGIN = intern('linecomment_begin') |
| 96 | TOKEN_LINECOMMENT_END = intern('linecomment_end') |
| 97 | TOKEN_LINECOMMENT = intern('linecomment') |
| 98 | TOKEN_DATA = intern('data') |
| 99 | TOKEN_INITIAL = intern('initial') |
| 100 | TOKEN_EOF = intern('eof') |
| 101 | |
| 102 | # bind operators to token types |
| 103 | operators = { |
| 104 | '+': TOKEN_ADD, |
| 105 | '-': TOKEN_SUB, |
| 106 | '/': TOKEN_DIV, |
| 107 | '//': TOKEN_FLOORDIV, |
| 108 | '*': TOKEN_MUL, |
| 109 | '%': TOKEN_MOD, |
| 110 | '**': TOKEN_POW, |
| 111 | '~': TOKEN_TILDE, |
| 112 | '[': TOKEN_LBRACKET, |
| 113 | ']': TOKEN_RBRACKET, |
| 114 | '(': TOKEN_LPAREN, |
| 115 | ')': TOKEN_RPAREN, |
| 116 | '{': TOKEN_LBRACE, |
| 117 | '}': TOKEN_RBRACE, |
| 118 | '==': TOKEN_EQ, |
| 119 | '!=': TOKEN_NE, |
| 120 | '>': TOKEN_GT, |
| 121 | '>=': TOKEN_GTEQ, |
| 122 | '<': TOKEN_LT, |
| 123 | '<=': TOKEN_LTEQ, |
| 124 | '=': TOKEN_ASSIGN, |
| 125 | '.': TOKEN_DOT, |
| 126 | ':': TOKEN_COLON, |
| 127 | '|': TOKEN_PIPE, |
| 128 | ',': TOKEN_COMMA, |
| 129 | ';': TOKEN_SEMICOLON |
| 130 | } |
| 131 | |
| 132 | reverse_operators = dict([(v, k) for k, v in iteritems(operators)]) |
| 133 | assert len(operators) == len(reverse_operators), 'operators dropped' |
| 134 | operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in |
| 135 | sorted(operators, key=lambda x: -len(x)))) |
| 136 | |
| 137 | ignored_tokens = frozenset([TOKEN_COMMENT_BEGIN, TOKEN_COMMENT, |
| 138 | TOKEN_COMMENT_END, TOKEN_WHITESPACE, |
| 139 | TOKEN_WHITESPACE, TOKEN_LINECOMMENT_BEGIN, |
| 140 | TOKEN_LINECOMMENT_END, TOKEN_LINECOMMENT]) |
| 141 | ignore_if_empty = frozenset([TOKEN_WHITESPACE, TOKEN_DATA, |
| 142 | TOKEN_COMMENT, TOKEN_LINECOMMENT]) |
| 143 | |
| 144 | |
| 145 | def _describe_token_type(token_type): |
| 146 | if token_type in reverse_operators: |
| 147 | return reverse_operators[token_type] |
| 148 | return { |
| 149 | TOKEN_COMMENT_BEGIN: 'begin of comment', |
| 150 | TOKEN_COMMENT_END: 'end of comment', |
| 151 | TOKEN_COMMENT: 'comment', |
| 152 | TOKEN_LINECOMMENT: 'comment', |
| 153 | TOKEN_BLOCK_BEGIN: 'begin of statement block', |
| 154 | TOKEN_BLOCK_END: 'end of statement block', |
| 155 | TOKEN_VARIABLE_BEGIN: 'begin of print statement', |
| 156 | TOKEN_VARIABLE_END: 'end of print statement', |
| 157 | TOKEN_LINESTATEMENT_BEGIN: 'begin of line statement', |
| 158 | TOKEN_LINESTATEMENT_END: 'end of line statement', |
| 159 | TOKEN_DATA: 'template data / text', |
| 160 | TOKEN_EOF: 'end of template' |
| 161 | }.get(token_type, token_type) |
| 162 | |
| 163 | |
| 164 | def describe_token(token): |
| 165 | """Returns a description of the token.""" |
| 166 | if token.type == 'name': |
| 167 | return token.value |
| 168 | return _describe_token_type(token.type) |
| 169 | |
| 170 | |
| 171 | def describe_token_expr(expr): |
| 172 | """Like `describe_token` but for token expressions.""" |
| 173 | if ':' in expr: |
| 174 | type, value = expr.split(':', 1) |
| 175 | if type == 'name': |
| 176 | return value |
| 177 | else: |
| 178 | type = expr |
| 179 | return _describe_token_type(type) |
| 180 | |
| 181 | |
| 182 | def count_newlines(value): |
| 183 | """Count the number of newline characters in the string. This is |
| 184 | useful for extensions that filter a stream. |
| 185 | """ |
| 186 | return len(newline_re.findall(value)) |
| 187 | |
| 188 | |
| 189 | def compile_rules(environment): |
| 190 | """Compiles all the rules from the environment into a list of rules.""" |
| 191 | e = re.escape |
| 192 | rules = [ |
| 193 | (len(environment.comment_start_string), 'comment', |
| 194 | e(environment.comment_start_string)), |
| 195 | (len(environment.block_start_string), 'block', |
| 196 | e(environment.block_start_string)), |
| 197 | (len(environment.variable_start_string), 'variable', |
| 198 | e(environment.variable_start_string)) |
| 199 | ] |
| 200 | |
| 201 | if environment.line_statement_prefix is not None: |
| 202 | rules.append((len(environment.line_statement_prefix), 'linestatement', |
| 203 | r'^[ \t\v]*' + e(environment.line_statement_prefix))) |
| 204 | if environment.line_comment_prefix is not None: |
| 205 | rules.append((len(environment.line_comment_prefix), 'linecomment', |
| 206 | r'(?:^|(?<=\S))[^\S\r\n]*' + |
| 207 | e(environment.line_comment_prefix))) |
| 208 | |
| 209 | return [x[1:] for x in sorted(rules, reverse=True)] |
| 210 | |
| 211 | |
| 212 | class Failure(object): |
| 213 | """Class that raises a `TemplateSyntaxError` if called. |
| 214 | Used by the `Lexer` to specify known errors. |
| 215 | """ |
| 216 | |
| 217 | def __init__(self, message, cls=TemplateSyntaxError): |
| 218 | self.message = message |
| 219 | self.error_class = cls |
| 220 | |
| 221 | def __call__(self, lineno, filename): |
| 222 | raise self.error_class(self.message, lineno, filename) |
| 223 | |
| 224 | |
| 225 | class Token(tuple): |
| 226 | """Token class.""" |
| 227 | __slots__ = () |
| 228 | lineno, type, value = (property(itemgetter(x)) for x in range(3)) |
| 229 | |
| 230 | def __new__(cls, lineno, type, value): |
| 231 | return tuple.__new__(cls, (lineno, intern(str(type)), value)) |
| 232 | |
| 233 | def __str__(self): |
| 234 | if self.type in reverse_operators: |
| 235 | return reverse_operators[self.type] |
| 236 | elif self.type == 'name': |
| 237 | return self.value |
| 238 | return self.type |
| 239 | |
| 240 | def test(self, expr): |
| 241 | """Test a token against a token expression. This can either be a |
| 242 | token type or ``'token_type:token_value'``. This can only test |
| 243 | against string values and types. |
| 244 | """ |
| 245 | # here we do a regular string equality check as test_any is usually |
| 246 | # passed an iterable of not interned strings. |
| 247 | if self.type == expr: |
| 248 | return True |
| 249 | elif ':' in expr: |
| 250 | return expr.split(':', 1) == [self.type, self.value] |
| 251 | return False |
| 252 | |
| 253 | def test_any(self, *iterable): |
| 254 | """Test against multiple token expressions.""" |
| 255 | for expr in iterable: |
| 256 | if self.test(expr): |
| 257 | return True |
| 258 | return False |
| 259 | |
| 260 | def __repr__(self): |
| 261 | return 'Token(%r, %r, %r)' % ( |
| 262 | self.lineno, |
| 263 | self.type, |
| 264 | self.value |
| 265 | ) |
| 266 | |
| 267 | |
| 268 | @implements_iterator |
| 269 | class TokenStreamIterator(object): |
| 270 | """The iterator for tokenstreams. Iterate over the stream |
| 271 | until the eof token is reached. |
| 272 | """ |
| 273 | |
| 274 | def __init__(self, stream): |
| 275 | self.stream = stream |
| 276 | |
| 277 | def __iter__(self): |
| 278 | return self |
| 279 | |
| 280 | def __next__(self): |
| 281 | token = self.stream.current |
| 282 | if token.type is TOKEN_EOF: |
| 283 | self.stream.close() |
| 284 | raise StopIteration() |
| 285 | next(self.stream) |
| 286 | return token |
| 287 | |
| 288 | |
| 289 | @implements_iterator |
| 290 | class TokenStream(object): |
| 291 | """A token stream is an iterable that yields :class:`Token`\s. The |
| 292 | parser however does not iterate over it but calls :meth:`next` to go |
| 293 | one token ahead. The current active token is stored as :attr:`current`. |
| 294 | """ |
| 295 | |
| 296 | def __init__(self, generator, name, filename): |
| 297 | self._iter = iter(generator) |
| 298 | self._pushed = deque() |
| 299 | self.name = name |
| 300 | self.filename = filename |
| 301 | self.closed = False |
| 302 | self.current = Token(1, TOKEN_INITIAL, '') |
| 303 | next(self) |
| 304 | |
| 305 | def __iter__(self): |
| 306 | return TokenStreamIterator(self) |
| 307 | |
| 308 | def __bool__(self): |
| 309 | return bool(self._pushed) or self.current.type is not TOKEN_EOF |
| 310 | __nonzero__ = __bool__ # py2 |
| 311 | |
| 312 | eos = property(lambda x: not x, doc="Are we at the end of the stream?") |
| 313 | |
| 314 | def push(self, token): |
| 315 | """Push a token back to the stream.""" |
| 316 | self._pushed.append(token) |
| 317 | |
| 318 | def look(self): |
| 319 | """Look at the next token.""" |
| 320 | old_token = next(self) |
| 321 | result = self.current |
| 322 | self.push(result) |
| 323 | self.current = old_token |
| 324 | return result |
| 325 | |
| 326 | def skip(self, n=1): |
| 327 | """Got n tokens ahead.""" |
| 328 | for x in range(n): |
| 329 | next(self) |
| 330 | |
| 331 | def next_if(self, expr): |
| 332 | """Perform the token test and return the token if it matched. |
| 333 | Otherwise the return value is `None`. |
| 334 | """ |
| 335 | if self.current.test(expr): |
| 336 | return next(self) |
| 337 | |
| 338 | def skip_if(self, expr): |
| 339 | """Like :meth:`next_if` but only returns `True` or `False`.""" |
| 340 | return self.next_if(expr) is not None |
| 341 | |
| 342 | def __next__(self): |
| 343 | """Go one token ahead and return the old one""" |
| 344 | rv = self.current |
| 345 | if self._pushed: |
| 346 | self.current = self._pushed.popleft() |
| 347 | elif self.current.type is not TOKEN_EOF: |
| 348 | try: |
| 349 | self.current = next(self._iter) |
| 350 | except StopIteration: |
| 351 | self.close() |
| 352 | return rv |
| 353 | |
| 354 | def close(self): |
| 355 | """Close the stream.""" |
| 356 | self.current = Token(self.current.lineno, TOKEN_EOF, '') |
| 357 | self._iter = None |
| 358 | self.closed = True |
| 359 | |
| 360 | def expect(self, expr): |
| 361 | """Expect a given token type and return it. This accepts the same |
| 362 | argument as :meth:`jinja2.lexer.Token.test`. |
| 363 | """ |
| 364 | if not self.current.test(expr): |
| 365 | expr = describe_token_expr(expr) |
| 366 | if self.current.type is TOKEN_EOF: |
| 367 | raise TemplateSyntaxError('unexpected end of template, ' |
| 368 | 'expected %r.' % expr, |
| 369 | self.current.lineno, |
| 370 | self.name, self.filename) |
| 371 | raise TemplateSyntaxError("expected token %r, got %r" % |
| 372 | (expr, describe_token(self.current)), |
| 373 | self.current.lineno, |
| 374 | self.name, self.filename) |
| 375 | try: |
| 376 | return self.current |
| 377 | finally: |
| 378 | next(self) |
| 379 | |
| 380 | |
| 381 | def get_lexer(environment): |
| 382 | """Return a lexer which is probably cached.""" |
| 383 | key = (environment.block_start_string, |
| 384 | environment.block_end_string, |
| 385 | environment.variable_start_string, |
| 386 | environment.variable_end_string, |
| 387 | environment.comment_start_string, |
| 388 | environment.comment_end_string, |
| 389 | environment.line_statement_prefix, |
| 390 | environment.line_comment_prefix, |
| 391 | environment.trim_blocks, |
| 392 | environment.lstrip_blocks, |
| 393 | environment.newline_sequence, |
| 394 | environment.keep_trailing_newline) |
| 395 | lexer = _lexer_cache.get(key) |
| 396 | if lexer is None: |
| 397 | lexer = Lexer(environment) |
| 398 | _lexer_cache[key] = lexer |
| 399 | return lexer |
| 400 | |
| 401 | |
| 402 | class Lexer(object): |
| 403 | """Class that implements a lexer for a given environment. Automatically |
| 404 | created by the environment class, usually you don't have to do that. |
| 405 | |
| 406 | Note that the lexer is not automatically bound to an environment. |
| 407 | Multiple environments can share the same lexer. |
| 408 | """ |
| 409 | |
| 410 | def __init__(self, environment): |
| 411 | # shortcuts |
| 412 | c = lambda x: re.compile(x, re.M | re.S) |
| 413 | e = re.escape |
| 414 | |
| 415 | # lexing rules for tags |
| 416 | tag_rules = [ |
| 417 | (whitespace_re, TOKEN_WHITESPACE, None), |
| 418 | (float_re, TOKEN_FLOAT, None), |
| 419 | (integer_re, TOKEN_INTEGER, None), |
| 420 | (name_re, TOKEN_NAME, None), |
| 421 | (string_re, TOKEN_STRING, None), |
| 422 | (operator_re, TOKEN_OPERATOR, None) |
| 423 | ] |
| 424 | |
| 425 | # assemble the root lexing rule. because "|" is ungreedy |
| 426 | # we have to sort by length so that the lexer continues working |
| 427 | # as expected when we have parsing rules like <% for block and |
| 428 | # <%= for variables. (if someone wants asp like syntax) |
| 429 | # variables are just part of the rules if variable processing |
| 430 | # is required. |
| 431 | root_tag_rules = compile_rules(environment) |
| 432 | |
| 433 | # block suffix if trimming is enabled |
| 434 | block_suffix_re = environment.trim_blocks and '\\n?' or '' |
| 435 | |
| 436 | # strip leading spaces if lstrip_blocks is enabled |
| 437 | prefix_re = {} |
| 438 | if environment.lstrip_blocks: |
| 439 | # use '{%+' to manually disable lstrip_blocks behavior |
| 440 | no_lstrip_re = e('+') |
| 441 | # detect overlap between block and variable or comment strings |
| 442 | block_diff = c(r'^%s(.*)' % e(environment.block_start_string)) |
| 443 | # make sure we don't mistake a block for a variable or a comment |
| 444 | m = block_diff.match(environment.comment_start_string) |
| 445 | no_lstrip_re += m and r'|%s' % e(m.group(1)) or '' |
| 446 | m = block_diff.match(environment.variable_start_string) |
| 447 | no_lstrip_re += m and r'|%s' % e(m.group(1)) or '' |
| 448 | |
| 449 | # detect overlap between comment and variable strings |
| 450 | comment_diff = c(r'^%s(.*)' % e(environment.comment_start_string)) |
| 451 | m = comment_diff.match(environment.variable_start_string) |
| 452 | no_variable_re = m and r'(?!%s)' % e(m.group(1)) or '' |
| 453 | |
| 454 | lstrip_re = r'^[ \t]*' |
| 455 | block_prefix_re = r'%s%s(?!%s)|%s\+?' % ( |
| 456 | lstrip_re, |
| 457 | e(environment.block_start_string), |
| 458 | no_lstrip_re, |
| 459 | e(environment.block_start_string), |
| 460 | ) |
| 461 | comment_prefix_re = r'%s%s%s|%s\+?' % ( |
| 462 | lstrip_re, |
| 463 | e(environment.comment_start_string), |
| 464 | no_variable_re, |
| 465 | e(environment.comment_start_string), |
| 466 | ) |
| 467 | prefix_re['block'] = block_prefix_re |
| 468 | prefix_re['comment'] = comment_prefix_re |
| 469 | else: |
| 470 | block_prefix_re = '%s' % e(environment.block_start_string) |
| 471 | |
| 472 | self.newline_sequence = environment.newline_sequence |
| 473 | self.keep_trailing_newline = environment.keep_trailing_newline |
| 474 | |
| 475 | # global lexing rules |
| 476 | self.rules = { |
| 477 | 'root': [ |
| 478 | # directives |
| 479 | (c('(.*?)(?:%s)' % '|'.join( |
| 480 | [r'(?P<raw_begin>(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))' % ( |
| 481 | e(environment.block_start_string), |
| 482 | block_prefix_re, |
| 483 | e(environment.block_end_string), |
| 484 | e(environment.block_end_string) |
| 485 | )] + [ |
| 486 | r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, prefix_re.get(n,r)) |
| 487 | for n, r in root_tag_rules |
| 488 | ])), (TOKEN_DATA, '#bygroup'), '#bygroup'), |
| 489 | # data |
| 490 | (c('.+'), TOKEN_DATA, None) |
| 491 | ], |
| 492 | # comments |
| 493 | TOKEN_COMMENT_BEGIN: [ |
| 494 | (c(r'(.*?)((?:\-%s\s*|%s)%s)' % ( |
| 495 | e(environment.comment_end_string), |
| 496 | e(environment.comment_end_string), |
| 497 | block_suffix_re |
| 498 | )), (TOKEN_COMMENT, TOKEN_COMMENT_END), '#pop'), |
| 499 | (c('(.)'), (Failure('Missing end of comment tag'),), None) |
| 500 | ], |
| 501 | # blocks |
| 502 | TOKEN_BLOCK_BEGIN: [ |
| 503 | (c('(?:\-%s\s*|%s)%s' % ( |
| 504 | e(environment.block_end_string), |
| 505 | e(environment.block_end_string), |
| 506 | block_suffix_re |
| 507 | )), TOKEN_BLOCK_END, '#pop'), |
| 508 | ] + tag_rules, |
| 509 | # variables |
| 510 | TOKEN_VARIABLE_BEGIN: [ |
| 511 | (c('\-%s\s*|%s' % ( |
| 512 | e(environment.variable_end_string), |
| 513 | e(environment.variable_end_string) |
| 514 | )), TOKEN_VARIABLE_END, '#pop') |
| 515 | ] + tag_rules, |
| 516 | # raw block |
| 517 | TOKEN_RAW_BEGIN: [ |
| 518 | (c('(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % ( |
| 519 | e(environment.block_start_string), |
| 520 | block_prefix_re, |
| 521 | e(environment.block_end_string), |
| 522 | e(environment.block_end_string), |
| 523 | block_suffix_re |
| 524 | )), (TOKEN_DATA, TOKEN_RAW_END), '#pop'), |
| 525 | (c('(.)'), (Failure('Missing end of raw directive'),), None) |
| 526 | ], |
| 527 | # line statements |
| 528 | TOKEN_LINESTATEMENT_BEGIN: [ |
| 529 | (c(r'\s*(\n|$)'), TOKEN_LINESTATEMENT_END, '#pop') |
| 530 | ] + tag_rules, |
| 531 | # line comments |
| 532 | TOKEN_LINECOMMENT_BEGIN: [ |
| 533 | (c(r'(.*?)()(?=\n|$)'), (TOKEN_LINECOMMENT, |
| 534 | TOKEN_LINECOMMENT_END), '#pop') |
| 535 | ] |
| 536 | } |
| 537 | |
| 538 | def _normalize_newlines(self, value): |
| 539 | """Called for strings and template data to normalize it to unicode.""" |
| 540 | return newline_re.sub(self.newline_sequence, value) |
| 541 | |
| 542 | def tokenize(self, source, name=None, filename=None, state=None): |
| 543 | """Calls tokeniter + tokenize and wraps it in a token stream. |
| 544 | """ |
| 545 | stream = self.tokeniter(source, name, filename, state) |
| 546 | return TokenStream(self.wrap(stream, name, filename), name, filename) |
| 547 | |
| 548 | def wrap(self, stream, name=None, filename=None): |
| 549 | """This is called with the stream as returned by `tokenize` and wraps |
| 550 | every token in a :class:`Token` and converts the value. |
| 551 | """ |
| 552 | for lineno, token, value in stream: |
| 553 | if token in ignored_tokens: |
| 554 | continue |
| 555 | elif token == 'linestatement_begin': |
| 556 | token = 'block_begin' |
| 557 | elif token == 'linestatement_end': |
| 558 | token = 'block_end' |
| 559 | # we are not interested in those tokens in the parser |
| 560 | elif token in ('raw_begin', 'raw_end'): |
| 561 | continue |
| 562 | elif token == 'data': |
| 563 | value = self._normalize_newlines(value) |
| 564 | elif token == 'keyword': |
| 565 | token = value |
| 566 | elif token == 'name': |
| 567 | value = str(value) |
| 568 | elif token == 'string': |
| 569 | # try to unescape string |
| 570 | try: |
| 571 | value = self._normalize_newlines(value[1:-1]) \ |
| 572 | .encode('ascii', 'backslashreplace') \ |
| 573 | .decode('unicode-escape') |
| 574 | except Exception as e: |
| 575 | msg = str(e).split(':')[-1].strip() |
| 576 | raise TemplateSyntaxError(msg, lineno, name, filename) |
| 577 | # if we can express it as bytestring (ascii only) |
| 578 | # we do that for support of semi broken APIs |
| 579 | # as datetime.datetime.strftime. On python 3 this |
| 580 | # call becomes a noop thanks to 2to3 |
| 581 | try: |
| 582 | value = str(value) |
| 583 | except UnicodeError: |
| 584 | pass |
| 585 | elif token == 'integer': |
| 586 | value = int(value) |
| 587 | elif token == 'float': |
| 588 | value = float(value) |
| 589 | elif token == 'operator': |
| 590 | token = operators[value] |
| 591 | yield Token(lineno, token, value) |
| 592 | |
| 593 | def tokeniter(self, source, name, filename=None, state=None): |
| 594 | """This method tokenizes the text and returns the tokens in a |
| 595 | generator. Use this method if you just want to tokenize a template. |
| 596 | """ |
| 597 | source = text_type(source) |
| 598 | lines = source.splitlines() |
| 599 | if self.keep_trailing_newline and source: |
| 600 | for newline in ('\r\n', '\r', '\n'): |
| 601 | if source.endswith(newline): |
| 602 | lines.append('') |
| 603 | break |
| 604 | source = '\n'.join(lines) |
| 605 | pos = 0 |
| 606 | lineno = 1 |
| 607 | stack = ['root'] |
| 608 | if state is not None and state != 'root': |
| 609 | assert state in ('variable', 'block'), 'invalid state' |
| 610 | stack.append(state + '_begin') |
| 611 | else: |
| 612 | state = 'root' |
| 613 | statetokens = self.rules[stack[-1]] |
| 614 | source_length = len(source) |
| 615 | |
| 616 | balancing_stack = [] |
| 617 | |
| 618 | while 1: |
| 619 | # tokenizer loop |
| 620 | for regex, tokens, new_state in statetokens: |
| 621 | m = regex.match(source, pos) |
| 622 | # if no match we try again with the next rule |
| 623 | if m is None: |
| 624 | continue |
| 625 | |
| 626 | # we only match blocks and variables if braces / parentheses |
| 627 | # are balanced. continue parsing with the lower rule which |
| 628 | # is the operator rule. do this only if the end tags look |
| 629 | # like operators |
| 630 | if balancing_stack and \ |
| 631 | tokens in ('variable_end', 'block_end', |
| 632 | 'linestatement_end'): |
| 633 | continue |
| 634 | |
| 635 | # tuples support more options |
| 636 | if isinstance(tokens, tuple): |
| 637 | for idx, token in enumerate(tokens): |
| 638 | # failure group |
| 639 | if token.__class__ is Failure: |
| 640 | raise token(lineno, filename) |
| 641 | # bygroup is a bit more complex, in that case we |
| 642 | # yield for the current token the first named |
| 643 | # group that matched |
| 644 | elif token == '#bygroup': |
| 645 | for key, value in iteritems(m.groupdict()): |
| 646 | if value is not None: |
| 647 | yield lineno, key, value |
| 648 | lineno += value.count('\n') |
| 649 | break |
| 650 | else: |
| 651 | raise RuntimeError('%r wanted to resolve ' |
| 652 | 'the token dynamically' |
| 653 | ' but no group matched' |
| 654 | % regex) |
| 655 | # normal group |
| 656 | else: |
| 657 | data = m.group(idx + 1) |
| 658 | if data or token not in ignore_if_empty: |
| 659 | yield lineno, token, data |
| 660 | lineno += data.count('\n') |
| 661 | |
| 662 | # strings as token just are yielded as it. |
| 663 | else: |
| 664 | data = m.group() |
| 665 | # update brace/parentheses balance |
| 666 | if tokens == 'operator': |
| 667 | if data == '{': |
| 668 | balancing_stack.append('}') |
| 669 | elif data == '(': |
| 670 | balancing_stack.append(')') |
| 671 | elif data == '[': |
| 672 | balancing_stack.append(']') |
| 673 | elif data in ('}', ')', ']'): |
| 674 | if not balancing_stack: |
| 675 | raise TemplateSyntaxError('unexpected \'%s\'' % |
| 676 | data, lineno, name, |
| 677 | filename) |
| 678 | expected_op = balancing_stack.pop() |
| 679 | if expected_op != data: |
| 680 | raise TemplateSyntaxError('unexpected \'%s\', ' |
| 681 | 'expected \'%s\'' % |
| 682 | (data, expected_op), |
| 683 | lineno, name, |
| 684 | filename) |
| 685 | # yield items |
| 686 | if data or tokens not in ignore_if_empty: |
| 687 | yield lineno, tokens, data |
| 688 | lineno += data.count('\n') |
| 689 | |
| 690 | # fetch new position into new variable so that we can check |
| 691 | # if there is a internal parsing error which would result |
| 692 | # in an infinite loop |
| 693 | pos2 = m.end() |
| 694 | |
| 695 | # handle state changes |
| 696 | if new_state is not None: |
| 697 | # remove the uppermost state |
| 698 | if new_state == '#pop': |
| 699 | stack.pop() |
| 700 | # resolve the new state by group checking |
| 701 | elif new_state == '#bygroup': |
| 702 | for key, value in iteritems(m.groupdict()): |
| 703 | if value is not None: |
| 704 | stack.append(key) |
| 705 | break |
| 706 | else: |
| 707 | raise RuntimeError('%r wanted to resolve the ' |
| 708 | 'new state dynamically but' |
| 709 | ' no group matched' % |
| 710 | regex) |
| 711 | # direct state name given |
| 712 | else: |
| 713 | stack.append(new_state) |
| 714 | statetokens = self.rules[stack[-1]] |
| 715 | # we are still at the same position and no stack change. |
| 716 | # this means a loop without break condition, avoid that and |
| 717 | # raise error |
| 718 | elif pos2 == pos: |
| 719 | raise RuntimeError('%r yielded empty string without ' |
| 720 | 'stack change' % regex) |
| 721 | # publish new function and start again |
| 722 | pos = pos2 |
| 723 | break |
| 724 | # if loop terminated without break we haven't found a single match |
| 725 | # either we are at the end of the file or we have a problem |
| 726 | else: |
| 727 | # end of text |
| 728 | if pos >= source_length: |
| 729 | return |
| 730 | # something went wrong |
| 731 | raise TemplateSyntaxError('unexpected char %r at %d' % |
| 732 | (source[pos], pos), lineno, |
| 733 | name, filename) |