diff options
Diffstat (limited to 'lib/python2.7/site-packages/Jinja2-2.6-py2.7.egg/jinja2/lexer.py')
| -rwxr-xr-x | lib/python2.7/site-packages/Jinja2-2.6-py2.7.egg/jinja2/lexer.py | 681 |
1 files changed, 0 insertions, 681 deletions
diff --git a/lib/python2.7/site-packages/Jinja2-2.6-py2.7.egg/jinja2/lexer.py b/lib/python2.7/site-packages/Jinja2-2.6-py2.7.egg/jinja2/lexer.py deleted file mode 100755 index 0d3f6961..00000000 --- a/lib/python2.7/site-packages/Jinja2-2.6-py2.7.egg/jinja2/lexer.py +++ /dev/null @@ -1,681 +0,0 @@ -# -*- coding: utf-8 -*- -""" - jinja2.lexer - ~~~~~~~~~~~~ - - This module implements a Jinja / Python combination lexer. The - `Lexer` class provided by this module is used to do some preprocessing - for Jinja. - - On the one hand it filters out invalid operators like the bitshift - operators we don't allow in templates. On the other hand it separates - template code and python code in expressions. - - :copyright: (c) 2010 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -import re -from operator import itemgetter -from collections import deque -from jinja2.exceptions import TemplateSyntaxError -from jinja2.utils import LRUCache, next - - -# cache for the lexers. Exists in order to be able to have multiple -# environments with the same lexer -_lexer_cache = LRUCache(50) - -# static regular expressions -whitespace_re = re.compile(r'\s+', re.U) -string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'" - r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S) -integer_re = re.compile(r'\d+') - -# we use the unicode identifier rule if this python version is able -# to handle unicode identifiers, otherwise the standard ASCII one. -try: - compile('föö', '<unknown>', 'eval') -except SyntaxError: - name_re = re.compile(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b') -else: - from jinja2 import _stringdefs - name_re = re.compile(r'[%s][%s]*' % (_stringdefs.xid_start, - _stringdefs.xid_continue)) - -float_re = re.compile(r'(?<!\.)\d+\.\d+') -newline_re = re.compile(r'(\r\n|\r|\n)') - -# internal the tokens and keep references to them -TOKEN_ADD = intern('add') -TOKEN_ASSIGN = intern('assign') -TOKEN_COLON = intern('colon') -TOKEN_COMMA = intern('comma') -TOKEN_DIV = intern('div') -TOKEN_DOT = intern('dot') -TOKEN_EQ = intern('eq') -TOKEN_FLOORDIV = intern('floordiv') -TOKEN_GT = intern('gt') -TOKEN_GTEQ = intern('gteq') -TOKEN_LBRACE = intern('lbrace') -TOKEN_LBRACKET = intern('lbracket') -TOKEN_LPAREN = intern('lparen') -TOKEN_LT = intern('lt') -TOKEN_LTEQ = intern('lteq') -TOKEN_MOD = intern('mod') -TOKEN_MUL = intern('mul') -TOKEN_NE = intern('ne') -TOKEN_PIPE = intern('pipe') -TOKEN_POW = intern('pow') -TOKEN_RBRACE = intern('rbrace') -TOKEN_RBRACKET = intern('rbracket') -TOKEN_RPAREN = intern('rparen') -TOKEN_SEMICOLON = intern('semicolon') -TOKEN_SUB = intern('sub') -TOKEN_TILDE = intern('tilde') -TOKEN_WHITESPACE = intern('whitespace') -TOKEN_FLOAT = intern('float') -TOKEN_INTEGER = intern('integer') -TOKEN_NAME = intern('name') -TOKEN_STRING = intern('string') -TOKEN_OPERATOR = intern('operator') -TOKEN_BLOCK_BEGIN = intern('block_begin') -TOKEN_BLOCK_END = intern('block_end') -TOKEN_VARIABLE_BEGIN = intern('variable_begin') -TOKEN_VARIABLE_END = intern('variable_end') -TOKEN_RAW_BEGIN = intern('raw_begin') -TOKEN_RAW_END = intern('raw_end') -TOKEN_COMMENT_BEGIN = intern('comment_begin') -TOKEN_COMMENT_END = intern('comment_end') -TOKEN_COMMENT = intern('comment') -TOKEN_LINESTATEMENT_BEGIN = intern('linestatement_begin') -TOKEN_LINESTATEMENT_END = intern('linestatement_end') -TOKEN_LINECOMMENT_BEGIN = intern('linecomment_begin') -TOKEN_LINECOMMENT_END = intern('linecomment_end') -TOKEN_LINECOMMENT = intern('linecomment') -TOKEN_DATA = intern('data') -TOKEN_INITIAL = intern('initial') -TOKEN_EOF = intern('eof') - -# bind operators to token types -operators = { - '+': TOKEN_ADD, - '-': TOKEN_SUB, - '/': TOKEN_DIV, - '//': TOKEN_FLOORDIV, - '*': TOKEN_MUL, - '%': TOKEN_MOD, - '**': TOKEN_POW, - '~': TOKEN_TILDE, - '[': TOKEN_LBRACKET, - ']': TOKEN_RBRACKET, - '(': TOKEN_LPAREN, - ')': TOKEN_RPAREN, - '{': TOKEN_LBRACE, - '}': TOKEN_RBRACE, - '==': TOKEN_EQ, - '!=': TOKEN_NE, - '>': TOKEN_GT, - '>=': TOKEN_GTEQ, - '<': TOKEN_LT, - '<=': TOKEN_LTEQ, - '=': TOKEN_ASSIGN, - '.': TOKEN_DOT, - ':': TOKEN_COLON, - '|': TOKEN_PIPE, - ',': TOKEN_COMMA, - ';': TOKEN_SEMICOLON -} - -reverse_operators = dict([(v, k) for k, v in operators.iteritems()]) -assert len(operators) == len(reverse_operators), 'operators dropped' -operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in - sorted(operators, key=lambda x: -len(x)))) - -ignored_tokens = frozenset([TOKEN_COMMENT_BEGIN, TOKEN_COMMENT, - TOKEN_COMMENT_END, TOKEN_WHITESPACE, - TOKEN_WHITESPACE, TOKEN_LINECOMMENT_BEGIN, - TOKEN_LINECOMMENT_END, TOKEN_LINECOMMENT]) -ignore_if_empty = frozenset([TOKEN_WHITESPACE, TOKEN_DATA, - TOKEN_COMMENT, TOKEN_LINECOMMENT]) - - -def _describe_token_type(token_type): - if token_type in reverse_operators: - return reverse_operators[token_type] - return { - TOKEN_COMMENT_BEGIN: 'begin of comment', - TOKEN_COMMENT_END: 'end of comment', - TOKEN_COMMENT: 'comment', - TOKEN_LINECOMMENT: 'comment', - TOKEN_BLOCK_BEGIN: 'begin of statement block', - TOKEN_BLOCK_END: 'end of statement block', - TOKEN_VARIABLE_BEGIN: 'begin of print statement', - TOKEN_VARIABLE_END: 'end of print statement', - TOKEN_LINESTATEMENT_BEGIN: 'begin of line statement', - TOKEN_LINESTATEMENT_END: 'end of line statement', - TOKEN_DATA: 'template data / text', - TOKEN_EOF: 'end of template' - }.get(token_type, token_type) - - -def describe_token(token): - """Returns a description of the token.""" - if token.type == 'name': - return token.value - return _describe_token_type(token.type) - - -def describe_token_expr(expr): - """Like `describe_token` but for token expressions.""" - if ':' in expr: - type, value = expr.split(':', 1) - if type == 'name': - return value - else: - type = expr - return _describe_token_type(type) - - -def count_newlines(value): - """Count the number of newline characters in the string. This is - useful for extensions that filter a stream. - """ - return len(newline_re.findall(value)) - - -def compile_rules(environment): - """Compiles all the rules from the environment into a list of rules.""" - e = re.escape - rules = [ - (len(environment.comment_start_string), 'comment', - e(environment.comment_start_string)), - (len(environment.block_start_string), 'block', - e(environment.block_start_string)), - (len(environment.variable_start_string), 'variable', - e(environment.variable_start_string)) - ] - - if environment.line_statement_prefix is not None: - rules.append((len(environment.line_statement_prefix), 'linestatement', - r'^\s*' + e(environment.line_statement_prefix))) - if environment.line_comment_prefix is not None: - rules.append((len(environment.line_comment_prefix), 'linecomment', - r'(?:^|(?<=\S))[^\S\r\n]*' + - e(environment.line_comment_prefix))) - - return [x[1:] for x in sorted(rules, reverse=True)] - - -class Failure(object): - """Class that raises a `TemplateSyntaxError` if called. - Used by the `Lexer` to specify known errors. - """ - - def __init__(self, message, cls=TemplateSyntaxError): - self.message = message - self.error_class = cls - - def __call__(self, lineno, filename): - raise self.error_class(self.message, lineno, filename) - - -class Token(tuple): - """Token class.""" - __slots__ = () - lineno, type, value = (property(itemgetter(x)) for x in range(3)) - - def __new__(cls, lineno, type, value): - return tuple.__new__(cls, (lineno, intern(str(type)), value)) - - def __str__(self): - if self.type in reverse_operators: - return reverse_operators[self.type] - elif self.type == 'name': - return self.value - return self.type - - def test(self, expr): - """Test a token against a token expression. This can either be a - token type or ``'token_type:token_value'``. This can only test - against string values and types. - """ - # here we do a regular string equality check as test_any is usually - # passed an iterable of not interned strings. - if self.type == expr: - return True - elif ':' in expr: - return expr.split(':', 1) == [self.type, self.value] - return False - - def test_any(self, *iterable): - """Test against multiple token expressions.""" - for expr in iterable: - if self.test(expr): - return True - return False - - def __repr__(self): - return 'Token(%r, %r, %r)' % ( - self.lineno, - self.type, - self.value - ) - - -class TokenStreamIterator(object): - """The iterator for tokenstreams. Iterate over the stream - until the eof token is reached. - """ - - def __init__(self, stream): - self.stream = stream - - def __iter__(self): - return self - - def next(self): - token = self.stream.current - if token.type is TOKEN_EOF: - self.stream.close() - raise StopIteration() - next(self.stream) - return token - - -class TokenStream(object): - """A token stream is an iterable that yields :class:`Token`\s. The - parser however does not iterate over it but calls :meth:`next` to go - one token ahead. The current active token is stored as :attr:`current`. - """ - - def __init__(self, generator, name, filename): - self._next = iter(generator).next - self._pushed = deque() - self.name = name - self.filename = filename - self.closed = False - self.current = Token(1, TOKEN_INITIAL, '') - next(self) - - def __iter__(self): - return TokenStreamIterator(self) - - def __nonzero__(self): - return bool(self._pushed) or self.current.type is not TOKEN_EOF - - eos = property(lambda x: not x, doc="Are we at the end of the stream?") - - def push(self, token): - """Push a token back to the stream.""" - self._pushed.append(token) - - def look(self): - """Look at the next token.""" - old_token = next(self) - result = self.current - self.push(result) - self.current = old_token - return result - - def skip(self, n=1): - """Got n tokens ahead.""" - for x in xrange(n): - next(self) - - def next_if(self, expr): - """Perform the token test and return the token if it matched. - Otherwise the return value is `None`. - """ - if self.current.test(expr): - return next(self) - - def skip_if(self, expr): - """Like :meth:`next_if` but only returns `True` or `False`.""" - return self.next_if(expr) is not None - - def next(self): - """Go one token ahead and return the old one""" - rv = self.current - if self._pushed: - self.current = self._pushed.popleft() - elif self.current.type is not TOKEN_EOF: - try: - self.current = self._next() - except StopIteration: - self.close() - return rv - - def close(self): - """Close the stream.""" - self.current = Token(self.current.lineno, TOKEN_EOF, '') - self._next = None - self.closed = True - - def expect(self, expr): - """Expect a given token type and return it. This accepts the same - argument as :meth:`jinja2.lexer.Token.test`. - """ - if not self.current.test(expr): - expr = describe_token_expr(expr) - if self.current.type is TOKEN_EOF: - raise TemplateSyntaxError('unexpected end of template, ' - 'expected %r.' % expr, - self.current.lineno, - self.name, self.filename) - raise TemplateSyntaxError("expected token %r, got %r" % - (expr, describe_token(self.current)), - self.current.lineno, - self.name, self.filename) - try: - return self.current - finally: - next(self) - - -def get_lexer(environment): - """Return a lexer which is probably cached.""" - key = (environment.block_start_string, - environment.block_end_string, - environment.variable_start_string, - environment.variable_end_string, - environment.comment_start_string, - environment.comment_end_string, - environment.line_statement_prefix, - environment.line_comment_prefix, - environment.trim_blocks, - environment.newline_sequence) - lexer = _lexer_cache.get(key) - if lexer is None: - lexer = Lexer(environment) - _lexer_cache[key] = lexer - return lexer - - -class Lexer(object): - """Class that implements a lexer for a given environment. Automatically - created by the environment class, usually you don't have to do that. - - Note that the lexer is not automatically bound to an environment. - Multiple environments can share the same lexer. - """ - - def __init__(self, environment): - # shortcuts - c = lambda x: re.compile(x, re.M | re.S) - e = re.escape - - # lexing rules for tags - tag_rules = [ - (whitespace_re, TOKEN_WHITESPACE, None), - (float_re, TOKEN_FLOAT, None), - (integer_re, TOKEN_INTEGER, None), - (name_re, TOKEN_NAME, None), - (string_re, TOKEN_STRING, None), - (operator_re, TOKEN_OPERATOR, None) - ] - - # assamble the root lexing rule. because "|" is ungreedy - # we have to sort by length so that the lexer continues working - # as expected when we have parsing rules like <% for block and - # <%= for variables. (if someone wants asp like syntax) - # variables are just part of the rules if variable processing - # is required. - root_tag_rules = compile_rules(environment) - - # block suffix if trimming is enabled - block_suffix_re = environment.trim_blocks and '\\n?' or '' - - self.newline_sequence = environment.newline_sequence - - # global lexing rules - self.rules = { - 'root': [ - # directives - (c('(.*?)(?:%s)' % '|'.join( - [r'(?P<raw_begin>(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))' % ( - e(environment.block_start_string), - e(environment.block_start_string), - e(environment.block_end_string), - e(environment.block_end_string) - )] + [ - r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, r) - for n, r in root_tag_rules - ])), (TOKEN_DATA, '#bygroup'), '#bygroup'), - # data - (c('.+'), TOKEN_DATA, None) - ], - # comments - TOKEN_COMMENT_BEGIN: [ - (c(r'(.*?)((?:\-%s\s*|%s)%s)' % ( - e(environment.comment_end_string), - e(environment.comment_end_string), - block_suffix_re - )), (TOKEN_COMMENT, TOKEN_COMMENT_END), '#pop'), - (c('(.)'), (Failure('Missing end of comment tag'),), None) - ], - # blocks - TOKEN_BLOCK_BEGIN: [ - (c('(?:\-%s\s*|%s)%s' % ( - e(environment.block_end_string), - e(environment.block_end_string), - block_suffix_re - )), TOKEN_BLOCK_END, '#pop'), - ] + tag_rules, - # variables - TOKEN_VARIABLE_BEGIN: [ - (c('\-%s\s*|%s' % ( - e(environment.variable_end_string), - e(environment.variable_end_string) - )), TOKEN_VARIABLE_END, '#pop') - ] + tag_rules, - # raw block - TOKEN_RAW_BEGIN: [ - (c('(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % ( - e(environment.block_start_string), - e(environment.block_start_string), - e(environment.block_end_string), - e(environment.block_end_string), - block_suffix_re - )), (TOKEN_DATA, TOKEN_RAW_END), '#pop'), - (c('(.)'), (Failure('Missing end of raw directive'),), None) - ], - # line statements - TOKEN_LINESTATEMENT_BEGIN: [ - (c(r'\s*(\n|$)'), TOKEN_LINESTATEMENT_END, '#pop') - ] + tag_rules, - # line comments - TOKEN_LINECOMMENT_BEGIN: [ - (c(r'(.*?)()(?=\n|$)'), (TOKEN_LINECOMMENT, - TOKEN_LINECOMMENT_END), '#pop') - ] - } - - def _normalize_newlines(self, value): - """Called for strings and template data to normlize it to unicode.""" - return newline_re.sub(self.newline_sequence, value) - - def tokenize(self, source, name=None, filename=None, state=None): - """Calls tokeniter + tokenize and wraps it in a token stream. - """ - stream = self.tokeniter(source, name, filename, state) - return TokenStream(self.wrap(stream, name, filename), name, filename) - - def wrap(self, stream, name=None, filename=None): - """This is called with the stream as returned by `tokenize` and wraps - every token in a :class:`Token` and converts the value. - """ - for lineno, token, value in stream: - if token in ignored_tokens: - continue - elif token == 'linestatement_begin': - token = 'block_begin' - elif token == 'linestatement_end': - token = 'block_end' - # we are not interested in those tokens in the parser - elif token in ('raw_begin', 'raw_end'): - continue - elif token == 'data': - value = self._normalize_newlines(value) - elif token == 'keyword': - token = value - elif token == 'name': - value = str(value) - elif token == 'string': - # try to unescape string - try: - value = self._normalize_newlines(value[1:-1]) \ - .encode('ascii', 'backslashreplace') \ - .decode('unicode-escape') - except Exception, e: - msg = str(e).split(':')[-1].strip() - raise TemplateSyntaxError(msg, lineno, name, filename) - # if we can express it as bytestring (ascii only) - # we do that for support of semi broken APIs - # as datetime.datetime.strftime. On python 3 this - # call becomes a noop thanks to 2to3 - try: - value = str(value) - except UnicodeError: - pass - elif token == 'integer': - value = int(value) - elif token == 'float': - value = float(value) - elif token == 'operator': - token = operators[value] - yield Token(lineno, token, value) - - def tokeniter(self, source, name, filename=None, state=None): - """This method tokenizes the text and returns the tokens in a - generator. Use this method if you just want to tokenize a template. - """ - source = '\n'.join(unicode(source).splitlines()) - pos = 0 - lineno = 1 - stack = ['root'] - if state is not None and state != 'root': - assert state in ('variable', 'block'), 'invalid state' - stack.append(state + '_begin') - else: - state = 'root' - statetokens = self.rules[stack[-1]] - source_length = len(source) - - balancing_stack = [] - - while 1: - # tokenizer loop - for regex, tokens, new_state in statetokens: - m = regex.match(source, pos) - # if no match we try again with the next rule - if m is None: - continue - - # we only match blocks and variables if brances / parentheses - # are balanced. continue parsing with the lower rule which - # is the operator rule. do this only if the end tags look - # like operators - if balancing_stack and \ - tokens in ('variable_end', 'block_end', - 'linestatement_end'): - continue - - # tuples support more options - if isinstance(tokens, tuple): - for idx, token in enumerate(tokens): - # failure group - if token.__class__ is Failure: - raise token(lineno, filename) - # bygroup is a bit more complex, in that case we - # yield for the current token the first named - # group that matched - elif token == '#bygroup': - for key, value in m.groupdict().iteritems(): - if value is not None: - yield lineno, key, value - lineno += value.count('\n') - break - else: - raise RuntimeError('%r wanted to resolve ' - 'the token dynamically' - ' but no group matched' - % regex) - # normal group - else: - data = m.group(idx + 1) - if data or token not in ignore_if_empty: - yield lineno, token, data - lineno += data.count('\n') - - # strings as token just are yielded as it. - else: - data = m.group() - # update brace/parentheses balance - if tokens == 'operator': - if data == '{': - balancing_stack.append('}') - elif data == '(': - balancing_stack.append(')') - elif data == '[': - balancing_stack.append(']') - elif data in ('}', ')', ']'): - if not balancing_stack: - raise TemplateSyntaxError('unexpected \'%s\'' % - data, lineno, name, - filename) - expected_op = balancing_stack.pop() - if expected_op != data: - raise TemplateSyntaxError('unexpected \'%s\', ' - 'expected \'%s\'' % - (data, expected_op), - lineno, name, - filename) - # yield items - if data or tokens not in ignore_if_empty: - yield lineno, tokens, data - lineno += data.count('\n') - - # fetch new position into new variable so that we can check - # if there is a internal parsing error which would result - # in an infinite loop - pos2 = m.end() - - # handle state changes - if new_state is not None: - # remove the uppermost state - if new_state == '#pop': - stack.pop() - # resolve the new state by group checking - elif new_state == '#bygroup': - for key, value in m.groupdict().iteritems(): - if value is not None: - stack.append(key) - break - else: - raise RuntimeError('%r wanted to resolve the ' - 'new state dynamically but' - ' no group matched' % - regex) - # direct state name given - else: - stack.append(new_state) - statetokens = self.rules[stack[-1]] - # we are still at the same position and no stack change. - # this means a loop without break condition, avoid that and - # raise error - elif pos2 == pos: - raise RuntimeError('%r yielded empty string without ' - 'stack change' % regex) - # publish new function and start again - pos = pos2 - break - # if loop terminated without break we havn't found a single match - # either we are at the end of the file or we have a problem - else: - # end of text - if pos >= source_length: - return - # something went wrong - raise TemplateSyntaxError('unexpected char %r at %d' % - (source[pos], pos), lineno, - name, filename) |