a33b47e485
- adds `jsgrammar.LINETERMINATORSEQ_RE` - lexer `tstream.TokenStream` checks for lineterminators in tokens - adds `tstream.Token` - refractors `tstream.TokenStream` and `jsparser.Parser` and to use it
838 lines
34 KiB
Python
838 lines
34 KiB
Python
from __future__ import unicode_literals
|
|
|
|
from .jsgrammar import TokenTypes, token_keys
|
|
from .tstream import TokenStream, convert_to_unary
|
|
from ..utils import ExtractorError
|
|
|
|
|
|
class Parser(object):
|
|
|
|
def __init__(self, code, pos=0, stack_size=100):
|
|
super(Parser, self).__init__()
|
|
self.token_stream = TokenStream(code, pos)
|
|
self.stack_top = stack_size
|
|
self._no_in = True
|
|
|
|
def parse(self):
|
|
while not self.token_stream.ended:
|
|
yield self._source_element(self.stack_top)
|
|
|
|
def _source_element(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
token = self.token_stream.peek()
|
|
if token.id is TokenTypes.ID and token.value == 'function':
|
|
source_element = self._function(stack_top - 1)
|
|
else:
|
|
source_element = self._statement(stack_top - 1)
|
|
|
|
return source_element
|
|
|
|
def _statement(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
statement = None
|
|
token = self.token_stream.peek()
|
|
if token.id is TokenTypes.END:
|
|
# empty statement goes straight here
|
|
self.token_stream.pop()
|
|
return statement
|
|
|
|
# block
|
|
elif token.id is TokenTypes.COPEN:
|
|
# XXX refactor will deprecate some _statement calls
|
|
open_pos = token.pos
|
|
self.token_stream.pop()
|
|
block = []
|
|
while True:
|
|
token = self.token_stream.peek()
|
|
if token.id is TokenTypes.CCLOSE:
|
|
self.token_stream.pop()
|
|
break
|
|
elif token.id is TokenTypes.END and self.token_stream.ended:
|
|
raise ExtractorError('Unbalanced parentheses at %d' % open_pos)
|
|
block.append(self._statement(stack_top - 1))
|
|
|
|
statement = (TokenTypes.BLOCK, block)
|
|
|
|
elif token.id is TokenTypes.ID:
|
|
if token.value == 'var':
|
|
self.token_stream.pop()
|
|
variables = []
|
|
init = []
|
|
has_another = True
|
|
while has_another:
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.ID:
|
|
raise ExtractorError('Missing variable name at %d' % token.pos)
|
|
self.token_stream.chk_id(last=True)
|
|
variables.append(token.value)
|
|
|
|
peek = self.token_stream.peek()
|
|
if peek.id is TokenTypes.AOP:
|
|
self.token_stream.pop()
|
|
init.append(self._assign_expression(stack_top - 1))
|
|
peek = self.token_stream.peek()
|
|
else:
|
|
init.append(None)
|
|
|
|
if peek.id is TokenTypes.END:
|
|
if self._no_in:
|
|
self.token_stream.pop()
|
|
has_another = False
|
|
elif peek.id is TokenTypes.COMMA:
|
|
# TODO for not NoIn
|
|
pass
|
|
else:
|
|
# FIXME automatic end insertion
|
|
# - token_id is Token.CCLOSE
|
|
# - check line terminator
|
|
# - restricted token
|
|
raise ExtractorError('Unexpected sequence at %d' % peek.pos)
|
|
statement = (TokenTypes.VAR, zip(variables, init))
|
|
|
|
elif token.value == 'if':
|
|
statement = self._if_statement(stack_top - 1)
|
|
|
|
elif token.value == 'for':
|
|
statement = self._for_loop(stack_top - 1)
|
|
|
|
elif token.value == 'do':
|
|
statement = self._do_loop(stack_top - 1)
|
|
|
|
elif token.value == 'while':
|
|
statement = self._while_loop(stack_top - 1)
|
|
|
|
elif token.value in ('break', 'continue'):
|
|
self.token_stream.pop()
|
|
token = {'break': TokenTypes.BREAK, 'continue': TokenTypes.CONTINUE}[token.value]
|
|
peek = self.token_stream.peek()
|
|
# XXX no line break here
|
|
label_name = None
|
|
if peek.id is not TokenTypes.END:
|
|
self.token_stream.chk_id()
|
|
label_name = peek.value
|
|
self.token_stream.pop()
|
|
statement = (token, label_name)
|
|
peek = self.token_stream.peek()
|
|
if peek.id is TokenTypes.END:
|
|
self.token_stream.pop()
|
|
else:
|
|
# FIXME automatic end insertion
|
|
raise ExtractorError('Unexpected sequence at %d' % peek.pos)
|
|
|
|
elif token.value == 'return':
|
|
statement = self._return_statement(stack_top - 1)
|
|
peek = self.token_stream.peek()
|
|
if peek.id is TokenTypes.END:
|
|
self.token_stream.pop()
|
|
else:
|
|
# FIXME automatic end insertion
|
|
raise ExtractorError('Unexpected sequence at %d' % peek.pos)
|
|
|
|
elif token.value == 'with':
|
|
statement = self._with_statement(stack_top - 1)
|
|
|
|
elif token.value == 'switch':
|
|
statement = self._switch_statement(stack_top - 1)
|
|
|
|
elif token.value == 'throw':
|
|
self.token_stream.pop()
|
|
# XXX no line break here
|
|
expr = self._expression(stack_top - 1)
|
|
statement = (TokenTypes.RETURN, expr)
|
|
peek = self.token_stream.peek()
|
|
if peek.id is TokenTypes.END:
|
|
self.token_stream.pop()
|
|
else:
|
|
# FIXME automatic end insertion
|
|
raise ExtractorError('Unexpected sequence at %d' % peek.pos)
|
|
|
|
elif token.value == 'try':
|
|
statement = self._try_statement(stack_top - 1)
|
|
|
|
elif token.value == 'debugger':
|
|
self.token_stream.pop()
|
|
statement = (TokenTypes.DEBUG,)
|
|
peek = self.token_stream.peek()
|
|
if peek.id is TokenTypes.END:
|
|
self.token_stream.pop()
|
|
else:
|
|
# FIXME automatic end insertion
|
|
raise ExtractorError('Unexpected sequence at %d' % peek.pos)
|
|
else: # label
|
|
# XXX possible refactoring (this is the only branch not popping)
|
|
token = self.token_stream.peek(2)
|
|
if token.id is TokenTypes.COLON:
|
|
token = self.token_stream.pop(2)
|
|
self.token_stream.chk_id(last=True)
|
|
statement = (TokenTypes.LABEL, token.value, self._statement(stack_top - 1))
|
|
|
|
# expr
|
|
if statement is None:
|
|
statement = self._expression(stack_top - 1)
|
|
peek = self.token_stream.peek()
|
|
if peek.id is TokenTypes.END:
|
|
self.token_stream.pop()
|
|
else:
|
|
# FIXME automatic end insertion
|
|
raise ExtractorError('Unexpected sequence at %d' % peek.pos)
|
|
|
|
return statement
|
|
|
|
def _if_statement(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
self.token_stream.pop()
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.POPEN:
|
|
raise ExtractorError('Missing condition at %d' % token.pos)
|
|
cond_expr = self._expression(stack_top - 1)
|
|
self.token_stream.pop() # Token.PCLOSE
|
|
true_stmt = self._statement(stack_top - 1)
|
|
false_stmt = None
|
|
token = self.token_stream.peek()
|
|
if token.id is TokenTypes.ID and token.value == 'else':
|
|
self.token_stream.pop()
|
|
false_stmt = self._statement(stack_top - 1)
|
|
return (TokenTypes.IF, cond_expr, true_stmt, false_stmt)
|
|
|
|
def _for_loop(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
self.token_stream.pop()
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.POPEN:
|
|
raise ExtractorError('''Expected '(' at %d''' % token.pos)
|
|
|
|
# FIXME set infor True (checked by variable declaration and relation expression)
|
|
self._no_in = False
|
|
token = self.token_stream.peek()
|
|
if token.id is TokenTypes.END:
|
|
init = None
|
|
elif token.id is TokenTypes.ID and token.value == 'var':
|
|
# XXX change it on refactoring variable declaration list
|
|
init = self._statement(stack_top - 1)
|
|
else:
|
|
init = self._expression(stack_top - 1)
|
|
self._no_in = True
|
|
|
|
token = self.token_stream.pop()
|
|
if token.id is TokenTypes.ID and token.value == 'in':
|
|
cond = self._expression(stack_top - 1)
|
|
# FIXME further processing of operator 'in' needed for interpretation
|
|
incr = None
|
|
# NOTE ES6 has 'of' operator
|
|
elif token.id is TokenTypes.END:
|
|
token = self.token_stream.peek()
|
|
cond = None if token.id is TokenTypes.END else self._expression(stack_top - 1)
|
|
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.END:
|
|
raise ExtractorError('''Expected ';' at %d''' % token.pos)
|
|
|
|
token = self.token_stream.peek()
|
|
incr = None if token.id is TokenTypes.END else self._expression(stack_top - 1)
|
|
else:
|
|
raise ExtractorError('Invalid condition in for loop initialization at %d' % token.pos)
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.PCLOSE:
|
|
raise ExtractorError('''Expected ')' at %d''' % token.pos)
|
|
body = self._statement(stack_top - 1)
|
|
return (TokenTypes.FOR, init, cond, incr, body)
|
|
|
|
def _do_loop(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
self.token_stream.pop()
|
|
body = self._statement(stack_top - 1)
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.ID and token.value != 'while':
|
|
raise ExtractorError('''Expected 'while' at %d''' % token.pos)
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.POPEN:
|
|
raise ExtractorError('''Expected '(' at %d''' % token.pos)
|
|
expr = self._expression(stack_top - 1)
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.PCLOSE:
|
|
raise ExtractorError('''Expected ')' at %d''' % token.pos)
|
|
peek = self.token_stream.peek()
|
|
if peek.id is TokenTypes.END:
|
|
self.token_stream.pop()
|
|
else:
|
|
# FIXME automatic end insertion
|
|
raise ExtractorError('''Expected ';' at %d''' % peek.pos)
|
|
return (TokenTypes.DO, expr, body)
|
|
|
|
def _while_loop(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
self.token_stream.pop()
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.POPEN:
|
|
raise ExtractorError('''Expected '(' at %d''' % token.pos)
|
|
expr = self._expression(stack_top - 1)
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.PCLOSE:
|
|
raise ExtractorError('''Expected ')' at %d''' % token.pos)
|
|
body = self._statement(stack_top)
|
|
return (TokenTypes.WHILE, expr, body)
|
|
|
|
def _return_statement(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
self.token_stream.pop()
|
|
peek = self.token_stream.peek()
|
|
# XXX no line break here
|
|
expr = self._expression(stack_top - 1) if peek.id is not TokenTypes.END else None
|
|
return (TokenTypes.RETURN, expr)
|
|
|
|
def _with_statement(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
self.token_stream.pop()
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.POPEN:
|
|
raise ExtractorError('Missing expression at %d' % token.pos)
|
|
expr = self._expression(stack_top - 1)
|
|
self.token_stream.pop() # Token.PCLOSE
|
|
return (TokenTypes.WITH, expr, self._statement(stack_top - 1))
|
|
|
|
def _switch_statement(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
self.token_stream.pop()
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.POPEN:
|
|
raise ExtractorError('Missing expression at %d' % token.pos)
|
|
discriminant = self._expression(stack_top - 1)
|
|
self.token_stream.pop() # Token.PCLOSE
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.COPEN:
|
|
raise ExtractorError('Missing case block at %d' % token.pos)
|
|
open_pos = token.pos
|
|
has_default = False
|
|
block = []
|
|
while True:
|
|
token = self.token_stream.peek()
|
|
if token.id is TokenTypes.CCLOSE:
|
|
break
|
|
elif token.id is TokenTypes.ID and token.value == 'case':
|
|
self.token_stream.pop()
|
|
expr = self._expression(stack_top - 1)
|
|
|
|
elif token.id is TokenTypes.ID and token.value == 'default':
|
|
if has_default:
|
|
raise ExtractorError('Multiple default clause')
|
|
self.token_stream.pop()
|
|
has_default = True
|
|
expr = None
|
|
|
|
elif token.id is TokenTypes.END and self.token_stream.ended:
|
|
raise ExtractorError('Unbalanced parentheses at %d' % open_pos)
|
|
else:
|
|
raise ExtractorError('Unexpected sequence at %d, default or case clause is expected' %
|
|
token.pos)
|
|
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.COLON:
|
|
raise ExtractorError('''Unexpected sequence at %d, ':' is expected''' % token.pos)
|
|
|
|
statement_list = []
|
|
while True:
|
|
token = self.token_stream.peek()
|
|
if token.id == TokenTypes.CCLOSE or (
|
|
token.id is TokenTypes.ID and (token.value in ('default', 'case'))):
|
|
break
|
|
elif token.id is TokenTypes.END and self.token_stream.ended:
|
|
raise ExtractorError('Unbalanced parentheses at %d' % open_pos)
|
|
statement_list.append(self._statement(stack_top - 1))
|
|
|
|
block.append((expr, statement_list))
|
|
self.token_stream.pop()
|
|
return (TokenTypes.SWITCH, discriminant, block)
|
|
|
|
def _try_statement(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
self.token_stream.pop()
|
|
token = self.token_stream.peek()
|
|
if token.id is not TokenTypes.COPEN:
|
|
raise ExtractorError('Block is expected at %d' % token.pos)
|
|
try_block = self._statement(stack_top - 1)
|
|
token = self.token_stream.pop()
|
|
catch_block = None
|
|
if token.id is TokenTypes.ID and token.value == 'catch':
|
|
token = self.token_stream.peek()
|
|
if token.id is not TokenTypes.POPEN:
|
|
raise ExtractorError('Catch clause is missing an identifier at %d' % token.pos)
|
|
self.token_stream.pop()
|
|
self.token_stream.chk_id()
|
|
error = self.token_stream.pop()
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.PCLOSE:
|
|
raise ExtractorError('Catch clause expects a single identifier at %d' % token.pos)
|
|
token = self.token_stream.peek()
|
|
if token.id is not TokenTypes.COPEN:
|
|
raise ExtractorError('Block is expected at %d' % token.pos)
|
|
catch_block = (error.value, self._statement(stack_top - 1))
|
|
finally_block = None
|
|
if token.id is TokenTypes.ID and token.value == 'finally':
|
|
token = self.token_stream.peek()
|
|
if token.id is not TokenTypes.COPEN:
|
|
raise ExtractorError('Block is expected at %d' % token.pos)
|
|
finally_block = self._statement(stack_top - 1)
|
|
if catch_block is None and finally_block is None:
|
|
raise ExtractorError('Try statement is expecting catch or finally at %d' % token.pos)
|
|
return (TokenTypes.TRY, try_block, catch_block, finally_block)
|
|
|
|
def _expression(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
expr_list = []
|
|
has_another = True
|
|
while has_another:
|
|
expr_list.append(self._assign_expression(stack_top - 1))
|
|
peek = self.token_stream.peek()
|
|
if peek.id is TokenTypes.COMMA:
|
|
self.token_stream.pop()
|
|
elif peek.id is TokenTypes.ID and peek.value == 'yield':
|
|
# TODO parse yield
|
|
raise ExtractorError('Yield statement is not yet supported at %d' % peek.pos)
|
|
else:
|
|
has_another = False
|
|
return (TokenTypes.EXPR, expr_list)
|
|
|
|
def _assign_expression(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
left = self._conditional_expression(stack_top - 1)
|
|
peek = self.token_stream.peek()
|
|
if peek.id is TokenTypes.AOP:
|
|
self.token_stream.pop()
|
|
_, op = peek.value
|
|
right = self._assign_expression(stack_top - 1)
|
|
else:
|
|
op = None
|
|
right = None
|
|
return (TokenTypes.ASSIGN, op, left, right)
|
|
|
|
def _member_expression(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
peek = self.token_stream.peek()
|
|
if peek.id is TokenTypes.ID and peek.value == 'new':
|
|
self.token_stream.pop()
|
|
target = self._member_expression(stack_top - 1)
|
|
args = self._arguments(stack_top - 1)
|
|
# Rhino has check for args length
|
|
# Rhino has experimental syntax allowing an object literal to follow a new expression
|
|
else:
|
|
target = self._primary_expression(stack_top - 1)
|
|
args = None
|
|
|
|
return (TokenTypes.MEMBER, target, args, self._member_tail(stack_top - 1))
|
|
|
|
def _member_tail(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
peek = self.token_stream.peek()
|
|
if peek.id is TokenTypes.DOT:
|
|
self.token_stream.pop()
|
|
peek = self.token_stream.peek()
|
|
if peek.id is TokenTypes.DOT:
|
|
self.token_stream.pop()
|
|
peek = self.token_stream.peek()
|
|
elif peek.id is TokenTypes.POPEN:
|
|
# TODO parse field query
|
|
raise ExtractorError('Field query is not yet supported at %d' % peek.pos)
|
|
|
|
if peek.id is TokenTypes.ID:
|
|
self.token_stream.pop()
|
|
return (TokenTypes.FIELD, peek.value, self._member_tail(stack_top - 1))
|
|
else:
|
|
raise ExtractorError('Identifier name expected at %d' % peek.pos)
|
|
elif peek.id is TokenTypes.SOPEN:
|
|
self.token_stream.pop()
|
|
index = self._expression(stack_top - 1)
|
|
token = self.token_stream.pop()
|
|
if token.id is TokenTypes.SCLOSE:
|
|
return (TokenTypes.ELEM, index, self._member_tail(stack_top - 1))
|
|
else:
|
|
raise ExtractorError('Unexpected sequence at %d' % token.pos)
|
|
elif peek.id is TokenTypes.POPEN:
|
|
args = self._arguments(stack_top - 1)
|
|
return (TokenTypes.CALL, args, self._member_tail(stack_top - 1))
|
|
else:
|
|
return None
|
|
|
|
def _primary_expression(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
# TODO support let
|
|
peek = self.token_stream.peek()
|
|
if peek.id in token_keys:
|
|
if peek.id is TokenTypes.ID:
|
|
# this
|
|
if peek.value == 'this':
|
|
self.token_stream.pop()
|
|
return (TokenTypes.RSV, 'this')
|
|
# function expr
|
|
elif peek.value == 'function':
|
|
return self._function(stack_top - 1, True)
|
|
# id
|
|
else:
|
|
self.token_stream.chk_id()
|
|
self.token_stream.pop()
|
|
return (TokenTypes.ID, peek.value)
|
|
# literals
|
|
else:
|
|
self.token_stream.pop()
|
|
return (peek.id, peek.value)
|
|
# array
|
|
elif peek.id is TokenTypes.SOPEN:
|
|
return self._array_literal(stack_top - 1)
|
|
# object
|
|
elif peek.id is TokenTypes.COPEN:
|
|
return self._object_literal(stack_top)
|
|
# expr
|
|
elif peek.id is TokenTypes.POPEN:
|
|
self.token_stream.pop()
|
|
open_pos = peek.pos
|
|
expr = self._expression(stack_top - 1)
|
|
peek = self.token_stream.peek()
|
|
if peek.id is not TokenTypes.PCLOSE:
|
|
raise ExtractorError('Unbalanced parentheses at %d' % open_pos)
|
|
self.token_stream.pop()
|
|
return expr
|
|
else:
|
|
raise ExtractorError('Syntax error at %d' % peek.pos)
|
|
|
|
def _function(self, stack_top, is_expr=False):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
self.token_stream.pop()
|
|
token = self.token_stream.peek()
|
|
|
|
name = None
|
|
if token.id is TokenTypes.ID:
|
|
self.token_stream.chk_id()
|
|
name = self.token_stream.pop().value
|
|
token = self.token_stream.peek()
|
|
elif not is_expr:
|
|
raise ExtractorError('Function declaration at %d is missing identifier' % token.pos)
|
|
|
|
if token.id is not TokenTypes.POPEN:
|
|
raise ExtractorError('Expected argument list at %d' % token.pos)
|
|
|
|
# args
|
|
self.token_stream.pop()
|
|
open_pos = token.pos
|
|
args = []
|
|
while True:
|
|
token = self.token_stream.peek()
|
|
if token.id is TokenTypes.PCLOSE:
|
|
self.token_stream.pop()
|
|
break
|
|
self.token_stream.chk_id()
|
|
self.token_stream.pop()
|
|
args.append(token.value)
|
|
token = self.token_stream.peek()
|
|
if token.id is TokenTypes.COMMA:
|
|
self.token_stream.pop()
|
|
elif token.id is TokenTypes.PCLOSE:
|
|
pass
|
|
elif token.id is TokenTypes.END and self.token_stream.ended:
|
|
raise ExtractorError('Unbalanced parentheses at %d' % open_pos)
|
|
else:
|
|
raise ExtractorError('Expected , separator at %d' % token.pos)
|
|
|
|
token = self.token_stream.peek()
|
|
if token.id is not TokenTypes.COPEN:
|
|
raise ExtractorError('Expected function body at %d' % token.pos)
|
|
|
|
return (TokenTypes.FUNC, name, args, (self._function_body(stack_top - 1)))
|
|
|
|
def _function_body(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
open_pos = self.token_stream.pop().pos
|
|
body = []
|
|
while True:
|
|
token = self.token_stream.peek()
|
|
if token.id is TokenTypes.CCLOSE:
|
|
self.token_stream.pop()
|
|
break
|
|
elif token.id is TokenTypes.END and self.token_stream.ended:
|
|
raise ExtractorError('Unbalanced parentheses at %d' % open_pos)
|
|
body.append(self._source_element(stack_top - 1))
|
|
|
|
return body
|
|
|
|
def _arguments(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
peek = self.token_stream.peek()
|
|
if peek.id is TokenTypes.POPEN:
|
|
self.token_stream.pop()
|
|
open_pos = peek.pos
|
|
else:
|
|
return None
|
|
args = []
|
|
while True:
|
|
peek = self.token_stream.peek()
|
|
if peek.id is TokenTypes.PCLOSE:
|
|
self.token_stream.pop()
|
|
return args
|
|
# FIXME handle infor
|
|
args.append(self._assign_expression(stack_top - 1))
|
|
# TODO parse generator expression
|
|
peek = self.token_stream.peek()
|
|
|
|
if peek.id is TokenTypes.COMMA:
|
|
self.token_stream.pop()
|
|
elif peek.id is TokenTypes.PCLOSE:
|
|
pass
|
|
elif peek.id is TokenTypes.END and self.token_stream.ended:
|
|
raise ExtractorError('Unbalanced parentheses at %d' % open_pos)
|
|
else:
|
|
raise ExtractorError('''Expected ',' separator at %d''' % peek.pos)
|
|
|
|
def _array_literal(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
# XXX check no linebreak here
|
|
peek = self.token_stream.peek()
|
|
if peek.id is not TokenTypes.SOPEN:
|
|
raise ExtractorError('Array expected at %d' % peek.pos)
|
|
self.token_stream.pop()
|
|
elements = []
|
|
|
|
has_another = True
|
|
while has_another:
|
|
peek = self.token_stream.peek()
|
|
if peek.id is TokenTypes.COMMA:
|
|
self.token_stream.pop()
|
|
elements.append(None)
|
|
elif peek.id is TokenTypes.SCLOSE:
|
|
self.token_stream.pop()
|
|
has_another = False
|
|
elif peek.id is TokenTypes.ID and peek.value == 'for':
|
|
# TODO parse array comprehension
|
|
raise ExtractorError('Array comprehension is not yet supported at %d' % peek.pos)
|
|
else:
|
|
elements.append(self._assign_expression(stack_top - 1))
|
|
peek = self.token_stream.pop()
|
|
if peek.id is TokenTypes.SCLOSE:
|
|
has_another = False
|
|
elif peek.id is not TokenTypes.COMMA:
|
|
raise ExtractorError('''Expected ',' after element at %d''' % peek.pos)
|
|
|
|
return (TokenTypes.ARRAY, elements)
|
|
|
|
def _object_literal(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
open_pos = self.token_stream.pop().pos
|
|
property_list = []
|
|
while True:
|
|
token = self.token_stream.pop()
|
|
if token.id is TokenTypes.CCLOSE:
|
|
break
|
|
elif token.id is TokenTypes.COMMA:
|
|
continue
|
|
elif token.id is TokenTypes.ID and token.value in ('get', 'set'):
|
|
is_set = token.id is TokenTypes.ID and token.value == 'set'
|
|
|
|
token = self.token_stream.pop()
|
|
if token.id not in (TokenTypes.ID, TokenTypes.STR, TokenTypes.INT, TokenTypes.FLOAT):
|
|
raise ExtractorError('Property name is expected at %d' % token.pos)
|
|
property_name = token.value
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.POPEN:
|
|
raise ExtractorError('''Expected '(' at %d''' % token.pos)
|
|
|
|
if is_set:
|
|
self.token_stream.chk_id()
|
|
arg = self.token_stream.pop().value
|
|
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.PCLOSE:
|
|
raise ExtractorError('''Expected ')' at %d''' % token.pos)
|
|
|
|
if is_set:
|
|
desc = (TokenTypes.PROPSET, arg, self._function_body(stack_top - 1))
|
|
else:
|
|
desc = (TokenTypes.PROPGET, self._function_body(stack_top - 1))
|
|
|
|
elif token.id in (TokenTypes.ID, TokenTypes.STR, TokenTypes.INT, TokenTypes.FLOAT):
|
|
property_name = token.value
|
|
token = self.token_stream.pop()
|
|
if token.id is not TokenTypes.COLON:
|
|
raise ExtractorError('Property name is expected at %d' % token.pos)
|
|
|
|
desc = (TokenTypes.PROPVALUE, self._assign_expression(stack_top - 1))
|
|
|
|
elif self.token_stream.ended:
|
|
raise ExtractorError('Unmatched parentheses at %d' % open_pos)
|
|
else:
|
|
raise ExtractorError('Property assignment is expected at %d' % token.pos)
|
|
|
|
property_list.append((property_name, desc))
|
|
|
|
return (TokenTypes.OBJECT, property_list)
|
|
|
|
def _conditional_expression(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
expr = self._operator_expression(stack_top - 1)
|
|
peek = self.token_stream.peek()
|
|
if peek.id is TokenTypes.HOOK:
|
|
hook_pos = peek.pos
|
|
true_expr = self._assign_expression(stack_top - 1)
|
|
peek = self.token_stream.peek()
|
|
if peek.id is TokenTypes.COLON:
|
|
false_expr = self._assign_expression(stack_top - 1)
|
|
else:
|
|
raise ExtractorError('Missing : in conditional expression at %d' % hook_pos)
|
|
return (TokenTypes.COND, expr, true_expr, false_expr)
|
|
return expr
|
|
|
|
def _operator_expression(self, stack_top):
|
|
if stack_top < 0:
|
|
raise ExtractorError('Recursion limit reached')
|
|
|
|
# --<---------------------------------<-- op --<--------------------------<----
|
|
# | |
|
|
# | --<-- prefix --<-- -->-- postfix -->-- |
|
|
# | | ^ ^ | ^
|
|
# v v | | v |
|
|
# ->------------>----------->-- lefthand-side expression -->----------->------------>---|
|
|
#
|
|
# 20 grouping
|
|
# ... # handled by lefthandside_expression
|
|
# 17 postfix
|
|
# 16 unary
|
|
# 15 exponentiation # not yet found in grammar
|
|
# 14 mul
|
|
# 13 add
|
|
# 12 shift
|
|
# 11 rel
|
|
# 10 eq
|
|
# 9 band
|
|
# 8 bxor
|
|
# 7 bor
|
|
# 6 land
|
|
# 5 lor
|
|
# 4 cond # handled by conditional_expression
|
|
|
|
out = []
|
|
stack = []
|
|
|
|
while True:
|
|
had_inc = False
|
|
has_prefix = True
|
|
while has_prefix:
|
|
token = self.token_stream.peek()
|
|
peek_id = token.id
|
|
peek_value = token.value
|
|
if peek_id is TokenTypes.OP and peek_value[0] in (TokenTypes.ADD, TokenTypes.SUB):
|
|
# any binary operators will be consumed later
|
|
peek_id = TokenTypes.UOP
|
|
peek_value = convert_to_unary(peek_value)
|
|
if peek_id is TokenTypes.UOP:
|
|
name, op = peek_value
|
|
had_inc = name in (TokenTypes.INC, TokenTypes.DEC)
|
|
if had_inc:
|
|
peek_id = TokenTypes.PREFIX
|
|
while stack and stack[-1][0] > 16:
|
|
_, stack_id, stack_op = stack.pop()
|
|
out.append((stack_id, stack_op))
|
|
stack.append((16, peek_id, op))
|
|
self.token_stream.pop()
|
|
token = self.token_stream.peek()
|
|
if had_inc and token.id is not TokenTypes.ID:
|
|
raise ExtractorError('Prefix operator has to be followed by an identifier at %d' % token.pos)
|
|
has_prefix = token.id is TokenTypes.UOP
|
|
else:
|
|
has_prefix = False
|
|
|
|
left = self._member_expression(stack_top - 1)
|
|
out.append(left)
|
|
|
|
token = self.token_stream.peek()
|
|
# postfix
|
|
if token.id is TokenTypes.UOP:
|
|
if had_inc:
|
|
raise ExtractorError('''Can't have prefix and postfix operator at the same time at %d''' % token.pos)
|
|
name, op = token.value
|
|
if name in (TokenTypes.INC, TokenTypes.DEC):
|
|
peek_id = TokenTypes.POSTFIX
|
|
prec = 17
|
|
else:
|
|
raise ExtractorError('Unexpected operator at %d' % token.pos)
|
|
while stack and stack[-1][0] >= 17:
|
|
_, stack_id, stack_op = stack.pop()
|
|
out.append((stack_id, stack_op))
|
|
stack.append((prec, peek_id, op))
|
|
self.token_stream.pop()
|
|
token = self.token_stream.peek()
|
|
|
|
if token.id is TokenTypes.REL:
|
|
name, op = token.value
|
|
prec = 11
|
|
elif token.id is TokenTypes.OP:
|
|
name, op = token.value
|
|
if name in (TokenTypes.MUL, TokenTypes.DIV, TokenTypes.MOD):
|
|
prec = 14
|
|
elif name in (TokenTypes.ADD, TokenTypes.SUB):
|
|
prec = 13
|
|
elif name in (TokenTypes.RSHIFT, TokenTypes.LSHIFT, TokenTypes.URSHIFT):
|
|
prec = 12
|
|
elif name is TokenTypes.BAND:
|
|
prec = 9
|
|
elif name is TokenTypes.BXOR:
|
|
prec = 8
|
|
elif name is TokenTypes.BOR:
|
|
prec = 7
|
|
else:
|
|
raise ExtractorError('Unexpected operator at %d' % token.pos)
|
|
elif token.id is TokenTypes.LOP:
|
|
name, op = token.value
|
|
prec = {TokenTypes.OR: 5, TokenTypes.AND: 6}[name]
|
|
else:
|
|
op = None
|
|
prec = 4 # empties stack
|
|
|
|
while stack and stack[-1][0] >= prec:
|
|
_, stack_id, stack_op = stack.pop()
|
|
out.append((stack_id, stack_op))
|
|
if op is None:
|
|
break
|
|
else:
|
|
stack.append((prec, token.id, op))
|
|
self.token_stream.pop()
|
|
|
|
return (TokenTypes.OPEXPR, out)
|