diff --git a/test/test_jsinterp_parser.py b/test/test_jsinterp_parser.py index f226746af..98accd28b 100644 --- a/test/test_jsinterp_parser.py +++ b/test/test_jsinterp_parser.py @@ -835,24 +835,47 @@ class TestJSInterpreterParser(unittest.TestCase): ] self.assertEqual(list(traverse(list(jsi.statements()))), list(traverse(ast))) - @unittest.skip('Test not yet implemented: missing ast') def test_for_empty(self): - # ASAP for empty statement test jsi = JSInterpreter(''' function f(x){ - var h = 0 + var h = 0; for (; h <= x; ++h) { a = h; } return a; } ''') - ast = [] + ast = [ + (Token.FUNC, 'f', ['x'], + (Token.BLOCK, [ + (Token.VAR, zip(['h'], [ + (Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 0), None, None)]), None) + ])), + (Token.FOR, + None, + (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ + (Token.MEMBER, (Token.ID, 'h'), None, None), + (Token.MEMBER, (Token.ID, 'x'), None, None), + (Token.REL, _RELATIONS['<='][1]) + ]), None)]), + (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ + (Token.MEMBER, (Token.ID, 'h'), None, None), + (Token.UOP, _UNARY_OPERATORS['++'][1]) + ]), None)]), + (Token.BLOCK, [ + (Token.EXPR, [ + (Token.ASSIGN, _ASSIGN_OPERATORS['='][1], + (Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'a'), None, None)]), + (Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'h'), None, None)]), None)) + ]) + ])), + (Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ + (Token.MEMBER, (Token.ID, 'a'), None, None)]), None)])) + ])) + ] self.assertEqual(list(traverse(list(jsi.statements()))), list(traverse(ast))) - @unittest.skip('Test not yet implemented: missing ast') def test_for_in(self): - # ASAP for in statement test jsi = JSInterpreter(''' function f(z){ for (h in z) { @@ -861,7 +884,28 @@ class TestJSInterpreterParser(unittest.TestCase): return a; } ''') - ast = [] + ast = [ + (Token.FUNC, 'f', ['z'], + (Token.BLOCK, [ + (Token.FOR, + (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ + (Token.MEMBER, (Token.ID, 'h'), None, None) + ]), None)]), + (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ + (Token.MEMBER, (Token.ID, 'z'), None, None) + ]), None)]), + None, + (Token.BLOCK, [ + (Token.EXPR, [ + (Token.ASSIGN, _ASSIGN_OPERATORS['='][1], + (Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'a'), None, None)]), + (Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'h'), None, None)]), None)) + ]) + ])), + (Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ + (Token.MEMBER, (Token.ID, 'a'), None, None)]), None)])) + ])) + ] self.assertEqual(list(jsi.statements()), ast) def test_do(self): diff --git a/youtube_dl/jsinterp/jsgrammar.py b/youtube_dl/jsinterp/jsgrammar.py index 6f131511a..9eb0b64a4 100644 --- a/youtube_dl/jsinterp/jsgrammar.py +++ b/youtube_dl/jsinterp/jsgrammar.py @@ -30,9 +30,10 @@ __ESC_HEX_RE = r'x[0-9a-fA-F]{2}' # NOTE order is fixed due to regex matching, does not represent any precedence +# NOTE unary operator 'delete', 'void', 'instanceof' and relation 'in' and 'instanceof' do not handled this way _logical_operator = ['||', '&&'] _relation = ['===', '!==', '==', '!=', '<=', '>=', '<', '>'] -_unary_operator = ['++', '--', '!', '~', 'delete', 'void', 'typeof'] +_unary_operator = ['++', '--', '!', '~'] _operator = ['|', '^', '&', '>>>', '>>', '<<', '-', '+', '%', '/', '*'] _assign_operator = [op + '=' for op in _operator] _assign_operator.append('=') diff --git a/youtube_dl/jsinterp/jsinterp.py b/youtube_dl/jsinterp/jsinterp.py index 91b46565d..db9f14625 100644 --- a/youtube_dl/jsinterp/jsinterp.py +++ b/youtube_dl/jsinterp/jsinterp.py @@ -102,7 +102,6 @@ class JSInterpreter(object): elif token_id is Token.ID: if token_value == 'var': - # XXX refactor (create dedicated method for handling variable declaration list) token_stream.pop() variables = [] init = [] @@ -252,17 +251,18 @@ class JSInterpreter(object): if token_id is Token.END: init = None elif token_id is Token.ID and token_value == 'var': + # XXX change it on refactoring variable declaration list init = self._statement(token_stream, stack_top - 1) else: init = self._expression(token_stream, stack_top - 1) self._context.no_in = True token_id, token_value, token_pos = token_stream.pop() - if token_id is Token.IN: + if token_id is Token.ID and token_value == 'in': cond = self._expression(token_stream, stack_top - 1) - # FIXME further processing might be needed for interpretation + # FIXME further processing of operator 'in' needed for interpretation incr = None - # NOTE ES6 has of operator + # NOTE ES6 has 'of' operator elif token_id is Token.END: token_id, token_value, token_pos = token_stream.peek() cond = None if token_id is Token.END else self._expression(token_stream, stack_top - 1) @@ -881,6 +881,8 @@ class JSInterpreter(object): rpn = expr[1][:] while rpn: token = rpn.pop(0) + # XXX add unary operator 'delete', 'void', 'instanceof' + # XXX relation 'in' 'instanceof' if token[0] in (Token.OP, Token.AOP, Token.UOP, Token.LOP, Token.REL): right = stack.pop() left = stack.pop() diff --git a/youtube_dl/jsinterp/tstream.py b/youtube_dl/jsinterp/tstream.py index d5f0cdfca..4e72774b4 100644 --- a/youtube_dl/jsinterp/tstream.py +++ b/youtube_dl/jsinterp/tstream.py @@ -48,12 +48,14 @@ _RELATIONS = { '>': (Token.GT, operator.gt), '<=': (Token.LE, operator.le), '>=': (Token.GE, operator.ge), - # XXX add instanceof and in operators # XXX check python and JavaScript equality difference '==': (Token.EQ, operator.eq), '!=': (Token.NE, operator.ne), '===': (Token.SEQ, lambda cur, right: cur == right and type(cur) == type(right)), - '!==': (Token.SNE, lambda cur, right: not cur == right or not type(cur) == type(right)) + '!==': (Token.SNE, lambda cur, right: not cur == right or not type(cur) == type(right)), + # XXX define instanceof and in operators + 'in': (Token.IN, None), + 'instanceof': (Token.INSTANCEOF, None) } _OPERATORS = { '|': (Token.BOR, operator.or_), @@ -132,7 +134,9 @@ class TokenStream(object): elif token_id is Token.ID: yield (token_id, token_value, pos) elif token_id in _operator_lookup: - yield (token_id, _operator_lookup[token_id][token_value], pos) + yield (token_id if token_value != 'in' else Token.IN, + _operator_lookup[token_id][token_value], + pos) elif token_id is Token.PUNCT: yield (_PUNCTUATIONS[token_value], token_value, pos) else: