From d7443e1233fc9abcd7f9163d7a4ce257533b390a Mon Sep 17 00:00:00 2001 From: sulyi Date: Thu, 15 Dec 2016 20:02:04 +0100 Subject: [PATCH] [jsinterp] Adding interpreter support for pre- and postfix expressions --- test/jstests/__init__.py | 9 +++++---- test/jstests/do_loop.py | 2 +- test/jstests/for_empty.py | 2 +- test/jstests/for_loop.py | 2 +- test/jstests/operators.py | 11 ----------- test/jstests/switch.py | 4 ++-- test/jstests/unary.py | 17 +++++++++++++++++ test/jstests/while_loop.py | 2 +- test/test_jsinterp.py | 2 +- test/test_jsinterp_parse.py | 21 +++++++++++---------- youtube_dl/jsinterp/jsgrammar.py | 2 +- youtube_dl/jsinterp/jsinterp.py | 12 ++++++++++++ 12 files changed, 53 insertions(+), 33 deletions(-) create mode 100644 test/jstests/unary.py diff --git a/test/jstests/__init__.py b/test/jstests/__init__.py index 03e51047b..21c6e673b 100644 --- a/test/jstests/__init__.py +++ b/test/jstests/__init__.py @@ -5,6 +5,7 @@ from . import ( morespace, strange_chars, operators, + unary, array_access, parens, assignments, @@ -29,15 +30,15 @@ from . import ( ) -modules = [basic, calc, empty_return, morespace, strange_chars, operators, array_access, parens, assignments, comments, - precedence, call, getfield, branch, switch, for_loop, for_empty, for_in, do_loop, while_loop, label, - func_expr, object_literal, try_statement, with_statement, debug, unshift] +modules = [basic, calc, empty_return, morespace, strange_chars, operators, unary, array_access, parens, assignments, + comments, precedence, call, getfield, branch, switch, for_loop, for_empty, for_in, do_loop, while_loop, + label, func_expr, object_literal, try_statement, with_statement, debug, unshift] def gettestcases(): for module in modules: if hasattr(module, 'tests'): - case = {'name': module.__name__[len(__name__) + 1:], 'subtests': []} + case = {'name': module.__name__[len(__name__) + 1:], 'subtests': [], 'skip': {}} for test in getattr(module, 'tests'): if 'code' in test: case['subtests'].append(test) diff --git a/test/jstests/do_loop.py b/test/jstests/do_loop.py index 80caff65f..dce1fe984 100644 --- a/test/jstests/do_loop.py +++ b/test/jstests/do_loop.py @@ -35,7 +35,7 @@ tests = [ (Token.EXPR, [ (Token.ASSIGN, None, (Token.OPEXPR, [ (Token.MEMBER, (Token.ID, 'i'), None, None), - (Token.UOP, _UNARY_OPERATORS['++'][1]) + (Token.POSTFIX, _UNARY_OPERATORS['++'][1]) ]), None) ]) ])), diff --git a/test/jstests/for_empty.py b/test/jstests/for_empty.py index b3a83c11c..87ee4f873 100644 --- a/test/jstests/for_empty.py +++ b/test/jstests/for_empty.py @@ -30,7 +30,7 @@ tests = [ ]), None)]), (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ (Token.MEMBER, (Token.ID, 'h'), None, None), - (Token.UOP, _UNARY_OPERATORS['++'][1]) + (Token.PREFIX, _UNARY_OPERATORS['++'][1]) ]), None)]), (Token.BLOCK, [ (Token.EXPR, [ diff --git a/test/jstests/for_loop.py b/test/jstests/for_loop.py index 147a3c8b1..d53c57384 100644 --- a/test/jstests/for_loop.py +++ b/test/jstests/for_loop.py @@ -28,7 +28,7 @@ tests = [ ]), None)]), (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ (Token.MEMBER, (Token.ID, 'h'), None, None), - (Token.UOP, _UNARY_OPERATORS['++'][1]) + (Token.PREFIX, _UNARY_OPERATORS['++'][1]) ]), None)]), (Token.BLOCK, [ (Token.EXPR, [ diff --git a/test/jstests/operators.py b/test/jstests/operators.py index c70bac39b..c95a8baca 100644 --- a/test/jstests/operators.py +++ b/test/jstests/operators.py @@ -39,16 +39,5 @@ tests = [ (Token.OP, _OPERATORS['>>'][1]) ]), None) ]))] - }, { - 'code': 'return -5 + +3;', - 'asserts': [{'value': -2}] - }, { - 'code': 'return -5 + ++a;', - 'globals': {'a': -3}, - 'asserts': [{'value': -7}] - }, { - 'code': 'function f() {return -5 + a++;}', - 'globals': {'a': -3}, - 'asserts': [{'value': -8, 'call': ('f',)}, {'value': -7, 'call': ('f',)}] } ] diff --git a/test/jstests/switch.py b/test/jstests/switch.py index 0777bd119..7442a8480 100644 --- a/test/jstests/switch.py +++ b/test/jstests/switch.py @@ -42,7 +42,7 @@ tests = [ [ (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ (Token.MEMBER, (Token.ID, 'x'), None, None), - (Token.UOP, _UNARY_OPERATORS['++'][1]) + (Token.POSTFIX, _UNARY_OPERATORS['++'][1]) ]), None)]) ]), ((Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ @@ -50,7 +50,7 @@ tests = [ [ (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ (Token.MEMBER, (Token.ID, 'x'), None, None), - (Token.UOP, _UNARY_OPERATORS['--'][1]) + (Token.POSTFIX, _UNARY_OPERATORS['--'][1]) ]), None)]), (Token.BREAK, None) ]), diff --git a/test/jstests/unary.py b/test/jstests/unary.py new file mode 100644 index 000000000..400d2b6f4 --- /dev/null +++ b/test/jstests/unary.py @@ -0,0 +1,17 @@ + +skip = {'p': True} + +tests = [ + { + 'code': 'return -5 + +3;', + 'asserts': [{'value': -2}] + }, { + 'code': 'function f() {return -5 + ++a;}', + 'globals': {'a': -3}, + 'asserts': [{'value': -7, 'call': ('f',)}, {'value': -6, 'call': ('f',)}] + }, { + 'code': 'function f() {return -5 + a++;}', + 'globals': {'a': -3}, + 'asserts': [{'value': -8, 'call': ('f',)}, {'value': -7, 'call': ('f',)}] + } +] diff --git a/test/jstests/while_loop.py b/test/jstests/while_loop.py index 9c8228d23..5aa545d05 100644 --- a/test/jstests/while_loop.py +++ b/test/jstests/while_loop.py @@ -35,7 +35,7 @@ tests = [ (Token.EXPR, [ (Token.ASSIGN, None, (Token.OPEXPR, [ (Token.MEMBER, (Token.ID, 'i'), None, None), - (Token.UOP, _UNARY_OPERATORS['++'][1]) + (Token.POSTFIX, _UNARY_OPERATORS['++'][1]) ]), None) ]) ])), diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py index 3313e40a0..98dc89dbd 100644 --- a/test/test_jsinterp.py +++ b/test/test_jsinterp.py @@ -33,7 +33,7 @@ def generator(test_case): else: self.assertEqual(jsi.run(), a['value']) - if 'skip' not in test_case or 'i' not in test_case['skip']: + if 'i' not in test_case['skip']: reason = False else: reason = test_case['skip']['i'] diff --git a/test/test_jsinterp_parse.py b/test/test_jsinterp_parse.py index 323bb747c..6228e22b5 100644 --- a/test/test_jsinterp_parse.py +++ b/test/test_jsinterp_parse.py @@ -44,7 +44,7 @@ def generator(test_case): if 'ast' in a: self.assertEqual(traverse(parsed), traverse(a['ast'])) - if 'skip' not in test_case or 'p' not in test_case['skip']: + if 'p' not in test_case['skip']: reason = False else: reason = test_case['skip']['p'] @@ -54,12 +54,13 @@ def generator(test_case): # And add them to TestJSInterpreter for n, tc in enumerate(defs): - test_method = generator(tc) - tname = 'test_' + str(tc['name']) - i = 1 - while hasattr(TestJSInterpreterParse, tname): - tname = 'test_%s_%d' % (tc['name'], i) - i += 1 - test_method.__name__ = str(tname) - setattr(TestJSInterpreterParse, test_method.__name__, test_method) - del test_method + if 'p' not in tc['skip'] or tc['skip']['p'] is not True: + test_method = generator(tc) + tname = 'test_' + str(tc['name']) + i = 1 + while hasattr(TestJSInterpreterParse, tname): + tname = 'test_%s_%d' % (tc['name'], i) + i += 1 + test_method.__name__ = str(tname) + setattr(TestJSInterpreterParse, test_method.__name__, test_method) + del test_method diff --git a/youtube_dl/jsinterp/jsgrammar.py b/youtube_dl/jsinterp/jsgrammar.py index a306df770..77f6a1175 100644 --- a/youtube_dl/jsinterp/jsgrammar.py +++ b/youtube_dl/jsinterp/jsgrammar.py @@ -9,7 +9,7 @@ _token_keys = ('COPEN', 'CCLOSE', 'POPEN', 'PCLOSE', 'SOPEN', 'SCLOSE', 'AND', 'OR', 'PLUS', 'NEG', 'INC', 'DEC', 'NOT', 'BNOT', 'DEL', 'VOID', 'TYPE', 'LT', 'GT', 'LE', 'GE', 'EQ', 'NE', 'SEQ', 'SNE', 'IN', 'INSTANCEOF', 'BOR', 'BXOR', 'BAND', 'RSHIFT', 'LSHIFT', 'URSHIFT', 'SUB', 'ADD', 'MOD', 'DIV', 'MUL', - 'OP', 'AOP', 'UOP', 'LOP', 'REL', + 'OP', 'AOP', 'UOP', 'LOP', 'REL', 'PREFIX', 'POSTFIX', 'COMMENT', 'TOKEN', 'PUNCT', 'NULL', 'BOOL', 'ID', 'STR', 'INT', 'FLOAT', 'REGEX', 'OBJECT', 'REFLAGS', 'REBODY', diff --git a/youtube_dl/jsinterp/jsinterp.py b/youtube_dl/jsinterp/jsinterp.py index 0fbb734d5..f35acb530 100644 --- a/youtube_dl/jsinterp/jsinterp.py +++ b/youtube_dl/jsinterp/jsinterp.py @@ -33,6 +33,7 @@ class Reference(object): if not hasattr(parent, '__setitem__'): raise ExtractorError('Unknown reference') parent.__setitem__(key, Reference(value, (parent, key))) + return value def __repr__(self): if self._parent is not None: @@ -748,6 +749,8 @@ class JSInterpreter(object): if peek_id is Token.UOP: name, op = peek_value had_inc = name in (Token.INC, Token.DEC) + if had_inc: + peek_id = Token.PREFIX while stack and stack[-1][0] > 16: _, stack_id, stack_op = stack.pop() out.append((stack_id, stack_op)) @@ -770,6 +773,7 @@ class JSInterpreter(object): raise ExtractorError('''Can't have prefix and postfix operator at the same time at %d''' % peek_pos) name, op = peek_value if name in (Token.INC, Token.DEC): + peek_id = Token.POSTFIX prec = 17 else: raise ExtractorError('Unexpected operator at %d' % peek_pos) @@ -880,6 +884,7 @@ class JSInterpreter(object): elif name is Token.OPEXPR: stack = [] + postfix = [] rpn = expr[1][:] # FIXME support pre- and postfix operators while rpn: @@ -893,10 +898,17 @@ class JSInterpreter(object): elif token[0] is Token.UOP: right = stack.pop() stack.append(Reference(token[1](right.getvalue()))) + elif token[0] is Token.PREFIX: + right = stack.pop() + stack.append(Reference(right.putvalue(token[1](right.getvalue())))) + elif token[0] is Token.POSTFIX: + postfix.append((stack[-1], token[1])) else: stack.append(self.interpret_expression(token)) result = stack.pop() if not stack: + for operand, op in postfix: + operand.putvalue(op(operand.getvalue())) ref = result else: raise ExtractorError('Expression has too many values')