From 48aaa4178ec0c73dd2abe05d5d4b42a28a20c15b Mon Sep 17 00:00:00 2001 From: sulyi Date: Sun, 11 Dec 2016 17:36:19 +0100 Subject: [PATCH] [jsinterp] Finished parser if test --- test/test_jsinterp_parser.py | 68 ++++++++++++++++++++++++++++++--- youtube_dl/jsinterp/jsinterp.py | 17 ++++++--- youtube_dl/jsinterp/tstream.py | 11 +++++- 3 files changed, 83 insertions(+), 13 deletions(-) diff --git a/test/test_jsinterp_parser.py b/test/test_jsinterp_parser.py index 40d026a36..ca7154932 100644 --- a/test/test_jsinterp_parser.py +++ b/test/test_jsinterp_parser.py @@ -630,7 +630,6 @@ class TestJSInterpreterParser(unittest.TestCase): ] self.assertEqual(list(jsi.statements()), ast) - @unittest.skip('Test not yet implemented: missing ast') def test_if(self): # TODO if test jsi = JSInterpreter( @@ -643,7 +642,23 @@ class TestJSInterpreterParser(unittest.TestCase): } ''' ) - ast = [] + ast = [ + (Token.FUNC, 'a', + ['x'], + (Token.BLOCK, [ + (Token.IF, + (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ + (Token.MEMBER, (Token.ID, 'x'), None, None), + (Token.MEMBER, (Token.INT, 0), None, None), + (Token.REL, _RELATIONS['>'][1]) + ]), None)]), + (Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ + (Token.MEMBER, (Token.BOOL, True), None, None)]), None)])), + (Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ + (Token.MEMBER, (Token.BOOL, False), None, None)]), None)]))) + + ])) + ] self.assertEqual(list(jsi.statements()), ast) jsi = JSInterpreter( @@ -655,7 +670,23 @@ class TestJSInterpreterParser(unittest.TestCase): } ''' ) - ast = [] + ast = [ + (Token.FUNC, 'a', + ['x'], + (Token.BLOCK, [ + (Token.IF, + (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ + (Token.MEMBER, (Token.ID, 'x'), None, None), + (Token.MEMBER, (Token.INT, 0), None, None), + (Token.REL, _RELATIONS['>'][1]) + ]), None)]), + (Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ + (Token.MEMBER, (Token.BOOL, True), None, None)]), None)])), + None), + (Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ + (Token.MEMBER, (Token.BOOL, False), None, None)]), None)])) + ])) + ] self.assertEqual(list(jsi.statements()), ast) jsi = JSInterpreter( @@ -666,12 +697,39 @@ class TestJSInterpreterParser(unittest.TestCase): return x; } else { x++; - return false; + return x; } } ''' ) - ast = [] + ast = [ + (Token.FUNC, 'a', + ['x'], + (Token.BLOCK, [ + (Token.IF, + (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ + (Token.MEMBER, (Token.ID, 'x'), None, None), + (Token.MEMBER, (Token.INT, 0), None, None), + (Token.REL, _RELATIONS['>'][1]) + ]), None)]), + (Token.BLOCK, [ + (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ + (Token.MEMBER, (Token.ID, 'x'), None, None), + (Token.UOP, _UNARY_OPERATORS['--'][1]) + ]), None)]), + (Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ + (Token.MEMBER, (Token.ID, 'x'), None, None)]), None)])) + ]), + (Token.BLOCK, [ + (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ + (Token.MEMBER, (Token.ID, 'x'), None, None), + (Token.UOP, _UNARY_OPERATORS['++'][1]) + ]), None)]), + (Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ + (Token.MEMBER, (Token.ID, 'x'), None, None)]), None)])) + ])) + ])) + ] self.assertEqual(list(jsi.statements()), ast) @unittest.skip('Test not yet implemented: missing code and ast') diff --git a/youtube_dl/jsinterp/jsinterp.py b/youtube_dl/jsinterp/jsinterp.py index 9d655ed22..4dec1aa72 100644 --- a/youtube_dl/jsinterp/jsinterp.py +++ b/youtube_dl/jsinterp/jsinterp.py @@ -66,6 +66,7 @@ class JSInterpreter(object): token_id, token_value, token_pos = token_stream.peek() if token_id in (Token.CCLOSE, Token.END): # empty statement goes straight here + token_stream.pop() return statement if token_id is Token.ID and token_value == 'function': @@ -111,11 +112,11 @@ class JSInterpreter(object): while True: token_id, token_value, token_pos = token_stream.peek() if token_id is Token.CCLOSE: + token_stream.pop() break elif token_id is Token.END and token_stream.ended: raise ExtractorError('Unbalanced parentheses at %d' % open_pos) block.append(self._next_statement(token_stream, stack_top - 1)) - token_stream.pop() statement = (Token.BLOCK, block) @@ -142,6 +143,7 @@ class JSInterpreter(object): init.append(JSInterpreter.undefined) if peek_id is Token.END: + token_stream.pop() has_another = False elif peek_id is Token.COMMA: pass @@ -161,11 +163,8 @@ class JSInterpreter(object): cond_expr = self._expression(token_stream, stack_top - 1) token_stream.pop() # Token.PCLOSE true_expr = self._next_statement(token_stream, stack_top - 1) - token_id, token_value, token_pos = token_stream.peek() - if token_id is Token.CCLOSE: - token_stream.pop() - token_id, token_value, token_pos = token_stream.peek() false_expr = None + token_id, token_value, token_pos = token_stream.peek() if token_id is Token.ID and token_value == 'else': token_stream.pop() false_expr = self._next_statement(token_stream, stack_top - 1) @@ -190,6 +189,8 @@ class JSInterpreter(object): if peek_id is not Token.END: # FIXME automatic end insertion raise ExtractorError('Unexpected sequence %s at %d' % (peek_value, peek_pos)) + else: + token_stream.pop() elif token_value == 'return': token_stream.pop() @@ -201,6 +202,8 @@ class JSInterpreter(object): if peek_id is not Token.END: # FIXME automatic end insertion raise ExtractorError('Unexpected sequence %s at %d' % (peek_value, peek_pos)) + else: + token_stream.pop() elif token_value == 'with': token_stream.pop() @@ -288,6 +291,7 @@ class JSInterpreter(object): peek_id, peek_value, peek_pos = token_stream.peek() if peek_id is Token.END: + token_stream.pop() has_another = False elif peek_id is Token.COMMA: pass @@ -305,7 +309,7 @@ class JSInterpreter(object): while not ts.ended: yield self._next_statement(ts, stack_size) - ts.pop() + # ts.pop() raise StopIteration def _expression(self, token_stream, stack_top): @@ -584,6 +588,7 @@ class JSInterpreter(object): if peek_id is Token.REL: name, op = peek_value + prec = 11 elif peek_id is Token.OP: name, op = peek_value if name in (Token.MUL, Token.DIV, Token.MOD): diff --git a/youtube_dl/jsinterp/tstream.py b/youtube_dl/jsinterp/tstream.py index 1f7ffacea..47ab6edea 100644 --- a/youtube_dl/jsinterp/tstream.py +++ b/youtube_dl/jsinterp/tstream.py @@ -158,10 +158,17 @@ class TokenStream(object): self.peeked.append(token) return self.peeked[count - 1] - def pop(self): + def pop(self, count=1): if not self.peeked: self.peek() - self._last = self.peeked.pop(0) + for _ in range(count): + self._last = self.peeked.pop() + return self._last + + def flush(self): + if self.peeked: + self._last = self.peeked[-1] + self.peeked = [] return self._last def last(self):