R-yan/l1ving_youtube-dl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[jsinterp] Finished parser if test

Browse Source
This commit is contained in:
sulyi 2016-12-11 17:36:19 +01:00
parent ad288aaabd
commit 48aaa4178e
3 changed files with 83 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
test/test_jsinterp_parser.py
View File

@ -630,7 +630,6 @@ class TestJSInterpreterParser(unittest.TestCase):
] ]
self.assertEqual(list(jsi.statements()), ast) self.assertEqual(list(jsi.statements()), ast)
@unittest.skip('Test not yet implemented: missing ast')
def test_if(self): def test_if(self):
# TODO if test # TODO if test
jsi = JSInterpreter( jsi = JSInterpreter(
@ -643,7 +642,23 @@ class TestJSInterpreterParser(unittest.TestCase):
} }
''' '''
) )
ast = [] ast = [
(Token.FUNC, 'a',
['x'],
(Token.BLOCK, [
(Token.IF,
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.MEMBER, (Token.INT, 0), None, None),
(Token.REL, _RELATIONS['>'][1])
]), None)]),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.BOOL, True), None, None)]), None)])),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.BOOL, False), None, None)]), None)])))
]))
]
self.assertEqual(list(jsi.statements()), ast) self.assertEqual(list(jsi.statements()), ast)
jsi = JSInterpreter( jsi = JSInterpreter(
@ -655,7 +670,23 @@ class TestJSInterpreterParser(unittest.TestCase):
} }
''' '''
) )
ast = [] ast = [
(Token.FUNC, 'a',
['x'],
(Token.BLOCK, [
(Token.IF,
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.MEMBER, (Token.INT, 0), None, None),
(Token.REL, _RELATIONS['>'][1])
]), None)]),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.BOOL, True), None, None)]), None)])),
None),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.BOOL, False), None, None)]), None)]))
]))
]
self.assertEqual(list(jsi.statements()), ast) self.assertEqual(list(jsi.statements()), ast)
jsi = JSInterpreter( jsi = JSInterpreter(
@ -666,12 +697,39 @@ class TestJSInterpreterParser(unittest.TestCase):
return x; return x;
} else { } else {
x++; x++;
return false; return x;
} }
} }
''' '''
) )
ast = [] ast = [
(Token.FUNC, 'a',
['x'],
(Token.BLOCK, [
(Token.IF,
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.MEMBER, (Token.INT, 0), None, None),
(Token.REL, _RELATIONS['>'][1])
]), None)]),
(Token.BLOCK, [
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.UOP, _UNARY_OPERATORS['--'][1])
]), None)]),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None)]), None)]))
]),
(Token.BLOCK, [
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.UOP, _UNARY_OPERATORS['++'][1])
]), None)]),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None)]), None)]))
]))
]))
]
self.assertEqual(list(jsi.statements()), ast) self.assertEqual(list(jsi.statements()), ast)
@unittest.skip('Test not yet implemented: missing code and ast') @unittest.skip('Test not yet implemented: missing code and ast')

17
youtube_dl/jsinterp/jsinterp.py
View File

@ -66,6 +66,7 @@ class JSInterpreter(object):
token_id, token_value, token_pos = token_stream.peek() token_id, token_value, token_pos = token_stream.peek()
if token_id in (Token.CCLOSE, Token.END): if token_id in (Token.CCLOSE, Token.END):
# empty statement goes straight here # empty statement goes straight here
token_stream.pop()
return statement return statement
if token_id is Token.ID and token_value == 'function': if token_id is Token.ID and token_value == 'function':
@ -111,11 +112,11 @@ class JSInterpreter(object):
while True: while True:
token_id, token_value, token_pos = token_stream.peek() token_id, token_value, token_pos = token_stream.peek()
if token_id is Token.CCLOSE: if token_id is Token.CCLOSE:
token_stream.pop()
break break
elif token_id is Token.END and token_stream.ended: elif token_id is Token.END and token_stream.ended:
raise ExtractorError('Unbalanced parentheses at %d' % open_pos) raise ExtractorError('Unbalanced parentheses at %d' % open_pos)
block.append(self._next_statement(token_stream, stack_top - 1)) block.append(self._next_statement(token_stream, stack_top - 1))
token_stream.pop()
statement = (Token.BLOCK, block) statement = (Token.BLOCK, block)
@ -142,6 +143,7 @@ class JSInterpreter(object):
init.append(JSInterpreter.undefined) init.append(JSInterpreter.undefined)
if peek_id is Token.END: if peek_id is Token.END:
token_stream.pop()
has_another = False has_another = False
elif peek_id is Token.COMMA: elif peek_id is Token.COMMA:
pass pass
@ -161,11 +163,8 @@ class JSInterpreter(object):
cond_expr = self._expression(token_stream, stack_top - 1) cond_expr = self._expression(token_stream, stack_top - 1)
token_stream.pop() # Token.PCLOSE token_stream.pop() # Token.PCLOSE
true_expr = self._next_statement(token_stream, stack_top - 1) true_expr = self._next_statement(token_stream, stack_top - 1)
token_id, token_value, token_pos = token_stream.peek()
if token_id is Token.CCLOSE:
token_stream.pop()
token_id, token_value, token_pos = token_stream.peek()
false_expr = None false_expr = None
token_id, token_value, token_pos = token_stream.peek()
if token_id is Token.ID and token_value == 'else': if token_id is Token.ID and token_value == 'else':
token_stream.pop() token_stream.pop()
false_expr = self._next_statement(token_stream, stack_top - 1) false_expr = self._next_statement(token_stream, stack_top - 1)
@ -190,6 +189,8 @@ class JSInterpreter(object):
if peek_id is not Token.END: if peek_id is not Token.END:
# FIXME automatic end insertion # FIXME automatic end insertion
raise ExtractorError('Unexpected sequence %s at %d' % (peek_value, peek_pos)) raise ExtractorError('Unexpected sequence %s at %d' % (peek_value, peek_pos))
else:
token_stream.pop()
elif token_value == 'return': elif token_value == 'return':
token_stream.pop() token_stream.pop()
@ -201,6 +202,8 @@ class JSInterpreter(object):
if peek_id is not Token.END: if peek_id is not Token.END:
# FIXME automatic end insertion # FIXME automatic end insertion
raise ExtractorError('Unexpected sequence %s at %d' % (peek_value, peek_pos)) raise ExtractorError('Unexpected sequence %s at %d' % (peek_value, peek_pos))
else:
token_stream.pop()
elif token_value == 'with': elif token_value == 'with':
token_stream.pop() token_stream.pop()
@ -288,6 +291,7 @@ class JSInterpreter(object):
peek_id, peek_value, peek_pos = token_stream.peek() peek_id, peek_value, peek_pos = token_stream.peek()
if peek_id is Token.END: if peek_id is Token.END:
token_stream.pop()
has_another = False has_another = False
elif peek_id is Token.COMMA: elif peek_id is Token.COMMA:
pass pass
@ -305,7 +309,7 @@ class JSInterpreter(object):
while not ts.ended: while not ts.ended:
yield self._next_statement(ts, stack_size) yield self._next_statement(ts, stack_size)
ts.pop() # ts.pop()
raise StopIteration raise StopIteration
def _expression(self, token_stream, stack_top): def _expression(self, token_stream, stack_top):
@ -584,6 +588,7 @@ class JSInterpreter(object):
if peek_id is Token.REL: if peek_id is Token.REL:
name, op = peek_value name, op = peek_value
prec = 11
elif peek_id is Token.OP: elif peek_id is Token.OP:
name, op = peek_value name, op = peek_value
if name in (Token.MUL, Token.DIV, Token.MOD): if name in (Token.MUL, Token.DIV, Token.MOD):

11
youtube_dl/jsinterp/tstream.py
View File

@ -158,10 +158,17 @@ class TokenStream(object):
self.peeked.append(token) self.peeked.append(token)
return self.peeked[count - 1] return self.peeked[count - 1]
def pop(self): def pop(self, count=1):
if not self.peeked: if not self.peeked:
self.peek() self.peek()
self._last = self.peeked.pop(0) for _ in range(count):
self._last = self.peeked.pop()
return self._last
def flush(self):
if self.peeked:
self._last = self.peeked[-1]
self.peeked = []
return self._last return self._last
def last(self): def last(self):