R-yan/l1ving_youtube-dl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[jsinterp] Adding ast to for parser test

Browse Source
This commit is contained in:
sulyi 2016-12-12 22:56:07 +01:00
parent 4b8754c6b0
commit b397ea2bdd
3 changed files with 51 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
test/test_jsinterp_parser.py
View File

@ -789,28 +789,50 @@ class TestJSInterpreterParser(unittest.TestCase):
) )
]) ])
]) ])
] ]
), ),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [ (Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None)]), None)])) (Token.MEMBER, (Token.ID, 'x'), None, None)]), None)]))
])) ]))
] ]
result = list(jsi.statements()) self.assertEqual(list(jsi.statements()), ast)
self.assertEqual(result, ast)
@unittest.skip('Test not yet implemented: missing ast')
def test_for(self): def test_for(self):
# ASAP for statement test
jsi = JSInterpreter(''' jsi = JSInterpreter('''
function f(x){ function f(x){
for (var h = 0; h <= x; ++h) { for (var h = 0; h <= x; ++h) {
a = h; a = h;
} }
return a return a;
} }
''') ''')
ast = [] ast = [
(Token.FUNC, 'f', ['x'],
(Token.BLOCK, [
(Token.FOR,
(Token.VAR, zip(['h'], [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 0), None, None)]), None)
])),
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'h'), None, None),
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.REL, _RELATIONS['<='][1])
]), None)]),
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'h'), None, None),
(Token.UOP, _UNARY_OPERATORS['++'][1])
]), None)]),
(Token.BLOCK, [
(Token.EXPR, [
(Token.ASSIGN, _ASSIGN_OPERATORS['='][1],
(Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'a'), None, None)]),
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'h'), None, None)]), None))
])
])),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'a'), None, None)]), None)]))
]))
]
self.assertEqual(list(traverse(list(jsi.statements()))), list(traverse(ast))) self.assertEqual(list(traverse(list(jsi.statements()))), list(traverse(ast)))
@unittest.skip('Test not yet implemented: missing ast') @unittest.skip('Test not yet implemented: missing ast')
@ -822,7 +844,7 @@ class TestJSInterpreterParser(unittest.TestCase):
for (; h <= x; ++h) { for (; h <= x; ++h) {
a = h; a = h;
} }
return a return a;
} }
''') ''')
ast = [] ast = []
@ -836,7 +858,7 @@ class TestJSInterpreterParser(unittest.TestCase):
for (h in z) { for (h in z) {
a = h; a = h;
} }
return a return a;
} }
''') ''')
ast = [] ast = []

2
youtube_dl/jsinterp/jsgrammar.py
View File

@ -7,7 +7,7 @@ from collections import namedtuple
_token_keys = ('COPEN', 'CCLOSE', 'POPEN', 'PCLOSE', 'SOPEN', 'SCLOSE', _token_keys = ('COPEN', 'CCLOSE', 'POPEN', 'PCLOSE', 'SOPEN', 'SCLOSE',
'DOT', 'END', 'COMMA', 'HOOK', 'COLON', 'DOT', 'END', 'COMMA', 'HOOK', 'COLON',
'AND', 'OR', 'INC', 'DEC', 'NOT', 'BNOT', 'DEL', 'VOID', 'TYPE', 'AND', 'OR', 'INC', 'DEC', 'NOT', 'BNOT', 'DEL', 'VOID', 'TYPE',
'LT', 'GT', 'LE', 'GE', 'EQ', 'NE', 'SEQ', 'SNE', 'LT', 'GT', 'LE', 'GE', 'EQ', 'NE', 'SEQ', 'SNE', 'IN', 'INSTANCEOF',
'BOR', 'BXOR', 'BAND', 'RSHIFT', 'LSHIFT', 'URSHIFT', 'SUB', 'ADD', 'MOD', 'DIV', 'MUL', 'BOR', 'BXOR', 'BAND', 'RSHIFT', 'LSHIFT', 'URSHIFT', 'SUB', 'ADD', 'MOD', 'DIV', 'MUL',
'OP', 'AOP', 'UOP', 'LOP', 'REL', 'OP', 'AOP', 'UOP', 'LOP', 'REL',
'COMMENT', 'TOKEN', 'PUNCT', 'COMMENT', 'TOKEN', 'PUNCT',

33
youtube_dl/jsinterp/jsinterp.py
View File

@ -12,6 +12,7 @@ _token_keys = set((Token.NULL, Token.BOOL, Token.ID, Token.STR, Token.INT, Token
class Context(object): class Context(object):
def __init__(self, variables=None, ended=False): def __init__(self, variables=None, ended=False):
self.ended = ended self.ended = ended
self.no_in = True
self.local_vars = {} self.local_vars = {}
if variables is not None: if variables is not None:
for k, v in dict(variables).items(): for k, v in dict(variables).items():
@ -54,7 +55,7 @@ class JSInterpreter(object):
for k, v in dict(variables).items(): for k, v in dict(variables).items():
# XXX validate identifiers # XXX validate identifiers
self.global_vars[k] = Reference(v, (self.global_vars, k)) self.global_vars[k] = Reference(v, (self.global_vars, k))
self.context = Context(self.global_vars) self._context = Context(self.global_vars)
self._context_stack = [] self._context_stack = []
def statements(self, code=None, pos=0, stack_size=100): def statements(self, code=None, pos=0, stack_size=100):
@ -101,6 +102,7 @@ class JSInterpreter(object):
elif token_id is Token.ID: elif token_id is Token.ID:
if token_value == 'var': if token_value == 'var':
# XXX refactor (create dedicated method for handling variable declaration list)
token_stream.pop() token_stream.pop()
variables = [] variables = []
init = [] init = []
@ -121,7 +123,8 @@ class JSInterpreter(object):
init.append(JSInterpreter.undefined) init.append(JSInterpreter.undefined)
if peek_id is Token.END: if peek_id is Token.END:
token_stream.pop() if self._context.no_in:
token_stream.pop()
has_another = False has_another = False
elif peek_id is Token.COMMA: elif peek_id is Token.COMMA:
pass pass
@ -244,14 +247,16 @@ class JSInterpreter(object):
raise ExtractorError('''Expected '(' at %d''' % token_pos) raise ExtractorError('''Expected '(' at %d''' % token_pos)
# FIXME set infor True (checked by variable declaration and relation expression) # FIXME set infor True (checked by variable declaration and relation expression)
self._context.no_in = False
token_id, token_value, token_pos = token_stream.peek() token_id, token_value, token_pos = token_stream.peek()
if token_id is Token.END: if token_id is Token.END:
init = None init = None
elif token_id.ID and token_value == 'var': elif token_id is Token.ID and token_value == 'var':
# XXX refactor (create dedicated method for handling variable declaration list)
init = self._statement(token_stream, stack_top - 1) init = self._statement(token_stream, stack_top - 1)
else: else:
init = self._expression(token_stream, stack_top - 1) init = self._expression(token_stream, stack_top - 1)
self._context.no_in = True
token_id, token_value, token_pos = token_stream.pop() token_id, token_value, token_pos = token_stream.pop()
if token_id is Token.IN: if token_id is Token.IN:
cond = self._expression(token_stream, stack_top - 1) cond = self._expression(token_stream, stack_top - 1)
@ -263,8 +268,8 @@ class JSInterpreter(object):
cond = None if token_id is Token.END else self._expression(token_stream, stack_top - 1) cond = None if token_id is Token.END else self._expression(token_stream, stack_top - 1)
token_id, token_value, token_pos = token_stream.pop() token_id, token_value, token_pos = token_stream.pop()
if token_id is not Token.PCLOSE: if token_id is not Token.END:
raise ExtractorError('''Expected ')' at %d''' % token_pos) raise ExtractorError('''Expected ';' at %d''' % token_pos)
token_id, token_value, token_pos = token_stream.peek() token_id, token_value, token_pos = token_stream.peek()
incr = None if token_id is Token.END else self._expression(token_stream, stack_top - 1) incr = None if token_id is Token.END else self._expression(token_stream, stack_top - 1)
@ -825,8 +830,8 @@ class JSInterpreter(object):
ref = s.getvalue() ref = s.getvalue()
elif name is Token.VAR: elif name is Token.VAR:
for name, value in stmt[1]: for name, value in stmt[1]:
self.context.local_vars[name] = Reference(self.interpret_expression(value).getvalue(), self._context.local_vars[name] = Reference(self.interpret_expression(value).getvalue(),
(self.context.local_vars, name)) (self._context.local_vars, name))
elif name is Token.EXPR: elif name is Token.EXPR:
for expr in stmt[1]: for expr in stmt[1]:
ref = self.interpret_expression(expr) ref = self.interpret_expression(expr)
@ -839,7 +844,7 @@ class JSInterpreter(object):
# TODO test nested arrays # TODO test nested arrays
ref = [elem.getvalue() for elem in ref] ref = [elem.getvalue() for elem in ref]
self.context.ended = True self._context.ended = True
# with # with
# label # label
# switch # switch
@ -913,7 +918,7 @@ class JSInterpreter(object):
elif name is Token.ID: elif name is Token.ID:
# XXX error handling (unknown id) # XXX error handling (unknown id)
ref = self.context.local_vars[expr[1]] if expr[1] in self.context.local_vars else self.global_vars[expr[1]] ref = self._context.local_vars[expr[1]] if expr[1] in self._context.local_vars else self.global_vars[expr[1]]
# literal # literal
elif name in _token_keys: elif name in _token_keys:
@ -966,12 +971,12 @@ class JSInterpreter(object):
return self.build_function(argnames, func_m.group('code')) return self.build_function(argnames, func_m.group('code'))
def push_context(self, cx): def push_context(self, cx):
self._context_stack.append(self.context) self._context_stack.append(self._context)
self.context = cx self._context = cx
def pop_context(self): def pop_context(self):
# XXX check underflow # XXX check underflow
self.context = self._context_stack.pop() self._context = self._context_stack.pop()
def call_function(self, funcname, *args): def call_function(self, funcname, *args):
f = self.extract_function(funcname) f = self.extract_function(funcname)
@ -982,7 +987,7 @@ class JSInterpreter(object):
self.push_context(Context(dict(zip(argnames, args)))) self.push_context(Context(dict(zip(argnames, args))))
for stmt in self.statements(code): for stmt in self.statements(code):
res = self.interpret_statement(stmt) res = self.interpret_statement(stmt)
if self.context.ended: if self._context.ended:
self.pop_context() self.pop_context()
break break
return res return res