R-yan/l1ving_youtube-dl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[jsinterp] Adding function declaration and call

Browse Source 
Refractors ast:
 * function declaration is no longer a statement
 * function body is no longer a block
This commit is contained in:
sulyi 2016-12-17 01:13:03 +01:00
parent dca2e9e965
commit 3b536690d7
12 changed files with 456 additions and 391 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
test/jstests/branch.py
View File

@ -15,21 +15,18 @@ tests = [
''',
'asserts': [{'value': True, 'call': ('a', 1)}, {'value': False, 'call': ('a', 0)}],
'ast': [
(Token.FUNC, 'a',
['x'],
(Token.BLOCK, [
(Token.IF,
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.MEMBER, (Token.INT, 0), None, None),
(Token.REL, _RELATIONS['>'][1])
]), None)]),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.BOOL, True), None, None)]), None)])),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.BOOL, False), None, None)]), None)])))
]))
(Token.FUNC, 'a', ['x'], [
(Token.IF,
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.MEMBER, (Token.INT, 0), None, None),
(Token.REL, _RELATIONS['>'][1])
]), None)]),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.BOOL, True), None, None)]), None)])),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.BOOL, False), None, None)]), None)])))
])
]
}
]

166
test/jstests/call.py
View File

@ -1,111 +1,109 @@
from youtube_dl.jsinterp.jsgrammar import Token
from youtube_dl.jsinterp.tstream import _OPERATORS
skip = {'i': 'Interpreting function call not yet implemented'}
tests = [
{
'code': '''
function x() { return 2; }
function y(a) { return x() + a; }
function z() { return y(3); }
z();
''',
'asserts': [{'value': 5, 'call': ('z',)}],
'asserts': [{'value': 5}],
'ast': [
(Token.FUNC, 'x',
[],
(Token.BLOCK, [
(Token.RETURN, (Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 2), None, None)]), None)
])
)
])),
(Token.FUNC, 'y',
['a'],
(Token.BLOCK, [
(Token.RETURN, (Token.EXPR, [
(Token.ASSIGN, None,
(Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, (Token.CALL, [], None)),
(Token.MEMBER, (Token.ID, 'a'), None, None),
(Token.OP, _OPERATORS['+'][1])
]), None)
])
)
])),
(Token.FUNC, 'z',
[],
(Token.BLOCK, [
(Token.RETURN, (Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'y'), None, (Token.CALL, [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 3), None, None)]), None)
], None))
(Token.FUNC, 'x', [], [
(Token.RETURN, (Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 2), None, None)]), None)
]))
]),
(Token.FUNC, 'y', ['a'], [
(Token.RETURN, (Token.EXPR, [
(Token.ASSIGN, None,
(Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, (Token.CALL, [], None)),
(Token.MEMBER, (Token.ID, 'a'), None, None),
(Token.OP, _OPERATORS['+'][1])
]), None)
])
)
]))
]))
]),
(Token.FUNC, 'z', [], [
(Token.RETURN, (Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'y'), None, (Token.CALL, [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 3), None, None)]), None)
], None))
]), None)
])
)
]),
(Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'z'), None, (Token.CALL, [], None))
]), None)
])
]
}, {
'code': 'function x(a) { return a.split(""); }',
'asserts': [{'value': ["a", "b", "c"], 'call': ('x',)}],
# built-in functions not yet implemented
# 'asserts': [{'value': ["a", "b", "c"], 'call': ('x',"abc")}],
'ast': [
(Token.FUNC, 'x',
['a'],
(Token.BLOCK, [
(Token.RETURN, (Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'a'), None,
(Token.FIELD, 'split',
(Token.CALL, [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.STR, ''), None, None)]), None)
], None))
)]),
None)
])
)
]))
(Token.FUNC, 'x', ['a'], [
(Token.RETURN, (Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'a'), None,
(Token.FIELD, 'split',
(Token.CALL, [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.STR, ''), None, None)]), None)
], None))
)]),
None)
]))
])
]
}, {
'code': '''
function a(x) { return x; }
function b(x) { return x; }
function b(x) { return x + 1; }
function c() { return [a, b][0](0); }
c();
''',
'asserts': [{'value': 0}],
'ast': [
(Token.FUNC, 'a', ['x'],
(Token.BLOCK, [
(Token.RETURN, (Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'x'), None, None)]), None)
])
)
])),
(Token.FUNC, 'b', ['x'],
(Token.BLOCK, [
(Token.RETURN, (Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'x'), None, None)]), None)
])
)
])),
(Token.FUNC, 'c', [],
(Token.BLOCK, [
(Token.RETURN, (Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ARRAY, [
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'a'), None, None)]), None),
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'b'), None, None)]), None)
]), None, (Token.ELEM, (Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 0), None, None)]), None)
]), (Token.CALL, [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 0), None, None)]), None)
], None)))
]), None)
])
)
]))
(Token.FUNC, 'a', ['x'], [
(Token.RETURN, (Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'x'), None, None)]), None)
]))
]),
(Token.FUNC, 'b', ['x'], [
(Token.RETURN, (Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.MEMBER, (Token.INT, 1), None, None),
(Token.OP, _OPERATORS['+'][1])
]), None)
]))
]),
(Token.FUNC, 'c', [], [
(Token.RETURN, (Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ARRAY, [
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'a'), None, None)]), None),
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'b'), None, None)]), None)
]), None, (Token.ELEM, (Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 0), None, None)]), None)
]), (Token.CALL, [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 0), None, None)]), None)
], None)))
]), None)
]))
]),
(Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'c'), None, (Token.CALL, [], None))
]), None)
])
]
}
]

47
test/jstests/do_loop.py
View File

@ -16,32 +16,31 @@ tests = [
''',
'asserts': [{'value': 5, 'call': 5}],
'ast': [
(Token.FUNC, 'f', ['x'],
(Token.BLOCK, [
(Token.FUNC, 'f', ['x'], [
(Token.EXPR, [
(Token.ASSIGN, _ASSIGN_OPERATORS['='][1],
(Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'i'), None, None)]),
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 1), None, None)]), None))
]),
(Token.DO,
(Token.EXPR, [
(Token.ASSIGN, _ASSIGN_OPERATORS['='][1],
(Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'i'), None, None)]),
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 1), None, None)]), None))
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'i'), None, None),
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.REL, _RELATIONS['<'][1])
]), None)
]),
(Token.DO,
(Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'i'), None, None),
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.REL, _RELATIONS['<'][1])
]), None)
]),
(Token.BLOCK, [
(Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'i'), None, None),
(Token.POSTFIX, _UNARY_OPERATORS['++'][1])
]), None)
])
])),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'i'), None, None)]), None)]))
]))
(Token.BLOCK, [
(Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'i'), None, None),
(Token.POSTFIX, _UNARY_OPERATORS['++'][1])
]), None)
])
])),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'i'), None, None)]), None)]))
])
]
}
]

49
test/jstests/for_empty.py
View File

@ -16,32 +16,31 @@ tests = [
''',
'asserts': [{'value': 5, 'call': ('f', 5)}],
'ast': [
(Token.FUNC, 'f', ['x'],
(Token.BLOCK, [
(Token.VAR, zip(['h'], [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 0), None, None)]), None)
(Token.FUNC, 'f', ['x'], [
(Token.VAR, zip(['h'], [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 0), None, None)]), None)
])),
(Token.FOR,
None,
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'h'), None, None),
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.REL, _RELATIONS['<='][1])
]), None)]),
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'h'), None, None),
(Token.PREFIX, _UNARY_OPERATORS['++'][1])
]), None)]),
(Token.BLOCK, [
(Token.EXPR, [
(Token.ASSIGN, _ASSIGN_OPERATORS['='][1],
(Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'a'), None, None)]),
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'h'), None, None)]), None))
])
])),
(Token.FOR,
None,
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'h'), None, None),
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.REL, _RELATIONS['<='][1])
]), None)]),
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'h'), None, None),
(Token.PREFIX, _UNARY_OPERATORS['++'][1])
]), None)]),
(Token.BLOCK, [
(Token.EXPR, [
(Token.ASSIGN, _ASSIGN_OPERATORS['='][1],
(Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'a'), None, None)]),
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'h'), None, None)]), None))
])
])),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'a'), None, None)]), None)]))
]))
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'a'), None, None)]), None)]))
])
]
}
]

39
test/jstests/for_in.py
View File

@ -15,26 +15,25 @@ tests = [
''',
'asserts': [{'value': 'c', 'call': ('f', ['a', 'b', 'c'])}],
'ast': [
(Token.FUNC, 'f', ['z'],
(Token.BLOCK, [
(Token.FOR,
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'h'), None, None)
]), None)]),
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'z'), None, None)
]), None)]),
None,
(Token.BLOCK, [
(Token.EXPR, [
(Token.ASSIGN, _ASSIGN_OPERATORS['='][1],
(Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'a'), None, None)]),
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'h'), None, None)]), None))
])
])),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'a'), None, None)]), None)]))
]))
(Token.FUNC, 'f', ['z'], [
(Token.FOR,
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'h'), None, None)
]), None)]),
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'z'), None, None)
]), None)]),
None,
(Token.BLOCK, [
(Token.EXPR, [
(Token.ASSIGN, _ASSIGN_OPERATORS['='][1],
(Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'a'), None, None)]),
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'h'), None, None)]), None))
])
])),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'a'), None, None)]), None)]))
])
]
}
]

49
test/jstests/for_loop.py
View File

@ -15,31 +15,30 @@ tests = [
''',
'asserts': [{'value': 5, 'call': ('f', 5)}],
'ast': [
(Token.FUNC, 'f', ['x'],
(Token.BLOCK, [
(Token.FOR,
(Token.VAR, zip(['h'], [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 0), None, None)]), None)
])),
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'h'), None, None),
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.REL, _RELATIONS['<='][1])
]), None)]),
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'h'), None, None),
(Token.PREFIX, _UNARY_OPERATORS['++'][1])
]), None)]),
(Token.BLOCK, [
(Token.EXPR, [
(Token.ASSIGN, _ASSIGN_OPERATORS['='][1],
(Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'a'), None, None)]),
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'h'), None, None)]), None))
])
])),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'a'), None, None)]), None)]))
]))
(Token.FUNC, 'f', ['x'], [
(Token.FOR,
(Token.VAR, zip(['h'], [
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 0), None, None)]), None)
])),
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'h'), None, None),
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.REL, _RELATIONS['<='][1])
]), None)]),
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'h'), None, None),
(Token.PREFIX, _UNARY_OPERATORS['++'][1])
]), None)]),
(Token.BLOCK, [
(Token.EXPR, [
(Token.ASSIGN, _ASSIGN_OPERATORS['='][1],
(Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'a'), None, None)]),
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'h'), None, None)]), None))
])
])),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'a'), None, None)]), None)]))
])
]
}
]

69
test/jstests/func_expr.py
View File

@ -18,41 +18,40 @@ tests = [
''',
'asserts': [{'value': 3, 'call': ('f',)}],
'ast': [
(Token.FUNC, 'f', [],
(Token.BLOCK, [
(Token.VAR, zip(['add'], [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.FUNC, None, [], (Token.BLOCK, [
(Token.VAR, zip(
['counter'],
[(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.INT, 0), None, None)
]), None)]
)),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.FUNC, None, [], (Token.BLOCK, [
(Token.RETURN, (Token.EXPR, [
(Token.ASSIGN, _ASSIGN_OPERATORS['+='][1], (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'counter'), None, None)
]), (Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.INT, 1), None, None)
]), None))
]))
])), None, None)
]), None)]))
])), None, None),
]), None)]), None, (Token.CALL, [], None))
]), None)])),
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'add'), None, (Token.CALL, [], None))
]), None)]),
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'add'), None, (Token.CALL, [], None))
]), None)]),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'add'), None, (Token.CALL, [], None))
]), None)]))
]))
(Token.FUNC, 'f', [], [
(Token.VAR, zip(['add'], [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.FUNC, None, [], [
(Token.VAR, zip(
['counter'],
[(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.INT, 0), None, None)
]), None)]
)),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.FUNC, None, [], [
(Token.RETURN, (Token.EXPR, [
(Token.ASSIGN, _ASSIGN_OPERATORS['+='][1], (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'counter'), None, None)
]), (Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.INT, 1), None, None)
]), None))
]))
]), None, None)
]), None)]))
]), None, None),
]), None)]), None, (Token.CALL, [], None))
]), None)])),
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'add'), None, (Token.CALL, [], None))
]), None)]),
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'add'), None, (Token.CALL, [], None))
]), None)]),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'add'), None, (Token.CALL, [], None))
]), None)]))
])
]
}
]

75
test/jstests/object_literal.py
View File

@ -16,44 +16,43 @@ tests = [
}
''',
'ast': [
(Token.FUNC, 'f', [],
(Token.BLOCK, [
(Token.VAR,
zip(['o'],
[(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.OBJECT, [
('a', (Token.PROPVALUE, (Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.INT, 7), None, None)
]), None))),
('b', (Token.PROPGET, (Token.BLOCK, [
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.RSV, 'this'), None, (Token.FIELD, 'a', None)),
(Token.MEMBER, (Token.INT, 1), None, None),
(Token.OP, _OPERATORS['+'][1])
]), None)]))
]))),
('c', (Token.PROPSET, 'x', (Token.BLOCK, [
(Token.EXPR, [
(Token.ASSIGN,
_ASSIGN_OPERATORS['='][1],
(Token.OPEXPR, [
(Token.MEMBER, (Token.RSV, 'this'), None, (Token.FIELD, 'a', None))
]),
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.MEMBER, (Token.INT, 2), None, None),
(Token.OP, _OPERATORS['/'][1])
]), None))
])
])))
]),
None, None)
]), None)]
)
),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'o'), None, None)]), None)]))
]))
(Token.FUNC, 'f', [], [
(Token.VAR,
zip(['o'],
[(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.OBJECT, [
('a', (Token.PROPVALUE, (Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.INT, 7), None, None)
]), None))),
('b', (Token.PROPGET, [
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.RSV, 'this'), None, (Token.FIELD, 'a', None)),
(Token.MEMBER, (Token.INT, 1), None, None),
(Token.OP, _OPERATORS['+'][1])
]), None)]))
])),
('c', (Token.PROPSET, 'x', [
(Token.EXPR, [
(Token.ASSIGN,
_ASSIGN_OPERATORS['='][1],
(Token.OPEXPR, [
(Token.MEMBER, (Token.RSV, 'this'), None, (Token.FIELD, 'a', None))
]),
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.MEMBER, (Token.INT, 2), None, None),
(Token.OP, _OPERATORS['/'][1])
]), None))
])
]))
]),
None, None)
]), None)]
)
),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'o'), None, None)]), None)]))
])
]
}
]

86
test/jstests/switch.py
View File

@ -26,48 +26,48 @@ tests = [
{'value': 6, 'call': ('a', 6)},
{'value': 8, 'call': ('a', 7)}],
'ast': [
(Token.FUNC, 'a', ['x'],
(Token.BLOCK, [
(Token.SWITCH, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None)
]), None)]),
[
((Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.INT, 6), None, None)]), None)]),
[
(Token.BREAK, None)
]),
((Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.INT, 5), None, None)]), None)]),
[
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.POSTFIX, _UNARY_OPERATORS['++'][1])
]), None)])
]),
((Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.INT, 8), None, None)]), None)]),
[
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.POSTFIX, _UNARY_OPERATORS['--'][1])
]), None)]),
(Token.BREAK, None)
]),
(None,
[
(Token.EXPR, [
(Token.ASSIGN,
_ASSIGN_OPERATORS['='][1],
(Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'x'), None, None)]),
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 0), None, None)]), None)
)
])
])
]
),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None)]), None)]))
]))]
(Token.FUNC, 'a', ['x'], [
(Token.SWITCH, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None)
]), None)]),
[
((Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.INT, 6), None, None)]), None)]),
[
(Token.BREAK, None)
]),
((Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.INT, 5), None, None)]), None)]),
[
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.POSTFIX, _UNARY_OPERATORS['++'][1])
]), None)])
]),
((Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.INT, 8), None, None)]), None)]),
[
(Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.POSTFIX, _UNARY_OPERATORS['--'][1])
]), None)]),
(Token.BREAK, None)
]),
(None,
[
(Token.EXPR, [
(Token.ASSIGN,
_ASSIGN_OPERATORS['='][1],
(Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'x'), None, None)]),
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 0), None, None)]), None)
)
])
])
]
),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'x'), None, None)]), None)]))
])
]
}
]

47
test/jstests/while_loop.py
View File

@ -16,32 +16,31 @@ tests = [
''',
'asserts': [{'value': 5, 'call': ('f', 5)}],
'ast': [
(Token.FUNC, 'f', ['x'],
(Token.BLOCK, [
(Token.FUNC, 'f', ['x'], [
(Token.EXPR, [
(Token.ASSIGN, _ASSIGN_OPERATORS['='][1],
(Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'i'), None, None)]),
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 1), None, None)]), None))
]),
(Token.WHILE,
(Token.EXPR, [
(Token.ASSIGN, _ASSIGN_OPERATORS['='][1],
(Token.OPEXPR, [(Token.MEMBER, (Token.ID, 'i'), None, None)]),
(Token.ASSIGN, None, (Token.OPEXPR, [(Token.MEMBER, (Token.INT, 1), None, None)]), None))
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'i'), None, None),
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.REL, _RELATIONS['<'][1])
]), None)
]),
(Token.WHILE,
(Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'i'), None, None),
(Token.MEMBER, (Token.ID, 'x'), None, None),
(Token.REL, _RELATIONS['<'][1])
]), None)
]),
(Token.BLOCK, [
(Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'i'), None, None),
(Token.POSTFIX, _UNARY_OPERATORS['++'][1])
]), None)
])
])),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'i'), None, None)]), None)]))
]))
(Token.BLOCK, [
(Token.EXPR, [
(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'i'), None, None),
(Token.POSTFIX, _UNARY_OPERATORS['++'][1])
]), None)
])
])),
(Token.RETURN, (Token.EXPR, [(Token.ASSIGN, None, (Token.OPEXPR, [
(Token.MEMBER, (Token.ID, 'i'), None, None)]), None)]))
])
]
}
]

2
test/test_jsinterp_parse.py
View File

@ -43,7 +43,7 @@ def generator(test_case, name):
def test_template(self):
for a in test_case['subtests']:
jsi = JSInterpreter(a['code'], variables=None if 'globals' not in a else a['globals'])
parsed = list(jsi.statements())
parsed = list(jsi.parse())
if 'ast' in a:
self.assertEqual(traverse(parsed), traverse(a['ast']))
else:

191
youtube_dl/jsinterp/jsinterp.py
View File

@ -24,8 +24,18 @@ class Reference(object):
self._value = value
self._parent = parent
def getvalue(self):
return self._value
def getvalue(self, deep=False):
value = self._value
if deep:
if isinstance(self._value, (list, tuple)):
# TODO test nested arrays
value = [elem.getvalue() for elem in self._value]
elif isinstance(self._value, dict):
value = {}
for key, prop in self._value.items():
value[key] = prop.getvalue()
return value
def putvalue(self, value):
if self._parent is None:
@ -34,6 +44,7 @@ class Reference(object):
if not hasattr(parent, '__setitem__'):
raise ExtractorError('Unknown reference')
parent.__setitem__(key, Reference(value, (parent, key)))
self._value = value
return value
def __repr__(self):
@ -62,12 +73,12 @@ class JSInterpreter(object):
def this(self):
return self._context.local_vars
def statements(self, code=None, pos=0, stack_size=100):
def parse(self, code=None, pos=0, stack_size=100):
if code is None:
code = self.code
ts = TokenStream(code, pos)
while not ts.ended:
yield self._statement(ts, stack_size)
yield self._source_element(ts, stack_size)
raise StopIteration
def create_reference(self, value, parent_key):
@ -86,23 +97,29 @@ class JSInterpreter(object):
return Reference(o, parent_key)
def _source_element(self, token_stream, stack_top):
if stack_top < 0:
raise ExtractorError('Recursion limit reached')
token_id, token_value, token_pos = token_stream.peek()
if token_id is Token.ID and token_value == 'function':
source_element = self._function(token_stream, stack_top - 1)
else:
source_element = self._statement(token_stream, stack_top - 1)
return source_element
def _statement(self, token_stream, stack_top):
if stack_top < 0:
raise ExtractorError('Recursion limit reached')
# ast
statement = None
statement = None
token_id, token_value, token_pos = token_stream.peek()
if token_id is Token.END:
# empty statement goes straight here
token_stream.pop()
return statement
elif token_id is Token.ID and token_value == 'function':
# FIXME allowed only in program and function body
# main, function expr, object literal (set, get), function declaration
statement = self._function(token_stream, stack_top - 1)
# block
elif token_id is Token.COPEN:
# XXX refactor will deprecate some _statement calls
@ -245,21 +262,27 @@ class JSInterpreter(object):
return statement
def _if_statement(self, token_stream, stack_top):
if stack_top < 0:
raise ExtractorError('Recursion limit reached')
token_stream.pop()
token_id, token_value, token_pos = token_stream.pop()
if token_id is not Token.POPEN:
raise ExtractorError('Missing condition at %d' % token_pos)
cond_expr = self._expression(token_stream, stack_top - 1)
token_stream.pop() # Token.PCLOSE
true_expr = self._statement(token_stream, stack_top - 1)
false_expr = None
true_stmt = self._statement(token_stream, stack_top - 1)
false_stmt = None
token_id, token_value, token_pos = token_stream.peek()
if token_id is Token.ID and token_value == 'else':
token_stream.pop()
false_expr = self._statement(token_stream, stack_top - 1)
return (Token.IF, cond_expr, true_expr, false_expr)
false_stmt = self._statement(token_stream, stack_top - 1)
return (Token.IF, cond_expr, true_stmt, false_stmt)
def _for_loop(self, token_stream, stack_top):
if stack_top < 0:
raise ExtractorError('Recursion limit reached')
token_stream.pop()
token_id, token_value, token_pos = token_stream.pop()
if token_id is not Token.POPEN:
@ -302,6 +325,9 @@ class JSInterpreter(object):
return (Token.FOR, init, cond, incr, body)
def _do_loop(self, token_stream, stack_top):
if stack_top < 0:
raise ExtractorError('Recursion limit reached')
token_stream.pop()
body = self._statement(token_stream, stack_top - 1)
token_id, token_value, token_pos = token_stream.pop()
@ -323,6 +349,9 @@ class JSInterpreter(object):
return (Token.DO, expr, body)
def _while_loop(self, token_stream, stack_top):
if stack_top < 0:
raise ExtractorError('Recursion limit reached')
token_stream.pop()
token_id, token_value, token_pos = token_stream.pop()
if token_id is not Token.POPEN:
@ -335,6 +364,9 @@ class JSInterpreter(object):
return (Token.WHILE, expr, body)
def _return_statement(self, token_stream, stack_top):
if stack_top < 0:
raise ExtractorError('Recursion limit reached')
token_stream.pop()
peek_id, peek_value, peek_pos = token_stream.peek()
# XXX no line break here
@ -342,6 +374,9 @@ class JSInterpreter(object):
return (Token.RETURN, expr)
def _with_statement(self, token_stream, stack_top):
if stack_top < 0:
raise ExtractorError('Recursion limit reached')
token_stream.pop()
token_id, token_value, token_pos = token_stream.pop()
if token_id is not Token.POPEN:
@ -351,6 +386,9 @@ class JSInterpreter(object):
return (Token.WITH, expr, self._statement(token_stream, stack_top - 1))
def _switch_statement(self, token_stream, stack_top):
if stack_top < 0:
raise ExtractorError('Recursion limit reached')
token_stream.pop()
token_id, token_value, token_pos = token_stream.pop()
if token_id is not Token.POPEN:
@ -402,6 +440,9 @@ class JSInterpreter(object):
return (Token.SWITCH, discriminant, block)
def _try_statement(self, token_stream, stack_top):
if stack_top < 0:
raise ExtractorError('Recursion limit reached')
token_stream.pop()
token_id, token_value, token_pos = token_stream.peek()
if token_id is not Token.COPEN:
@ -434,6 +475,9 @@ class JSInterpreter(object):
return (Token.TRY, try_block, catch_block, finally_block)
def _expression(self, token_stream, stack_top):
if stack_top < 0:
raise ExtractorError('Recursion limit reached')
expr_list = []
has_another = True
while has_another:
@ -464,6 +508,9 @@ class JSInterpreter(object):
return (Token.ASSIGN, op, left, right)
def _member_expression(self, token_stream, stack_top):
if stack_top < 0:
raise ExtractorError('Recursion limit reached')
peek_id, peek_value, peek_pos = token_stream.peek()
if peek_id is Token.ID and peek_value == 'new':
token_stream.pop()
@ -555,8 +602,12 @@ class JSInterpreter(object):
raise ExtractorError('Syntax error at %d' % peek_pos)
def _function(self, token_stream, stack_top, is_expr=False):
if stack_top < 0:
raise ExtractorError('Recursion limit reached')
token_stream.pop()
token_id, token_value, token_pos = token_stream.peek()
name = None
if token_id is Token.ID:
token_stream.chk_id()
@ -568,9 +619,9 @@ class JSInterpreter(object):
if token_id is not Token.POPEN:
raise ExtractorError('Expected argument list at %d' % token_pos)
# args
token_stream.pop()
open_pos = token_pos
args = []
while True:
token_id, token_value, token_pos = token_stream.peek()
@ -594,7 +645,24 @@ class JSInterpreter(object):
if token_id is not Token.COPEN:
raise ExtractorError('Expected function body at %d' % token_pos)
return (Token.FUNC, name, args, self._statement(token_stream, stack_top - 1))
return (Token.FUNC, name, args, (self._function_body(token_stream, stack_top - 1)))
def _function_body(self, token_stream, stack_top):
if stack_top < 0:
raise ExtractorError('Recursion limit reached')
token_id, token_value, open_pos = token_stream.pop()
body = []
while True:
token_id, token_value, token_pos = token_stream.peek()
if token_id is Token.CCLOSE:
token_stream.pop()
break
elif token_id is Token.END and token_stream.ended:
raise ExtractorError('Unbalanced parentheses at %d' % open_pos)
body.append(self._source_element(token_stream, stack_top - 1))
return body
def _arguments(self, token_stream, stack_top):
if stack_top < 0:
@ -660,6 +728,9 @@ class JSInterpreter(object):
return (Token.ARRAY, elements)
def _object_literal(self, token_stream, stack_top):
if stack_top < 0:
raise ExtractorError('Recursion limit reached')
token_id, token_value, open_pos = token_stream.pop()
property_list = []
while True:
@ -688,9 +759,9 @@ class JSInterpreter(object):
raise ExtractorError('''Expected ')' at %d''' % token_pos)
if is_set:
desc = (Token.PROPSET, arg, self._statement(token_stream, stack_top - 1))
desc = (Token.PROPSET, arg, self._function_body(token_stream, stack_top - 1))
else:
desc = (Token.PROPGET, self._statement(token_stream, stack_top - 1))
desc = (Token.PROPGET, self._function_body(token_stream, stack_top - 1))
elif token_id in (Token.ID, Token.STR, Token.INT, Token.FLOAT):
property_name = token_value
@ -757,8 +828,7 @@ class JSInterpreter(object):
out = []
stack = []
has_another = True
while has_another:
while True:
had_inc = False
has_prefix = True
while has_prefix:
@ -828,13 +898,15 @@ class JSInterpreter(object):
name, op = peek_value
prec = {Token.OR: 5, Token.AND: 6}[name]
else:
has_another = False
op = None
prec = 4 # empties stack
while stack and stack[-1][0] >= prec:
_, stack_id, stack_op = stack.pop()
out.append((stack_id, stack_op))
if has_another:
if op is None:
break
else:
stack.append((prec, peek_id, op))
token_stream.pop()
@ -846,9 +918,15 @@ class JSInterpreter(object):
name = stmt[0]
ref = None
if name == 'funcdecl':
# TODO interpret funcdecl
raise ExtractorError('''Can't interpret statement called %s''' % name)
if name == Token.FUNC:
name, args, body = stmt[1:]
if name is not None:
if self._context_stack:
self.this[name] = Reference(self.build_function(args, body), (self.this, name))
else:
self.global_vars[name] = Reference(self.build_function(args, body), (self.this, name))
else:
raise ExtractorError('Function expression is not yet implemented')
elif name is Token.BLOCK:
block = stmt[1]
for stmt in block:
@ -857,8 +935,8 @@ class JSInterpreter(object):
ref = s.getvalue()
elif name is Token.VAR:
for name, value in stmt[1]:
self._context.local_vars[name] = Reference(self.interpret_expression(value).getvalue(),
(self._context.local_vars, name))
self.this[name] = Reference(self.interpret_expression(value).getvalue(),
(self.this, name))
elif name is Token.EXPR:
for expr in stmt[1]:
ref = self.interpret_expression(expr)
@ -866,11 +944,6 @@ class JSInterpreter(object):
# continue, break
elif name is Token.RETURN:
ref = self.interpret_statement(stmt[1])
ref = None if ref is None else ref.getvalue()
if isinstance(ref, list):
# TODO test nested arrays
ref = [elem.getvalue() for elem in ref]
self._context.ended = True
# with
# label
@ -892,7 +965,6 @@ class JSInterpreter(object):
if op is None:
ref = self.interpret_expression(left)
else:
# TODO handle undeclared variables (create propery)
try:
leftref = self.interpret_expression(left)
except KeyError:
@ -908,7 +980,7 @@ class JSInterpreter(object):
u = Reference(self.undefined, (self.this, key))
leftref = self.this[key] = u
else:
raise ExtractorError('''Invalid left-hand side in assignment''')
raise ExtractorError('Invalid left-hand side in assignment')
leftvalue = leftref.getvalue()
rightvalue = self.interpret_expression(right).getvalue()
leftref.putvalue(op(leftvalue, rightvalue))
@ -964,13 +1036,13 @@ class JSInterpreter(object):
index = self.interpret_expression(tail_value).getvalue()
target = target.getvalue()[index]
elif tail_name is Token.CALL:
# TODO interpret call
raise ExtractorError('''Can't interpret expression called %s''' % tail_name)
args = (self.interpret_expression(arg).getvalue() for arg in tail_value)
target = Reference(target.getvalue()(*args))
ref = target
elif name is Token.ID:
# XXX error handling (unknown id)
ref = (self._context.local_vars[expr[1]] if expr[1] in self._context.local_vars else
ref = (self.this[expr[1]] if expr[1] in self.this else
self.global_vars[expr[1]])
# literal
@ -989,18 +1061,6 @@ class JSInterpreter(object):
return ref
def run(self, cx=None):
if cx is not None:
self.push_context(cx)
res = None
for stmt in self.statements():
res = self.interpret_statement(stmt)
if self._context.ended:
if cx is not None:
self.pop_context()
break
return res
def extract_object(self, objname):
obj = {}
obj_m = re.search(
@ -1016,7 +1076,7 @@ class JSInterpreter(object):
fields)
for f in fields_m:
argnames = f.group('args').split(',')
obj[f.group('key')] = self.build_function(argnames, f.group('code'))
obj[f.group('key')] = self.build_function(argnames, self.parse(f.group('code')))
return obj
@ -1032,7 +1092,7 @@ class JSInterpreter(object):
raise ExtractorError('Could not find JS function %r' % funcname)
argnames = func_m.group('args').split(',')
return self.build_function(argnames, func_m.group('code'))
return self.build_function(argnames, self.parse(func_m.group('code')))
def push_context(self, cx):
self._context_stack.append(self._context)
@ -1043,16 +1103,33 @@ class JSInterpreter(object):
self._context = self._context_stack.pop()
def call_function(self, funcname, *args):
f = self.extract_function(funcname)
return f(args)
f = (self.this[funcname] if funcname in self.this else
self.global_vars[funcname] if funcname in self.global_vars else
self.extract_function(funcname))
return f(*args)
def build_function(self, argnames, code):
def resf(args):
def build_function(self, argnames, ast):
def resf(*args):
self.push_context(Context(dict(zip(argnames, args))))
for stmt in self.statements(code):
res = None
for stmt in ast:
res = self.interpret_statement(stmt)
res = None if res is None else res.getvalue(deep=True)
if self._context.ended:
self.pop_context()
break
return res
return resf
def run(self, cx=None):
if cx is not None:
self.push_context(cx)
res = None
for stmt in self.parse():
res = self.interpret_statement(stmt)
res = None if res is None else res.getvalue(deep=True)
if self._context.ended:
if cx is not None:
self.pop_context()
break
return res