Fixed PEP8 issues except E501

This commit is contained in:
Ruben Roy 2014-02-20 09:57:43 +05:30
parent ccb079ee67
commit 3b5ee5c51e
182 changed files with 1471 additions and 1245 deletions

View File

@ -9,6 +9,7 @@ import youtube_dl
BASH_COMPLETION_FILE = "youtube-dl.bash-completion" BASH_COMPLETION_FILE = "youtube-dl.bash-completion"
BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in" BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in"
def build_completion(opt_parser): def build_completion(opt_parser):
opts_flag = [] opts_flag = []
for group in opt_parser.option_groups: for group in opt_parser.option_groups:

View File

@ -233,7 +233,9 @@ def rmtree(path):
#============================================================================== #==============================================================================
class BuildError(Exception): class BuildError(Exception):
def __init__(self, output, code=500): def __init__(self, output, code=500):
self.output = output self.output = output
self.code = code self.code = code
@ -247,6 +249,7 @@ class HTTPError(BuildError):
class PythonBuilder(object): class PythonBuilder(object):
def __init__(self, **kwargs): def __init__(self, **kwargs):
pythonVersion = kwargs.pop('python', '2.7') pythonVersion = kwargs.pop('python', '2.7')
try: try:
@ -262,6 +265,7 @@ class PythonBuilder(object):
class GITInfoBuilder(object): class GITInfoBuilder(object):
def __init__(self, **kwargs): def __init__(self, **kwargs):
try: try:
self.user, self.repoName = kwargs['path'][:2] self.user, self.repoName = kwargs['path'][:2]
@ -281,6 +285,7 @@ class GITInfoBuilder(object):
class GITBuilder(GITInfoBuilder): class GITBuilder(GITInfoBuilder):
def build(self): def build(self):
try: try:
subprocess.check_output(['git', 'clone', 'git://github.com/%s/%s.git' % (self.user, self.repoName), self.buildPath]) subprocess.check_output(['git', 'clone', 'git://github.com/%s/%s.git' % (self.user, self.repoName), self.buildPath])
@ -313,6 +318,7 @@ class YoutubeDLBuilder(object):
class DownloadBuilder(object): class DownloadBuilder(object):
def __init__(self, **kwargs): def __init__(self, **kwargs):
self.handler = kwargs.pop('handler') self.handler = kwargs.pop('handler')
self.srcPath = os.path.join(self.buildPath, *tuple(kwargs['path'][2:])) self.srcPath = os.path.join(self.buildPath, *tuple(kwargs['path'][2:]))
@ -341,6 +347,7 @@ class DownloadBuilder(object):
class CleanupTempDir(object): class CleanupTempDir(object):
def build(self): def build(self):
try: try:
rmtree(self.basePath) rmtree(self.basePath)
@ -351,6 +358,7 @@ class CleanupTempDir(object):
class Null(object): class Null(object):
def __init__(self, **kwargs): def __init__(self, **kwargs):
pass pass

View File

@ -39,8 +39,7 @@ now_iso = now.isoformat() + 'Z'
atom_template = atom_template.replace('@TIMESTAMP@', now_iso) atom_template = atom_template.replace('@TIMESTAMP@', now_iso)
versions_info = json.load(open('update/versions.json')) versions_info = json.load(open('update/versions.json'))
versions = list(versions_info['versions'].keys()) versions = sorted(versions_info['versions'].keys())
versions.sort()
entries = [] entries = []
for v in versions: for v in versions:
@ -73,4 +72,3 @@ atom_template = atom_template.replace('@ENTRIES@', entries_str)
with io.open('update/releases.atom', 'w', encoding='utf-8') as atom_file: with io.open('update/releases.atom', 'w', encoding='utf-8') as atom_file:
atom_file.write(atom_template) atom_file.write(atom_template)

View File

@ -9,6 +9,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(
import youtube_dl import youtube_dl
def main(): def main():
with open('supportedsites.html.in', 'r', encoding='utf-8') as tmplf: with open('supportedsites.html.in', 'r', encoding='utf-8') as tmplf:
template = tmplf.read() template = tmplf.read()
@ -21,7 +22,7 @@ def main():
continue continue
elif ie_desc is not None: elif ie_desc is not None:
ie_html += ': {}'.format(ie.IE_DESC) ie_html += ': {}'.format(ie.IE_DESC)
if ie.working() == False: if not ie.working():
ie_html += ' (Currently broken)' ie_html += ' (Currently broken)'
ie_htmls.append('<li>{}</li>'.format(ie_html)) ie_htmls.append('<li>{}</li>'.format(ie_html))

View File

@ -1,6 +1,7 @@
#!/usr/bin/env python #!/usr/bin/env python
import sys, os import sys
import os
try: try:
import urllib.request as compat_urllib_request import urllib.request as compat_urllib_request

View File

@ -1,17 +1,23 @@
#!/usr/bin/env python #!/usr/bin/env python
import sys, os import sys
import os
import urllib2 import urllib2
import json, hashlib import json
import hashlib
def rsa_verify(message, signature, key): def rsa_verify(message, signature, key):
from struct import pack from struct import pack
from hashlib import sha256 from hashlib import sha256
from sys import version_info from sys import version_info
def b(x): def b(x):
if version_info[0] == 2: return x if version_info[0] == 2:
else: return x.encode('latin1') return x
assert(type(message) == type(b(''))) else:
return x.encode('latin1')
assert(isinstance(message, type(b(''))))
block_size = 0 block_size = 0
n = key[0] n = key[0]
while n: while n:
@ -23,13 +29,17 @@ def rsa_verify(message, signature, key):
raw_bytes.insert(0, pack("B", signature & 0xFF)) raw_bytes.insert(0, pack("B", signature & 0xFF))
signature >>= 8 signature >>= 8
signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes) signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
if signature[0:2] != b('\x00\x01'): return False if signature[0:2] != b('\x00\x01'):
return False
signature = signature[2:] signature = signature[2:]
if not b('\x00') in signature: return False if not b('\x00') in signature:
return False
signature = signature[signature.index(b('\x00')) + 1:] signature = signature[signature.index(b('\x00')) + 1:]
if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')):
return False
signature = signature[19:] signature = signature[19:]
if signature != sha256(message).digest(): return False if signature != sha256(message).digest():
return False
return True return True
sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n') sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')

View File

@ -47,6 +47,7 @@ def report_warning(message):
class FakeYDL(YoutubeDL): class FakeYDL(YoutubeDL):
def __init__(self, override=None): def __init__(self, override=None):
# Different instances of the downloader can't share the same dictionary # Different instances of the downloader can't share the same dictionary
# some test set the "sublang" parameter, which would break the md5 checks. # some test set the "sublang" parameter, which would break the md5 checks.
@ -66,11 +67,14 @@ class FakeYDL(YoutubeDL):
def expect_warning(self, regex): def expect_warning(self, regex):
# Silence an expected warning matching a regex # Silence an expected warning matching a regex
old_report_warning = self.report_warning old_report_warning = self.report_warning
def report_warning(self, message): def report_warning(self, message):
if re.match(regex, message): return if re.match(regex, message):
return
old_report_warning(message) old_report_warning(message)
self.report_warning = types.MethodType(report_warning, self) self.report_warning = types.MethodType(report_warning, self)
def get_testcases(): def get_testcases():
for ie in youtube_dl.extractor.gen_extractors(): for ie in youtube_dl.extractor.gen_extractors():
t = getattr(ie, '_TEST', None) t = getattr(ie, '_TEST', None)

View File

@ -14,6 +14,7 @@ from youtube_dl.extractor import YoutubeIE
class YDL(FakeYDL): class YDL(FakeYDL):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super(YDL, self).__init__(*args, **kwargs) super(YDL, self).__init__(*args, **kwargs)
self.downloaded_info_dicts = [] self.downloaded_info_dicts = []
@ -27,6 +28,7 @@ class YDL(FakeYDL):
class TestFormatSelection(unittest.TestCase): class TestFormatSelection(unittest.TestCase):
def test_prefer_free_formats(self): def test_prefer_free_formats(self):
# Same resolution => download webm # Same resolution => download webm
ydl = YDL() ydl = YDL()
@ -236,6 +238,7 @@ class TestFormatSelection(unittest.TestCase):
'ext': 'mp4', 'ext': 'mp4',
'width': None, 'width': None,
} }
def fname(templ): def fname(templ):
ydl = YoutubeDL({'outtmpl': templ}) ydl = YoutubeDL({'outtmpl': templ})
return ydl.prepare_filename(info) return ydl.prepare_filename(info)

View File

@ -32,6 +32,7 @@ def _download_restricted(url, filename, age):
class TestAgeRestriction(unittest.TestCase): class TestAgeRestriction(unittest.TestCase):
def _assert_restricted(self, url, filename, age, old_age=None): def _assert_restricted(self, url, filename, age, old_age=None):
self.assertTrue(_download_restricted(url, filename, old_age)) self.assertTrue(_download_restricted(url, filename, old_age))
self.assertFalse(_download_restricted(url, filename, age)) self.assertFalse(_download_restricted(url, filename, age))

View File

@ -21,6 +21,7 @@ from youtube_dl.extractor import (
class TestAllURLsMatching(unittest.TestCase): class TestAllURLsMatching(unittest.TestCase):
def setUp(self): def setUp(self):
self.ies = gen_extractors() self.ies = gen_extractors()

View File

@ -34,18 +34,23 @@ from youtube_dl.extractor import get_info_extractor
RETRIES = 3 RETRIES = 3
class YoutubeDL(youtube_dl.YoutubeDL): class YoutubeDL(youtube_dl.YoutubeDL):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.to_stderr = self.to_screen self.to_stderr = self.to_screen
self.processed_info_dicts = [] self.processed_info_dicts = []
super(YoutubeDL, self).__init__(*args, **kwargs) super(YoutubeDL, self).__init__(*args, **kwargs)
def report_warning(self, message): def report_warning(self, message):
# Don't accept warnings during tests # Don't accept warnings during tests
raise ExtractorError(message) raise ExtractorError(message)
def process_info(self, info_dict): def process_info(self, info_dict):
self.processed_info_dicts.append(info_dict) self.processed_info_dicts.append(info_dict)
return super(YoutubeDL, self).process_info(info_dict) return super(YoutubeDL, self).process_info(info_dict)
def _file_md5(fn): def _file_md5(fn):
with open(fn, 'rb') as f: with open(fn, 'rb') as f:
return hashlib.md5(f.read()).hexdigest() return hashlib.md5(f.read()).hexdigest()
@ -55,15 +60,19 @@ defs = get_testcases()
class TestDownload(unittest.TestCase): class TestDownload(unittest.TestCase):
maxDiff = None maxDiff = None
def setUp(self): def setUp(self):
self.defs = defs self.defs = defs
### Dynamically generate tests # Dynamically generate tests
def generator(test_case): def generator(test_case):
def test_template(self): def test_template(self):
ie = youtube_dl.extractor.get_info_extractor(test_case['name']) ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])] other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])]
def print_skipping(reason): def print_skipping(reason):
print('Skipping %s: %s' % (test_case['name'], reason)) print('Skipping %s: %s' % (test_case['name'], reason))
if not ie.working(): if not ie.working():
@ -88,6 +97,7 @@ def generator(test_case):
ydl = YoutubeDL(params) ydl = YoutubeDL(params)
ydl.add_default_info_extractors() ydl.add_default_info_extractors()
finished_hook_called = set() finished_hook_called = set()
def _hook(status): def _hook(status):
if status['status'] == 'finished': if status['status'] == 'finished':
finished_hook_called.add(status['filename']) finished_hook_called.add(status['filename'])
@ -97,6 +107,7 @@ def generator(test_case):
return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {})) return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
test_cases = test_case.get('playlist', [test_case]) test_cases = test_case.get('playlist', [test_case])
def try_rm_tcs_files(): def try_rm_tcs_files():
for tc in test_cases: for tc in test_cases:
tc_filename = get_tc_filename(tc) tc_filename = get_tc_filename(tc)
@ -162,7 +173,7 @@ def generator(test_case):
return test_template return test_template
### And add them to TestDownload # And add them to TestDownload
for n, test_case in enumerate(defs): for n, test_case in enumerate(defs):
test_method = generator(test_case) test_method = generator(test_case)
tname = 'test_' + str(test_case['name']) tname = 'test_' + str(test_case['name'])

View File

@ -11,7 +11,9 @@ try:
except AttributeError: except AttributeError:
_DEV_NULL = open(os.devnull, 'wb') _DEV_NULL = open(os.devnull, 'wb')
class TestExecution(unittest.TestCase): class TestExecution(unittest.TestCase):
def test_import(self): def test_import(self):
subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir) subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)

View File

@ -40,6 +40,7 @@ from youtube_dl.extractor import (
class TestPlaylists(unittest.TestCase): class TestPlaylists(unittest.TestCase):
def assertIsPlaylist(self, info): def assertIsPlaylist(self, info):
"""Make sure the info has '_type' set to 'playlist'""" """Make sure the info has '_type' set to 'playlist'"""
self.assertEqual(info['_type'], 'playlist') self.assertEqual(info['_type'], 'playlist')

View File

@ -21,6 +21,7 @@ from youtube_dl.extractor import (
class BaseTestSubtitles(unittest.TestCase): class BaseTestSubtitles(unittest.TestCase):
url = None url = None
IE = None IE = None
def setUp(self): def setUp(self):
self.DL = FakeYDL() self.DL = FakeYDL()
self.ie = self.IE(self.DL) self.ie = self.IE(self.DL)

View File

@ -13,6 +13,7 @@ IGNORED_FILES = [
class TestUnicodeLiterals(unittest.TestCase): class TestUnicodeLiterals(unittest.TestCase):
def test_all_files(self): def test_all_files(self):
print('Skipping this test (not yet fully implemented)') print('Skipping this test (not yet fully implemented)')
return return

View File

@ -41,6 +41,7 @@ else:
class TestUtil(unittest.TestCase): class TestUtil(unittest.TestCase):
def test_timeconvert(self): def test_timeconvert(self):
self.assertTrue(timeconvert('') is None) self.assertTrue(timeconvert('') is None)
self.assertTrue(timeconvert('bougrg') is None) self.assertTrue(timeconvert('bougrg') is None)

View File

@ -19,6 +19,7 @@ import youtube_dl.extractor
class YoutubeDL(youtube_dl.YoutubeDL): class YoutubeDL(youtube_dl.YoutubeDL):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super(YoutubeDL, self).__init__(*args, **kwargs) super(YoutubeDL, self).__init__(*args, **kwargs)
self.to_stderr = self.to_screen self.to_stderr = self.to_screen
@ -31,17 +32,17 @@ params = get_params({
}) })
TEST_ID = 'gr51aVj-mLg' TEST_ID = 'gr51aVj-mLg'
ANNOTATIONS_FILE = TEST_ID + '.flv.annotations.xml' ANNOTATIONS_FILE = TEST_ID + '.flv.annotations.xml'
EXPECTED_ANNOTATIONS = ['Speech bubble', 'Note', 'Title', 'Spotlight', 'Label'] EXPECTED_ANNOTATIONS = ['Speech bubble', 'Note', 'Title', 'Spotlight', 'Label']
class TestAnnotations(unittest.TestCase): class TestAnnotations(unittest.TestCase):
def setUp(self): def setUp(self):
# Clear old files # Clear old files
self.tearDown() self.tearDown()
def test_info_json(self): def test_info_json(self):
expected = list(EXPECTED_ANNOTATIONS) # Two annotations could have the same text. expected = list(EXPECTED_ANNOTATIONS) # Two annotations could have the same text.
ie = youtube_dl.extractor.YoutubeIE() ie = youtube_dl.extractor.YoutubeIE()
@ -71,7 +72,6 @@ class TestAnnotations(unittest.TestCase):
# We should have seen (and removed) all the expected annotation texts. # We should have seen (and removed) all the expected annotation texts.
self.assertEqual(len(expected), 0, 'Not all expected annotations were found.') self.assertEqual(len(expected), 0, 'Not all expected annotations were found.')
def tearDown(self): def tearDown(self):
try_rm(ANNOTATIONS_FILE) try_rm(ANNOTATIONS_FILE)

View File

@ -18,6 +18,7 @@ import youtube_dl.extractor
class YoutubeDL(youtube_dl.YoutubeDL): class YoutubeDL(youtube_dl.YoutubeDL):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super(YoutubeDL, self).__init__(*args, **kwargs) super(YoutubeDL, self).__init__(*args, **kwargs)
self.to_stderr = self.to_screen self.to_stderr = self.to_screen
@ -41,6 +42,7 @@ For more information, contact phihag@phihag.de .'''
class TestInfoJSON(unittest.TestCase): class TestInfoJSON(unittest.TestCase):
def setUp(self): def setUp(self):
# Clear old files # Clear old files
self.tearDown() self.tearDown()

View File

@ -20,6 +20,7 @@ from youtube_dl.extractor import (
class TestYoutubeLists(unittest.TestCase): class TestYoutubeLists(unittest.TestCase):
def assertIsPlaylist(self, info): def assertIsPlaylist(self, info):
"""Make sure the info has '_type' set to 'playlist'""" """Make sure the info has '_type' set to 'playlist'"""
self.assertEqual(info['_type'], 'playlist') self.assertEqual(info['_type'], 'playlist')

View File

@ -37,6 +37,7 @@ _TESTS = [
class TestSignature(unittest.TestCase): class TestSignature(unittest.TestCase):
def setUp(self): def setUp(self):
TEST_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DIR = os.path.dirname(os.path.abspath(__file__))
self.TESTDATA_DIR = os.path.join(TEST_DIR, 'testdata') self.TESTDATA_DIR = os.path.join(TEST_DIR, 'testdata')

View File

@ -1,21 +1,27 @@
#!/usr/bin/env python #!/usr/bin/env python
import sys, os import sys
import json, hashlib import os
import json
import hashlib
try: try:
import urllib.request as compat_urllib_request import urllib.request as compat_urllib_request
except ImportError: # Python 2 except ImportError: # Python 2
import urllib2 as compat_urllib_request import urllib2 as compat_urllib_request
def rsa_verify(message, signature, key): def rsa_verify(message, signature, key):
from struct import pack from struct import pack
from hashlib import sha256 from hashlib import sha256
from sys import version_info from sys import version_info
def b(x): def b(x):
if version_info[0] == 2: return x if version_info[0] == 2:
else: return x.encode('latin1') return x
assert(type(message) == type(b(''))) else:
return x.encode('latin1')
assert(isinstance(message, type(b(''))))
block_size = 0 block_size = 0
n = key[0] n = key[0]
while n: while n:
@ -27,13 +33,17 @@ def rsa_verify(message, signature, key):
raw_bytes.insert(0, pack("B", signature & 0xFF)) raw_bytes.insert(0, pack("B", signature & 0xFF))
signature >>= 8 signature >>= 8
signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes) signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
if signature[0:2] != b('\x00\x01'): return False if signature[0:2] != b('\x00\x01'):
return False
signature = signature[2:] signature = signature[2:]
if not b('\x00') in signature: return False if not b('\x00') in signature:
return False
signature = signature[signature.index(b('\x00')) + 1:] signature = signature[signature.index(b('\x00')) + 1:]
if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')):
return False
signature = signature[19:] signature = signature[19:]
if signature != sha256(message).digest(): return False if signature != sha256(message).digest():
return False
return True return True
sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n') sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')

View File

@ -5,6 +5,7 @@ from .downloader import get_suitable_downloader
# This class reproduces the old behaviour of FileDownloader # This class reproduces the old behaviour of FileDownloader
class FileDownloader(RealFileDownloader): class FileDownloader(RealFileDownloader):
def _do_download(self, filename, info_dict): def _do_download(self, filename, info_dict):
real_fd = get_suitable_downloader(info_dict)(self.ydl, self.params) real_fd = get_suitable_downloader(info_dict)(self.ydl, self.params)
for ph in self._progress_hooks: for ph in self._progress_hooks:

View File

@ -61,6 +61,7 @@ from .version import __version__
class YoutubeDL(object): class YoutubeDL(object):
"""YoutubeDL class. """YoutubeDL class.
YoutubeDL objects are the ones responsible of downloading the YoutubeDL objects are the ones responsible of downloading the
@ -268,7 +269,7 @@ class YoutubeDL(object):
return message return message
assert hasattr(self, '_output_process') assert hasattr(self, '_output_process')
assert type(message) == type('') assert isinstance(message, type(''))
line_count = message.count('\n') + 1 line_count = message.count('\n') + 1
self._output_process.stdin.write((message + '\n').encode('utf-8')) self._output_process.stdin.write((message + '\n').encode('utf-8'))
self._output_process.stdin.flush() self._output_process.stdin.flush()
@ -293,7 +294,7 @@ class YoutubeDL(object):
def to_stderr(self, message): def to_stderr(self, message):
"""Print message to stderr.""" """Print message to stderr."""
assert type(message) == type('') assert isinstance(message, type(''))
if self.params.get('logger'): if self.params.get('logger'):
self.params['logger'].error(message) self.params['logger'].error(message)
else: else:

View File

@ -154,7 +154,8 @@ def parseOpts(overrideArguments=None):
if len(opts) > 1: if len(opts) > 1:
opts.insert(1, ', ') opts.insert(1, ', ')
if option.takes_value(): opts.append(' %s' % option.metavar) if option.takes_value():
opts.append(' %s' % option.metavar)
return "".join(opts) return "".join(opts)
@ -176,7 +177,8 @@ def parseOpts(overrideArguments=None):
# No need to wrap help messages if we're on a wide console # No need to wrap help messages if we're on a wide console
columns = get_term_width() columns = get_term_width()
if columns: max_width = columns if columns:
max_width = columns
fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position) fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
fmt.format_option_strings = _format_option_string fmt.format_option_strings = _format_option_string
@ -250,7 +252,6 @@ def parseOpts(overrideArguments=None):
action='store_true', action='store_true',
help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)') help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
selection.add_option( selection.add_option(
'--playlist-start', '--playlist-start',
dest='playliststart', metavar='NUMBER', default=1, type=int, dest='playliststart', metavar='NUMBER', default=1, type=int,
@ -306,7 +307,6 @@ def parseOpts(overrideArguments=None):
authentication.add_option('--video-password', authentication.add_option('--video-password',
dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)') dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
video_format.add_option('-f', '--format', video_format.add_option('-f', '--format',
action='store', dest='format', metavar='FORMAT', default=None, action='store', dest='format', metavar='FORMAT', default=None,
help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestaudio", "worst", and "worstaudio". By default, youtube-dl will pick the best quality.') help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestaudio", "worst", and "worstaudio". By default, youtube-dl will pick the best quality.')
@ -402,7 +402,6 @@ def parseOpts(overrideArguments=None):
dest='debug_printtraffic', action='store_true', default=False, dest='debug_printtraffic', action='store_true', default=False,
help='Display sent and read HTTP traffic') help='Display sent and read HTTP traffic')
filesystem.add_option('-t', '--title', filesystem.add_option('-t', '--title',
action='store_true', dest='usetitle', help='use title in file name (default)', default=False) action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
filesystem.add_option('--id', filesystem.add_option('--id',
@ -464,7 +463,6 @@ def parseOpts(overrideArguments=None):
action='store_true', dest='writethumbnail', action='store_true', dest='writethumbnail',
help='write thumbnail image to disk', default=False) help='write thumbnail image to disk', default=False)
postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False, postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)') help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best', postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
@ -488,7 +486,6 @@ def parseOpts(overrideArguments=None):
postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg', postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
help='Prefer ffmpeg over avconv for running the postprocessors') help='Prefer ffmpeg over avconv for running the postprocessors')
parser.add_option_group(general) parser.add_option_group(general)
parser.add_option_group(selection) parser.add_option_group(selection)
parser.add_option_group(downloader) parser.add_option_group(downloader)
@ -593,7 +590,6 @@ def _real_main(argv=None):
compat_print(desc) compat_print(desc)
sys.exit(0) sys.exit(0)
# Conflicting, missing and erroneous options # Conflicting, missing and erroneous options
if opts.usenetrc and (opts.username is not None or opts.password is not None): if opts.usenetrc and (opts.username is not None or opts.password is not None):
parser.error(u'using .netrc conflicts with giving username/password') parser.error(u'using .netrc conflicts with giving username/password')
@ -657,7 +653,7 @@ def _real_main(argv=None):
# --all-sub automatically sets --write-sub if --write-auto-sub is not given # --all-sub automatically sets --write-sub if --write-auto-sub is not given
# this was the old behaviour if only --all-sub was given. # this was the old behaviour if only --all-sub was given.
if opts.allsubtitles and (opts.writeautomaticsub == False): if opts.allsubtitles and not opts.writeautomaticsub:
opts.writesubtitles = True opts.writesubtitles = True
if sys.version_info < (3,): if sys.version_info < (3,):

View File

@ -7,6 +7,7 @@ from .utils import bytes_to_intlist, intlist_to_bytes
BLOCK_SIZE_BYTES = 16 BLOCK_SIZE_BYTES = 16
def aes_ctr_decrypt(data, key, counter): def aes_ctr_decrypt(data, key, counter):
""" """
Decrypt with aes in counter mode Decrypt with aes in counter mode
@ -32,6 +33,7 @@ def aes_ctr_decrypt(data, key, counter):
return decrypted_data return decrypted_data
def aes_cbc_decrypt(data, key, iv): def aes_cbc_decrypt(data, key, iv):
""" """
Decrypt with aes in CBC mode Decrypt with aes in CBC mode
@ -57,6 +59,7 @@ def aes_cbc_decrypt(data, key, iv):
return decrypted_data return decrypted_data
def key_expansion(data): def key_expansion(data):
""" """
Generate key schedule Generate key schedule
@ -91,6 +94,7 @@ def key_expansion(data):
return data return data
def aes_encrypt(data, expanded_key): def aes_encrypt(data, expanded_key):
""" """
Encrypt one block with aes Encrypt one block with aes
@ -111,6 +115,7 @@ def aes_encrypt(data, expanded_key):
return data return data
def aes_decrypt(data, expanded_key): def aes_decrypt(data, expanded_key):
""" """
Decrypt one block with aes Decrypt one block with aes
@ -131,6 +136,7 @@ def aes_decrypt(data, expanded_key):
return data return data
def aes_decrypt_text(data, password, key_size_bytes): def aes_decrypt_text(data, password, key_size_bytes):
""" """
Decrypt text Decrypt text
@ -157,6 +163,7 @@ def aes_decrypt_text(data, password, key_size_bytes):
class Counter: class Counter:
__value = nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES) __value = nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES)
def next_value(self): def next_value(self):
temp = self.__value temp = self.__value
self.__value = inc(self.__value) self.__value = inc(self.__value)
@ -241,15 +248,19 @@ RIJNDAEL_LOG_TABLE = (0x00, 0x00, 0x19, 0x01, 0x32, 0x02, 0x1a, 0xc6, 0x4b, 0xc7
0x44, 0x11, 0x92, 0xd9, 0x23, 0x20, 0x2e, 0x89, 0xb4, 0x7c, 0xb8, 0x26, 0x77, 0x99, 0xe3, 0xa5, 0x44, 0x11, 0x92, 0xd9, 0x23, 0x20, 0x2e, 0x89, 0xb4, 0x7c, 0xb8, 0x26, 0x77, 0x99, 0xe3, 0xa5,
0x67, 0x4a, 0xed, 0xde, 0xc5, 0x31, 0xfe, 0x18, 0x0d, 0x63, 0x8c, 0x80, 0xc0, 0xf7, 0x70, 0x07) 0x67, 0x4a, 0xed, 0xde, 0xc5, 0x31, 0xfe, 0x18, 0x0d, 0x63, 0x8c, 0x80, 0xc0, 0xf7, 0x70, 0x07)
def sub_bytes(data): def sub_bytes(data):
return [SBOX[x] for x in data] return [SBOX[x] for x in data]
def sub_bytes_inv(data): def sub_bytes_inv(data):
return [SBOX_INV[x] for x in data] return [SBOX_INV[x] for x in data]
def rotate(data): def rotate(data):
return data[1:] + [data[0]] return data[1:] + [data[0]]
def key_schedule_core(data, rcon_iteration): def key_schedule_core(data, rcon_iteration):
data = rotate(data) data = rotate(data)
data = sub_bytes(data) data = sub_bytes(data)
@ -257,14 +268,17 @@ def key_schedule_core(data, rcon_iteration):
return data return data
def xor(data1, data2): def xor(data1, data2):
return [x ^ y for x, y in zip(data1, data2)] return [x ^ y for x, y in zip(data1, data2)]
def rijndael_mul(a, b): def rijndael_mul(a, b):
if(a == 0 or b == 0): if(a == 0 or b == 0):
return 0 return 0
return RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[a] + RIJNDAEL_LOG_TABLE[b]) % 0xFF] return RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[a] + RIJNDAEL_LOG_TABLE[b]) % 0xFF]
def mix_column(data, matrix): def mix_column(data, matrix):
data_mixed = [] data_mixed = []
for row in range(4): for row in range(4):
@ -275,6 +289,7 @@ def mix_column(data, matrix):
data_mixed.append(mixed) data_mixed.append(mixed)
return data_mixed return data_mixed
def mix_columns(data, matrix=MIX_COLUMN_MATRIX): def mix_columns(data, matrix=MIX_COLUMN_MATRIX):
data_mixed = [] data_mixed = []
for i in range(4): for i in range(4):
@ -282,9 +297,11 @@ def mix_columns(data, matrix=MIX_COLUMN_MATRIX):
data_mixed += mix_column(column, matrix) data_mixed += mix_column(column, matrix)
return data_mixed return data_mixed
def mix_columns_inv(data): def mix_columns_inv(data):
return mix_columns(data, MIX_COLUMN_MATRIX_INV) return mix_columns(data, MIX_COLUMN_MATRIX_INV)
def shift_rows(data): def shift_rows(data):
data_shifted = [] data_shifted = []
for column in range(4): for column in range(4):
@ -292,6 +309,7 @@ def shift_rows(data):
data_shifted.append(data[((column + row) & 0b11) * 4 + row]) data_shifted.append(data[((column + row) & 0b11) * 4 + row])
return data_shifted return data_shifted
def shift_rows_inv(data): def shift_rows_inv(data):
data_shifted = [] data_shifted = []
for column in range(4): for column in range(4):
@ -299,6 +317,7 @@ def shift_rows_inv(data):
data_shifted.append(data[((column - row) & 0b11) * 4 + row]) data_shifted.append(data[((column - row) & 0b11) * 4 + row])
return data_shifted return data_shifted
def inc(data): def inc(data):
data = data[:] # copy data = data[:] # copy
for i in range(len(data) - 1, -1, -1): for i in range(len(data) - 1, -1, -1):

View File

@ -11,6 +11,7 @@ from ..utils import (
class FileDownloader(object): class FileDownloader(object):
"""File Downloader class. """File Downloader class.
File downloader objects are the ones responsible of downloading the File downloader objects are the ones responsible of downloading the

View File

@ -21,6 +21,7 @@ from ..utils import (
class FlvReader(io.BytesIO): class FlvReader(io.BytesIO):
""" """
Reader for Flv files Reader for Flv files
The file format is documented in https://www.adobe.com/devnet/f4v.html The file format is documented in https://www.adobe.com/devnet/f4v.html
@ -210,11 +211,13 @@ def _add_ns(prop):
class HttpQuietDownloader(HttpFD): class HttpQuietDownloader(HttpFD):
def to_screen(self, *args, **kargs): def to_screen(self, *args, **kargs):
pass pass
class F4mFD(FileDownloader): class F4mFD(FileDownloader):
""" """
A downloader for f4m manifests or AdobeHDS. A downloader for f4m manifests or AdobeHDS.
""" """

View File

@ -8,6 +8,7 @@ from ..utils import (
class HlsFD(FileDownloader): class HlsFD(FileDownloader):
def real_download(self, filename, info_dict): def real_download(self, filename, info_dict):
url = info_dict['url'] url = info_dict['url']
self.report_destination(filename) self.report_destination(filename)

View File

@ -14,6 +14,7 @@ from ..utils import (
class HttpFD(FileDownloader): class HttpFD(FileDownloader):
def real_download(self, filename, info_dict): def real_download(self, filename, info_dict):
url = info_dict['url'] url = info_dict['url']
tmpfilename = self.temp_name(filename) tmpfilename = self.temp_name(filename)

View File

@ -8,6 +8,7 @@ from ..utils import (
class MplayerFD(FileDownloader): class MplayerFD(FileDownloader):
def real_download(self, filename, info_dict): def real_download(self, filename, info_dict):
url = info_dict['url'] url = info_dict['url']
self.report_destination(filename) self.report_destination(filename)

View File

@ -12,6 +12,7 @@ from ..utils import (
class RtmpFD(FileDownloader): class RtmpFD(FileDownloader):
def real_download(self, filename, info_dict): def real_download(self, filename, info_dict):
def run_rtmpdump(args): def run_rtmpdump(args):
start = time.time() start = time.time()

View File

@ -66,11 +66,13 @@ class AppleTrailersIE(InfoExtractor):
uploader_id = mobj.group('company') uploader_id = mobj.group('company')
playlist_url = compat_urlparse.urljoin(url, u'includes/playlists/itunes.inc') playlist_url = compat_urlparse.urljoin(url, u'includes/playlists/itunes.inc')
def fix_html(s): def fix_html(s):
s = re.sub(r'(?s)<script[^<]*?>.*?</script>', u'', s) s = re.sub(r'(?s)<script[^<]*?>.*?</script>', u'', s)
s = re.sub(r'<img ([^<]*?)>', r'<img \1/>', s) s = re.sub(r'<img ([^<]*?)>', r'<img \1/>', s)
# The ' in the onClick attributes are not escaped, it couldn't be parsed # The ' in the onClick attributes are not escaped, it couldn't be parsed
# like: http://trailers.apple.com/trailers/wb/gravity/ # like: http://trailers.apple.com/trailers/wb/gravity/
def _clean_json(m): def _clean_json(m):
return u'iTunes.playURL(%s);' % m.group(1).replace('\'', '&#39;') return u'iTunes.playURL(%s);' % m.group(1).replace('\'', '&#39;')
s = re.sub(self._JSON_RE, _clean_json, s) s = re.sub(self._JSON_RE, _clean_json, s)

View File

@ -19,6 +19,7 @@ from ..utils import (
# is different for each one. The videos usually expire in 7 days, so we can't # is different for each one. The videos usually expire in 7 days, so we can't
# add tests. # add tests.
class ArteTvIE(InfoExtractor): class ArteTvIE(InfoExtractor):
_VIDEOS_URL = r'(?:http://)?videos\.arte\.tv/(?P<lang>fr|de)/.*-(?P<id>.*?)\.html' _VIDEOS_URL = r'(?:http://)?videos\.arte\.tv/(?P<lang>fr|de)/.*-(?P<id>.*?)\.html'
_LIVEWEB_URL = r'(?:http://)?liveweb\.arte\.tv/(?P<lang>fr|de)/(?P<subpage>.+?)/(?P<name>.+)' _LIVEWEB_URL = r'(?:http://)?liveweb\.arte\.tv/(?P<lang>fr|de)/(?P<subpage>.+?)/(?P<name>.+)'
@ -86,6 +87,7 @@ class ArteTvIE(InfoExtractor):
config_xml = self._download_webpage(config_xml_url, video_id, note=u'Downloading configuration') config_xml = self._download_webpage(config_xml_url, video_id, note=u'Downloading configuration')
video_urls = list(re.finditer(r'<url quality="(?P<quality>.*?)">(?P<url>.*?)</url>', config_xml)) video_urls = list(re.finditer(r'<url quality="(?P<quality>.*?)">(?P<url>.*?)</url>', config_xml))
def _key(m): def _key(m):
quality = m.group('quality') quality = m.group('quality')
if quality == 'hd': if quality == 'hd':
@ -164,6 +166,7 @@ class ArteTVPlus7IE(InfoExtractor):
all_formats = player_info['VSR'].values() all_formats = player_info['VSR'].values()
# Some formats use the m3u8 protocol # Some formats use the m3u8 protocol
all_formats = list(filter(lambda f: f.get('videoFormat') != 'M3U8', all_formats)) all_formats = list(filter(lambda f: f.get('videoFormat') != 'M3U8', all_formats))
def _match_lang(f): def _match_lang(f):
if f.get('versionCode') is None: if f.get('versionCode') is None:
return True return True
@ -200,6 +203,7 @@ class ArteTVPlus7IE(InfoExtractor):
re.match(r'VO?(F|A)-STM\1', f.get('versionCode', '')) is None, re.match(r'VO?(F|A)-STM\1', f.get('versionCode', '')) is None,
) )
formats = sorted(formats, key=sort_key) formats = sorted(formats, key=sort_key)
def _format(format_info): def _format(format_info):
quality = '' quality = ''
height = format_info.get('height') height = format_info.get('height')

View File

@ -14,6 +14,7 @@ from ..utils import (
class BlipTVIE(SubtitlesInfoExtractor): class BlipTVIE(SubtitlesInfoExtractor):
"""Information extractor for blip.tv""" """Information extractor for blip.tv"""
_VALID_URL = r'https?://(?:\w+\.)?blip\.tv/((.+/)|(play/)|(api\.swf#))(?P<presumptive_id>.+)$' _VALID_URL = r'https?://(?:\w+\.)?blip\.tv/((.+/)|(play/)|(api\.swf#))(?P<presumptive_id>.+)$'

View File

@ -5,7 +5,9 @@ import re
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import ExtractorError from ..utils import ExtractorError
class Channel9IE(InfoExtractor): class Channel9IE(InfoExtractor):
''' '''
Common extractor for channel9.msdn.com. Common extractor for channel9.msdn.com.

View File

@ -36,6 +36,7 @@ class ClipsyndicateIE(InfoExtractor):
transform_source=fix_xml_ampersands) transform_source=fix_xml_ampersands)
track_doc = pdoc.find('trackList/track') track_doc = pdoc.find('trackList/track')
def find_param(name): def find_param(name):
node = find_xpath_attr(track_doc, './/param', 'name', name) node = find_xpath_attr(track_doc, './/param', 'name', name)
if node is not None: if node is not None:

View File

@ -1,5 +1,6 @@
from .mtv import MTVIE from .mtv import MTVIE
class CMTIE(MTVIE): class CMTIE(MTVIE):
IE_NAME = u'cmt.com' IE_NAME = u'cmt.com'
_VALID_URL = r'https?://www\.cmt\.com/videos/.+?/(?P<videoid>[^/]+)\.jhtml' _VALID_URL = r'https?://www\.cmt\.com/videos/.+?/(?P<videoid>[^/]+)\.jhtml'

View File

@ -25,6 +25,7 @@ _NO_DEFAULT = object()
class InfoExtractor(object): class InfoExtractor(object):
"""Information Extractor class. """Information Extractor class.
Information extractors are the classes that, given a URL, extract Information extractors are the classes that, given a URL, extract
@ -317,6 +318,7 @@ class InfoExtractor(object):
if video_id is not None: if video_id is not None:
video_info['id'] = video_id video_info['id'] = video_id
return video_info return video_info
@staticmethod @staticmethod
def playlist_result(entries, playlist_id=None, playlist_title=None): def playlist_result(entries, playlist_id=None, playlist_title=None):
"""Returns a playlist""" """Returns a playlist"""
@ -340,7 +342,8 @@ class InfoExtractor(object):
else: else:
for p in pattern: for p in pattern:
mobj = re.search(p, string, flags) mobj = re.search(p, string, flags)
if mobj: break if mobj:
break
if os.name != 'nt' and sys.stderr.isatty(): if os.name != 'nt' and sys.stderr.isatty():
_name = u'\033[0;34m%s\033[0m' % name _name = u'\033[0;34m%s\033[0m' % name
@ -429,7 +432,8 @@ class InfoExtractor(object):
def _og_search_video_url(self, html, name='video url', secure=True, **kargs): def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
regexes = self._og_regexes('video') regexes = self._og_regexes('video')
if secure: regexes = self._og_regexes('video:secure_url') + regexes if secure:
regexes = self._og_regexes('video:secure_url') + regexes
return self._html_search_regex(regexes, html, name, **kargs) return self._html_search_regex(regexes, html, name, **kargs)
def _html_search_meta(self, name, html, display_name=None): def _html_search_meta(self, name, html, display_name=None):
@ -530,6 +534,7 @@ class InfoExtractor(object):
class SearchInfoExtractor(InfoExtractor): class SearchInfoExtractor(InfoExtractor):
""" """
Base class for paged search queries extractors. Base class for paged search queries extractors.
They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query} They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query}

View File

@ -14,6 +14,7 @@ from ..utils import (
class CondeNastIE(InfoExtractor): class CondeNastIE(InfoExtractor):
""" """
Condé Nast is a media group, some of its sites use a custom HTML5 player Condé Nast is a media group, some of its sites use a custom HTML5 player
that works the same in all of them. that works the same in all of them.

View File

@ -5,6 +5,7 @@ import re
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import determine_ext from ..utils import determine_ext
class CriterionIE(InfoExtractor): class CriterionIE(InfoExtractor):
_VALID_URL = r'https?://www\.criterion\.com/films/(\d*)-.+' _VALID_URL = r'https?://www\.criterion\.com/films/(\d*)-.+'
_TEST = { _TEST = {

View File

@ -1,7 +1,9 @@
# encoding: utf-8 # encoding: utf-8
from __future__ import unicode_literals from __future__ import unicode_literals
import re, base64, zlib import re
import base64
import zlib
from hashlib import sha1 from hashlib import sha1
from math import pow, sqrt, floor from math import pow, sqrt, floor
from .common import InfoExtractor from .common import InfoExtractor
@ -19,6 +21,7 @@ from ..aes import (
inc, inc,
) )
class CrunchyrollIE(InfoExtractor): class CrunchyrollIE(InfoExtractor):
_VALID_URL = r'(?:https?://)?(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)' _VALID_URL = r'(?:https?://)?(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
_TESTS = [{ _TESTS = [{
@ -70,8 +73,10 @@ class CrunchyrollIE(InfoExtractor):
return shaHash + [0] * 12 return shaHash + [0] * 12
key = obfuscate_key(id) key = obfuscate_key(id)
class Counter: class Counter:
__value = iv __value = iv
def next_value(self): def next_value(self):
temp = self.__value temp = self.__value
self.__value = inc(self.__value) self.__value = inc(self.__value)
@ -149,7 +154,7 @@ class CrunchyrollIE(InfoExtractor):
subtitles = {} subtitles = {}
for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage): for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
sub_page = self._download_webpage('http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id='+sub_id,\ sub_page = self._download_webpage('http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,
video_id, note='Downloading subtitles for ' + sub_name) video_id, note='Downloading subtitles for ' + sub_name)
id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False) id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False)
iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False) iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False)

View File

@ -16,7 +16,9 @@ from ..utils import (
ExtractorError, ExtractorError,
) )
class DailymotionBaseInfoExtractor(InfoExtractor): class DailymotionBaseInfoExtractor(InfoExtractor):
@staticmethod @staticmethod
def _build_request(url): def _build_request(url):
"""Build a request with the family filter disabled""" """Build a request with the family filter disabled"""
@ -25,7 +27,9 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
request.add_header('Cookie', 'ff=off') request.add_header('Cookie', 'ff=off')
return request return request
class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor): class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
"""Information Extractor for Dailymotion""" """Information Extractor for Dailymotion"""
_VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(embed|#)/)?video/(?P<id>[^/?_]+)' _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(embed|#)/)?video/(?P<id>[^/?_]+)'

View File

@ -15,6 +15,7 @@ from ..utils import (
class DepositFilesIE(InfoExtractor): class DepositFilesIE(InfoExtractor):
"""Information extractor for depositfiles.com""" """Information extractor for depositfiles.com"""
_VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles\.com/(?:../(?#locale))?files/(.+)' _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles\.com/(?:../(?#locale))?files/(.+)'

View File

@ -23,7 +23,6 @@ class DreiSatIE(InfoExtractor):
} }
} }
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id') video_id = mobj.group('id')

View File

@ -43,4 +43,3 @@ class EHowIE(InfoExtractor):
'description': self._og_search_description(webpage), 'description': self._og_search_description(webpage),
'uploader': uploader, 'uploader': uploader,
} }

View File

@ -82,7 +82,6 @@ class EightTracksIE(InfoExtractor):
] ]
} }
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
if mobj is None: if mobj is None:

View File

@ -8,6 +8,7 @@ from ..utils import (
compat_urllib_parse, compat_urllib_parse,
) )
class ExtremeTubeIE(InfoExtractor): class ExtremeTubeIE(InfoExtractor):
_VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>extremetube\.com/video/.+?(?P<videoid>[0-9]+))(?:[/?&]|$)' _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>extremetube\.com/video/.+?(?P<videoid>[0-9]+))(?:[/?&]|$)'
_TEST = { _TEST = {

View File

@ -15,6 +15,7 @@ from ..utils import (
class FacebookIE(InfoExtractor): class FacebookIE(InfoExtractor):
"""Information Extractor for Facebook""" """Information Extractor for Facebook"""
_VALID_URL = r'''(?x) _VALID_URL = r'''(?x)

View File

@ -10,6 +10,7 @@ from ..utils import (
class FlickrIE(InfoExtractor): class FlickrIE(InfoExtractor):
"""Information Extractor for Flickr videos""" """Information Extractor for Flickr videos"""
_VALID_URL = r'(?:https?://)?(?:www\.|secure\.)?flickr\.com/photos/(?P<uploader_id>[\w\-_@]+)/(?P<id>\d+).*' _VALID_URL = r'(?:https?://)?(?:www\.|secure\.)?flickr\.com/photos/(?P<uploader_id>[\w\-_@]+)/(?P<id>\d+).*'
_TEST = { _TEST = {

View File

@ -12,6 +12,7 @@ from ..utils import (
class FranceTVBaseInfoExtractor(InfoExtractor): class FranceTVBaseInfoExtractor(InfoExtractor):
def _extract_video(self, video_id): def _extract_video(self, video_id):
info = self._download_xml( info = self._download_xml(
'http://www.francetvinfo.fr/appftv/webservices/video/' 'http://www.francetvinfo.fr/appftv/webservices/video/'

View File

@ -116,10 +116,12 @@ class GenericIE(InfoExtractor):
"""Check if it is a redirect, like url shorteners, in case return the new url.""" """Check if it is a redirect, like url shorteners, in case return the new url."""
class HEADRedirectHandler(compat_urllib_request.HTTPRedirectHandler): class HEADRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
""" """
Subclass the HTTPRedirectHandler to make it use our Subclass the HTTPRedirectHandler to make it use our
HEADRequest also on the redirected URL HEADRequest also on the redirected URL
""" """
def redirect_request(self, req, fp, code, msg, headers, newurl): def redirect_request(self, req, fp, code, msg, headers, newurl):
if code in (301, 302, 303, 307): if code in (301, 302, 303, 307):
newurl = newurl.replace(' ', '%20') newurl = newurl.replace(' ', '%20')
@ -133,9 +135,11 @@ class GenericIE(InfoExtractor):
raise compat_urllib_error.HTTPError(req.get_full_url(), code, msg, headers, fp) raise compat_urllib_error.HTTPError(req.get_full_url(), code, msg, headers, fp)
class HTTPMethodFallback(compat_urllib_request.BaseHandler): class HTTPMethodFallback(compat_urllib_request.BaseHandler):
""" """
Fallback to GET if HEAD is not allowed (405 HTTP error) Fallback to GET if HEAD is not allowed (405 HTTP error)
""" """
def http_error_405(self, req, fp, code, msg, headers): def http_error_405(self, req, fp, code, msg, headers):
fp.read() fp.read()
fp.close() fp.close()

View File

@ -6,6 +6,7 @@ import json
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import determine_ext from ..utils import determine_ext
class HarkIE(InfoExtractor): class HarkIE(InfoExtractor):
_VALID_URL = r'https?://www\.hark\.com/clips/(.+?)-.+' _VALID_URL = r'https?://www\.hark\.com/clips/(.+?)-.+'
_TEST = { _TEST = {

View File

@ -13,6 +13,7 @@ from ..utils import (
class HypemIE(InfoExtractor): class HypemIE(InfoExtractor):
"""Information Extractor for hypem""" """Information Extractor for hypem"""
_VALID_URL = r'(?:http://)?(?:www\.)?hypem\.com/track/([^/]+)/([^/]+)' _VALID_URL = r'(?:http://)?(?:www\.)?hypem\.com/track/([^/]+)/([^/]+)'
_TEST = { _TEST = {

View File

@ -8,6 +8,7 @@ from ..utils import (
class IGNIE(InfoExtractor): class IGNIE(InfoExtractor):
""" """
Extractor for some of the IGN sites, like www.ign.com, es.ign.com de.ign.com. Extractor for some of the IGN sites, like www.ign.com, es.ign.com de.ign.com.
Some videos of it.ign.com are also supported Some videos of it.ign.com are also supported
@ -101,6 +102,7 @@ class IGNIE(InfoExtractor):
class OneUPIE(IGNIE): class OneUPIE(IGNIE):
"""Extractor for 1up.com, it uses the ign videos system.""" """Extractor for 1up.com, it uses the ign videos system."""
_VALID_URL = r'https?://gamevideos\.1up\.com/(?P<type>video)/id/(?P<name_or_id>.+)' _VALID_URL = r'https?://gamevideos\.1up\.com/(?P<type>video)/id/(?P<name_or_id>.+)'

View File

@ -45,4 +45,3 @@ class JadoreCettePubIE(InfoExtractor):
'title': title, 'title': title,
'description': description, 'description': description,
} }

View File

@ -7,6 +7,7 @@ from ..utils import (
unescapeHTML, unescapeHTML,
) )
class JukeboxIE(InfoExtractor): class JukeboxIE(InfoExtractor):
_VALID_URL = r'^http://www\.jukebox?\..+?\/.+[,](?P<video_id>[a-z0-9\-]+)\.html' _VALID_URL = r'^http://www\.jukebox?\..+?\/.+[,](?P<video_id>[a-z0-9\-]+)\.html'
_IFRAME = r'<iframe .*src="(?P<iframe>[^"]*)".*>' _IFRAME = r'<iframe .*src="(?P<iframe>[^"]*)".*>'

View File

@ -10,6 +10,7 @@ from ..utils import (
class JustinTVIE(InfoExtractor): class JustinTVIE(InfoExtractor):
"""Information extractor for justin.tv and twitch.tv""" """Information extractor for justin.tv and twitch.tv"""
# TODO: One broadcast may be split into multiple videos. The key # TODO: One broadcast may be split into multiple videos. The key
# 'broadcast_id' is the same for all parts, and 'broadcast_part' # 'broadcast_id' is the same for all parts, and 'broadcast_part'
@ -49,7 +50,7 @@ class JustinTVIE(InfoExtractor):
u'unable to download video info JSON') u'unable to download video info JSON')
response = json.loads(info_json) response = json.loads(info_json)
if type(response) != list: if not isinstance(response, list):
error_text = response.get('error', 'unknown error') error_text = response.get('error', 'unknown error')
raise ExtractorError(u'Justin.tv API: %s' % error_text) raise ExtractorError(u'Justin.tv API: %s' % error_text)
info = [] info = []

View File

@ -11,6 +11,7 @@ from ..aes import (
aes_decrypt_text aes_decrypt_text
) )
class KeezMoviesIE(InfoExtractor): class KeezMoviesIE(InfoExtractor):
_VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>keezmovies\.com/video/.+?(?P<videoid>[0-9]+))(?:[/?&]|$)' _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>keezmovies\.com/video/.+?(?P<videoid>[0-9]+))(?:[/?&]|$)'
_TEST = { _TEST = {

View File

@ -7,6 +7,7 @@ from ..utils import (
compat_urllib_parse, compat_urllib_parse,
) )
class MalemotionIE(InfoExtractor): class MalemotionIE(InfoExtractor):
_VALID_URL = r'^(?:https?://)?malemotion\.com/video/(.+?)\.(?P<id>.+?)(#|$)' _VALID_URL = r'^(?:https?://)?malemotion\.com/video/(.+?)\.(?P<id>.+?)(#|$)'
_TEST = { _TEST = {

View File

@ -9,7 +9,9 @@ from ..utils import (
ExtractorError, ExtractorError,
) )
class MetacafeIE(InfoExtractor): class MetacafeIE(InfoExtractor):
"""Information Extractor for metacafe.com.""" """Information Extractor for metacafe.com."""
_VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*' _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
@ -82,7 +84,6 @@ class MetacafeIE(InfoExtractor):
}, },
] ]
def report_disclaimer(self): def report_disclaimer(self):
"""Report disclaimer retrieval.""" """Report disclaimer retrieval."""
self.to_screen(u'Retrieving disclaimer') self.to_screen(u'Retrieving disclaimer')

View File

@ -8,6 +8,7 @@ from ..utils import (
compat_urllib_parse, compat_urllib_parse,
) )
class MofosexIE(InfoExtractor): class MofosexIE(InfoExtractor):
_VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>mofosex\.com/videos/(?P<videoid>[0-9]+)/.*?\.html)' _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>mofosex\.com/videos/(?P<videoid>[0-9]+)/.*?\.html)'
_TEST = { _TEST = {

View File

@ -18,6 +18,7 @@ def _media_xml_tag(tag):
class MTVServicesInfoExtractor(InfoExtractor): class MTVServicesInfoExtractor(InfoExtractor):
@staticmethod @staticmethod
def _id_from_uri(uri): def _id_from_uri(uri):
return uri.split(':')[-1] return uri.split(':')[-1]
@ -173,7 +174,7 @@ class MTVIE(MTVServicesInfoExtractor):
m_vevo = re.search(r'isVevoVideo = true;.*?vevoVideoId = "(.*?)";', m_vevo = re.search(r'isVevoVideo = true;.*?vevoVideoId = "(.*?)";',
webpage, re.DOTALL) webpage, re.DOTALL)
if m_vevo: if m_vevo:
vevo_id = m_vevo.group(1); vevo_id = m_vevo.group(1)
self.to_screen('Vevo video detected: %s' % vevo_id) self.to_screen('Vevo video detected: %s' % vevo_id)
return self.url_result('vevo:%s' % vevo_id, ie='Vevo') return self.url_result('vevo:%s' % vevo_id, ie='Vevo')

View File

@ -14,8 +14,8 @@ from ..utils import (
) )
class MyVideoIE(InfoExtractor): class MyVideoIE(InfoExtractor):
"""Information Extractor for myvideo.de.""" """Information Extractor for myvideo.de."""
_VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/(?:[^/]+/)?watch/([0-9]+)/([^?/]+).*' _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/(?:[^/]+/)?watch/([0-9]+)/([^?/]+).*'
@ -187,4 +187,3 @@ class MyVideoIE(InfoExtractor):
'video_hls_playlist': video_hls_playlist, 'video_hls_playlist': video_hls_playlist,
'player_url': video_swfobj, 'player_url': video_swfobj,
}] }]

View File

@ -11,6 +11,7 @@ from ..utils import (
class NHLBaseInfoExtractor(InfoExtractor): class NHLBaseInfoExtractor(InfoExtractor):
@staticmethod @staticmethod
def _fix_json(json_string): def _fix_json(json_string):
return json_string.replace('\\\'', '\'') return json_string.replace('\\\'', '\'')

View File

@ -7,6 +7,7 @@ from ..utils import (
unified_strdate, unified_strdate,
) )
class NormalbootsIE(InfoExtractor): class NormalbootsIE(InfoExtractor):
_VALID_URL = r'(?:http://)?(?:www\.)?normalboots\.com/video/(?P<videoid>[0-9a-z-]*)/?$' _VALID_URL = r'(?:http://)?(?:www\.)?normalboots\.com/video/(?P<videoid>[0-9a-z-]*)/?$'
_TEST = { _TEST = {

View File

@ -4,6 +4,7 @@ import json
from .common import InfoExtractor from .common import InfoExtractor
from ..utils import unescapeHTML from ..utils import unescapeHTML
class OoyalaIE(InfoExtractor): class OoyalaIE(InfoExtractor):
_VALID_URL = r'https?://.+?\.ooyala\.com/.*?(?:embedCode|ec)=(?P<id>.+?)(&|$)' _VALID_URL = r'https?://.+?\.ooyala\.com/.*?(?:embedCode|ec)=(?P<id>.+?)(&|$)'
@ -60,4 +61,3 @@ class OoyalaIE(InfoExtractor):
} }
else: else:
return self._extract_result(videos_info[0], videos_more_info) return self._extract_result(videos_info[0], videos_more_info)

View File

@ -8,7 +8,9 @@ from ..utils import (
ExtractorError, ExtractorError,
) )
class PhotobucketIE(InfoExtractor): class PhotobucketIE(InfoExtractor):
"""Information extractor for photobucket.com.""" """Information extractor for photobucket.com."""
# TODO: the original _VALID_URL was: # TODO: the original _VALID_URL was:

View File

@ -38,7 +38,8 @@ class PornotubeIE(InfoExtractor):
# Get the uploaded date # Get the uploaded date
VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by' VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by'
upload_date = self._html_search_regex(VIDEO_UPLOADED_RE, webpage, u'upload date', fatal=False) upload_date = self._html_search_regex(VIDEO_UPLOADED_RE, webpage, u'upload date', fatal=False)
if upload_date: upload_date = unified_strdate(upload_date) if upload_date:
upload_date = unified_strdate(upload_date)
age_limit = self._rta_search(webpage) age_limit = self._rta_search(webpage)
info = {'id': video_id, info = {'id': video_id,

View File

@ -41,4 +41,3 @@ class RingTVIE(InfoExtractor):
'thumbnail': thumbnail_url, 'thumbnail': thumbnail_url,
'description': description, 'description': description,
} }

View File

@ -12,6 +12,7 @@ from ..utils import (
class RTLnowIE(InfoExtractor): class RTLnowIE(InfoExtractor):
"""Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW""" """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
_VALID_URL = r'(?:http://)?(?P<url>(?P<domain>rtl-now\.rtl\.de|rtl2now\.rtl2\.de|(?:www\.)?voxnow\.de|(?:www\.)?rtlnitronow\.de|(?:www\.)?superrtlnow\.de|(?:www\.)?n-tvnow\.de)/+[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?(?:container_id|film_id)=(?P<video_id>[0-9]+)&player=1(?:&season=[0-9]+)?(?:&.*)?)' _VALID_URL = r'(?:http://)?(?P<url>(?P<domain>rtl-now\.rtl\.de|rtl2now\.rtl2\.de|(?:www\.)?voxnow\.de|(?:www\.)?rtlnitronow\.de|(?:www\.)?superrtlnow\.de|(?:www\.)?n-tvnow\.de)/+[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?(?:container_id|film_id)=(?P<video_id>[0-9]+)&player=1(?:&season=[0-9]+)?(?:&.*)?)'
_TESTS = [{ _TESTS = [{

View File

@ -67,5 +67,3 @@ class ServingSysIE(InfoExtractor):
'title': title, 'title': title,
'entries': entries, 'entries': entries,
} }

View File

@ -17,6 +17,7 @@ from ..utils import (
class SoundcloudIE(InfoExtractor): class SoundcloudIE(InfoExtractor):
"""Information extractor for soundcloud.com """Information extractor for soundcloud.com
To access the media, the uid of the song and a stream token To access the media, the uid of the song and a stream token
must be extracted from the page source and the script must make must be extracted from the page source and the script must make
@ -216,6 +217,7 @@ class SoundcloudIE(InfoExtractor):
info = json.loads(info_json) info = json.loads(info_json)
return self._extract_info_dict(info, full_title, secret_token=token) return self._extract_info_dict(info, full_title, secret_token=token)
class SoundcloudSetIE(SoundcloudIE): class SoundcloudSetIE(SoundcloudIE):
_VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/sets/([\w\d-]+)(?:[?].*)?$' _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/sets/([\w\d-]+)(?:[?].*)?$'
IE_NAME = 'soundcloud:set' IE_NAME = 'soundcloud:set'

View File

@ -38,7 +38,6 @@ class SteamIE(InfoExtractor):
] ]
} }
@classmethod @classmethod
def suitable(cls, url): def suitable(cls, url):
"""Receives a URL and returns True if suitable for this IE.""" """Receives a URL and returns True if suitable for this IE."""

View File

@ -7,6 +7,7 @@ from ..utils import (
class SubtitlesInfoExtractor(InfoExtractor): class SubtitlesInfoExtractor(InfoExtractor):
@property @property
def _have_to_download_any_subtitles(self): def _have_to_download_any_subtitles(self):
return any([self._downloader.params.get('writesubtitles', False), return any([self._downloader.params.get('writesubtitles', False),

View File

@ -45,7 +45,6 @@ class TEDIE(SubtitlesInfoExtractor):
self.to_screen(u'Getting info of playlist %s: "%s"' % (playlist_id, name)) self.to_screen(u'Getting info of playlist %s: "%s"' % (playlist_id, name))
return [self._playlist_videos_info(url, name, playlist_id)] return [self._playlist_videos_info(url, name, playlist_id)]
def _playlist_videos_info(self, url, name, playlist_id): def _playlist_videos_info(self, url, name, playlist_id):
'''Returns the videos of the playlist''' '''Returns the videos of the playlist'''

View File

@ -7,6 +7,7 @@ from ..utils import ExtractorError
class TestURLIE(InfoExtractor): class TestURLIE(InfoExtractor):
""" Allows adressing of the test cases as test:yout.*be_1 """ """ Allows adressing of the test cases as test:yout.*be_1 """
IE_DESC = False # Do not list IE_DESC = False # Do not list

View File

@ -5,7 +5,9 @@ import re
from .common import InfoExtractor from .common import InfoExtractor
class TF1IE(InfoExtractor): class TF1IE(InfoExtractor):
"""TF1 uses the wat.tv player.""" """TF1 uses the wat.tv player."""
_VALID_URL = r'http://videos\.tf1\.fr/.*-(.*?)\.html' _VALID_URL = r'http://videos\.tf1\.fr/.*-(.*?)\.html'
_TEST = { _TEST = {

View File

@ -63,4 +63,3 @@ class TriluliluIE(InfoExtractor):
'description': description, 'description': description,
'thumbnail': thumbnail, 'thumbnail': thumbnail,
} }

View File

@ -10,6 +10,7 @@ from ..aes import (
aes_decrypt_text aes_decrypt_text
) )
class Tube8IE(InfoExtractor): class Tube8IE(InfoExtractor):
_VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>tube8\.com/.+?/(?P<videoid>\d+)/?)$' _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>tube8\.com/.+?/(?P<videoid>\d+)/?)$'
_TEST = { _TEST = {

View File

@ -2,6 +2,7 @@ import re
from .common import InfoExtractor from .common import InfoExtractor
class UnistraIE(InfoExtractor): class UnistraIE(InfoExtractor):
_VALID_URL = r'http://utv\.unistra\.fr/(?:index|video)\.php\?id_video\=(\d+)' _VALID_URL = r'http://utv\.unistra\.fr/(?:index|video)\.php\?id_video\=(\d+)'

View File

@ -12,6 +12,7 @@ from ..utils import (
class VevoIE(InfoExtractor): class VevoIE(InfoExtractor):
""" """
Accepts urls from vevo.com or in the format 'vevo:{id}' Accepts urls from vevo.com or in the format 'vevo:{id}'
(currently used by MTVIE) (currently used by MTVIE)

View File

@ -35,4 +35,3 @@ class ViceIE(InfoExtractor):
except ExtractorError: except ExtractorError:
raise ExtractorError(u'The page doesn\'t contain a video', expected=True) raise ExtractorError(u'The page doesn\'t contain a video', expected=True)
return self.url_result(ooyala_url, ie='Ooyala') return self.url_result(ooyala_url, ie='Ooyala')

View File

@ -6,6 +6,7 @@ from ..utils import (
determine_ext, determine_ext,
) )
class VideofyMeIE(InfoExtractor): class VideofyMeIE(InfoExtractor):
_VALID_URL = r'https?://(www\.videofy\.me/.+?|p\.videofy\.me/v)/(?P<id>\d+)(&|#|$)' _VALID_URL = r'https?://(www\.videofy\.me/.+?|p\.videofy\.me/v)/(?P<id>\d+)(&|#|$)'
IE_NAME = u'videofy.me' IE_NAME = u'videofy.me'

View File

@ -20,6 +20,7 @@ from ..utils import (
class VimeoIE(SubtitlesInfoExtractor): class VimeoIE(SubtitlesInfoExtractor):
"""Information extractor for vimeo.com.""" """Information extractor for vimeo.com."""
# _VALID_URL matches Vimeo URLs # _VALID_URL matches Vimeo URLs
@ -227,7 +228,8 @@ class VimeoIE(SubtitlesInfoExtractor):
video_description = None video_description = None
try: try:
video_description = get_element_by_attribute("itemprop", "description", webpage) video_description = get_element_by_attribute("itemprop", "description", webpage)
if video_description: video_description = clean_html(video_description) if video_description:
video_description = clean_html(video_description)
except AssertionError as err: except AssertionError as err:
# On some pages like (http://player.vimeo.com/video/54469442) the # On some pages like (http://player.vimeo.com/video/54469442) the
# html tags are not closed, python 2.6 cannot handle it # html tags are not closed, python 2.6 cannot handle it

View File

@ -31,7 +31,6 @@ class WatIE(InfoExtractor):
info = json.loads(info) info = json.loads(info)
return info['media'] return info['media']
def _real_extract(self, url): def _real_extract(self, url):
def real_id_for_chapter(chapter): def real_id_for_chapter(chapter):
return chapter['tc_start'].split('-')[0] return chapter['tc_start'].split('-')[0]

View File

@ -5,7 +5,9 @@ import json
from .common import InfoExtractor from .common import InfoExtractor
class WeiboIE(InfoExtractor): class WeiboIE(InfoExtractor):
""" """
The videos in Weibo come from different sites, this IE just finds the link The videos in Weibo come from different sites, this IE just finds the link
to the external video and returns it. to the external video and returns it.
@ -46,4 +48,3 @@ class WeiboIE(InfoExtractor):
sina_id = m_sina.group(1) sina_id = m_sina.group(1)
player_url = 'http://you.video.sina.com.cn/swf/quotePlayer.swf?vid=%s' % sina_id player_url = 'http://you.video.sina.com.cn/swf/quotePlayer.swf?vid=%s' % sina_id
return self.url_result(player_url) return self.url_result(player_url)

View File

@ -14,7 +14,6 @@ class WorldStarHipHopIE(InfoExtractor):
} }
} }
def _real_extract(self, url): def _real_extract(self, url):
m = re.match(self._VALID_URL, url) m = re.match(self._VALID_URL, url)
video_id = m.group('id') video_id = m.group('id')

View File

@ -13,6 +13,7 @@ from ..utils import (
class XHamsterIE(InfoExtractor): class XHamsterIE(InfoExtractor):
"""Information Extractor for xHamster""" """Information Extractor for xHamster"""
_VALID_URL = r'http://(?:www\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.+?)\.html(?:\?.*)?' _VALID_URL = r'http://(?:www\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.+?)\.html(?:\?.*)?'
_TESTS = [ _TESTS = [

View File

@ -9,6 +9,7 @@ from ..utils import (
compat_urllib_request, compat_urllib_request,
) )
class XTubeIE(InfoExtractor): class XTubeIE(InfoExtractor):
_VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>xtube\.com/watch\.php\?v=(?P<videoid>[^/?&]+))' _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>xtube\.com/watch\.php\?v=(?P<videoid>[^/?&]+))'
_TEST = { _TEST = {

View File

@ -24,7 +24,6 @@ class YoukuIE(InfoExtractor):
} }
} }
def _gen_sid(self): def _gen_sid(self):
nowTime = int(time.time() * 1000) nowTime = int(time.time() * 1000)
random1 = random.randint(1000, 1998) random1 = random.randint(1000, 1998)
@ -92,7 +91,6 @@ class YoukuIE(InfoExtractor):
format = 'flv' format = 'flv'
ext = u'flv' ext = u'flv'
fileid = config['data'][0]['streamfileids'][format] fileid = config['data'][0]['streamfileids'][format]
keys = [s['k'] for s in config['data'][0]['segs'][format]] keys = [s['k'] for s in config['data'][0]['segs'][format]]
# segs is usually a dictionary, but an empty *list* if an error occured. # segs is usually a dictionary, but an empty *list* if an error occured.

View File

@ -37,7 +37,9 @@ from ..utils import (
uppercase_escape, uppercase_escape,
) )
class YoutubeBaseInfoExtractor(InfoExtractor): class YoutubeBaseInfoExtractor(InfoExtractor):
"""Provide base functions for Youtube extractors""" """Provide base functions for Youtube extractors"""
_LOGIN_URL = 'https://accounts.google.com/ServiceLogin' _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
_LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1' _LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
@ -299,11 +301,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
}, },
] ]
@classmethod @classmethod
def suitable(cls, url): def suitable(cls, url):
"""Receives a URL and returns True if suitable for this IE.""" """Receives a URL and returns True if suitable for this IE."""
if YoutubePlaylistIE.suitable(url): return False if YoutubePlaylistIE.suitable(url):
return False
return re.match(cls._VALID_URL, url) is not None return re.match(cls._VALID_URL, url) is not None
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
@ -1097,6 +1099,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
def _extract_from_m3u8(self, manifest_url, video_id): def _extract_from_m3u8(self, manifest_url, video_id):
url_map = {} url_map = {}
def _get_urls(_manifest): def _get_urls(_manifest):
lines = _manifest.split('\n') lines = _manifest.split('\n')
urls = filter(lambda l: l and not l.startswith('#'), urls = filter(lambda l: l and not l.startswith('#'),
@ -1423,6 +1426,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
'formats': formats, 'formats': formats,
} }
class YoutubePlaylistIE(YoutubeBaseInfoExtractor): class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
IE_DESC = u'YouTube.com playlists' IE_DESC = u'YouTube.com playlists'
_VALID_URL = r"""(?x)(?: _VALID_URL = r"""(?x)(?:
@ -1622,8 +1626,10 @@ class YoutubeUserIE(InfoExtractor):
# Don't return True if the url can be extracted with other youtube # Don't return True if the url can be extracted with other youtube
# extractor, the regex would is too permissive and it would match. # extractor, the regex would is too permissive and it would match.
other_ies = iter(klass for (name, klass) in globals().items() if name.endswith('IE') and klass is not cls) other_ies = iter(klass for (name, klass) in globals().items() if name.endswith('IE') and klass is not cls)
if any(ie.suitable(url) for ie in other_ies): return False if any(ie.suitable(url) for ie in other_ies):
else: return super(YoutubeUserIE, cls).suitable(url) return False
else:
return super(YoutubeUserIE, cls).suitable(url)
def _real_extract(self, url): def _real_extract(self, url):
# Extract username # Extract username
@ -1710,12 +1716,14 @@ class YoutubeSearchIE(SearchInfoExtractor):
for video_id in video_ids] for video_id in video_ids]
return self.playlist_result(videos, query) return self.playlist_result(videos, query)
class YoutubeSearchDateIE(YoutubeSearchIE): class YoutubeSearchDateIE(YoutubeSearchIE):
IE_NAME = YoutubeSearchIE.IE_NAME + ':date' IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
_API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc&orderby=published' _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc&orderby=published'
_SEARCH_KEY = 'ytsearchdate' _SEARCH_KEY = 'ytsearchdate'
IE_DESC = u'YouTube.com searches, newest videos first' IE_DESC = u'YouTube.com searches, newest videos first'
class YoutubeShowIE(InfoExtractor): class YoutubeShowIE(InfoExtractor):
IE_DESC = u'YouTube.com (multi-season) shows' IE_DESC = u'YouTube.com (multi-season) shows'
_VALID_URL = r'https?://www\.youtube\.com/show/(.*)' _VALID_URL = r'https?://www\.youtube\.com/show/(.*)'
@ -1732,6 +1740,7 @@ class YoutubeShowIE(InfoExtractor):
class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor): class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
""" """
Base class for extractors that fetch info from Base class for extractors that fetch info from
http://www.youtube.com/feed_ajax http://www.youtube.com/feed_ajax
@ -1774,18 +1783,21 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
paging = info['paging'] paging = info['paging']
return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE) return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE)
class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor): class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
IE_DESC = u'YouTube.com subscriptions feed, "ytsubs" keyword(requires authentication)' IE_DESC = u'YouTube.com subscriptions feed, "ytsubs" keyword(requires authentication)'
_VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?' _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
_FEED_NAME = 'subscriptions' _FEED_NAME = 'subscriptions'
_PLAYLIST_TITLE = u'Youtube Subscriptions' _PLAYLIST_TITLE = u'Youtube Subscriptions'
class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor): class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
IE_DESC = u'YouTube.com recommended videos, "ytrec" keyword (requires authentication)' IE_DESC = u'YouTube.com recommended videos, "ytrec" keyword (requires authentication)'
_VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?' _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
_FEED_NAME = 'recommended' _FEED_NAME = 'recommended'
_PLAYLIST_TITLE = u'Youtube Recommended videos' _PLAYLIST_TITLE = u'Youtube Recommended videos'
class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor): class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor):
IE_DESC = u'Youtube watch later list, "ytwatchlater" keyword (requires authentication)' IE_DESC = u'Youtube watch later list, "ytwatchlater" keyword (requires authentication)'
_VALID_URL = r'https?://www\.youtube\.com/feed/watch_later|:ytwatchlater' _VALID_URL = r'https?://www\.youtube\.com/feed/watch_later|:ytwatchlater'
@ -1793,6 +1805,7 @@ class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor):
_PLAYLIST_TITLE = u'Youtube Watch Later' _PLAYLIST_TITLE = u'Youtube Watch Later'
_PERSONAL_FEED = True _PERSONAL_FEED = True
class YoutubeHistoryIE(YoutubeFeedsInfoExtractor): class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
IE_DESC = u'Youtube watch history, "ythistory" keyword (requires authentication)' IE_DESC = u'Youtube watch history, "ythistory" keyword (requires authentication)'
_VALID_URL = u'https?://www\.youtube\.com/feed/history|:ythistory' _VALID_URL = u'https?://www\.youtube\.com/feed/history|:ythistory'
@ -1800,6 +1813,7 @@ class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
_PERSONAL_FEED = True _PERSONAL_FEED = True
_PLAYLIST_TITLE = u'Youtube Watch History' _PLAYLIST_TITLE = u'Youtube Watch History'
class YoutubeFavouritesIE(YoutubeBaseInfoExtractor): class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
IE_NAME = u'youtube:favorites' IE_NAME = u'youtube:favorites'
IE_DESC = u'YouTube.com favourite videos, "ytfav" keyword (requires authentication)' IE_DESC = u'YouTube.com favourite videos, "ytfav" keyword (requires authentication)'

View File

@ -2,6 +2,7 @@ from ..utils import PostProcessingError
class PostProcessor(object): class PostProcessor(object):
"""Post Processor class. """Post Processor class.
PostProcessor objects can be added to downloaders with their PostProcessor objects can be added to downloaders with their

View File

@ -17,11 +17,12 @@ from ..utils import (
) )
class FFmpegPostProcessorError(PostProcessingError): class FFmpegPostProcessorError(PostProcessingError):
pass pass
class FFmpegPostProcessor(PostProcessor): class FFmpegPostProcessor(PostProcessor):
def __init__(self, downloader=None): def __init__(self, downloader=None):
PostProcessor.__init__(self, downloader) PostProcessor.__init__(self, downloader)
self._exes = self.detect_executables() self._exes = self.detect_executables()
@ -71,6 +72,7 @@ class FFmpegPostProcessor(PostProcessor):
class FFmpegExtractAudioPP(FFmpegPostProcessor): class FFmpegExtractAudioPP(FFmpegPostProcessor):
def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, nopostoverwrites=False): def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, nopostoverwrites=False):
FFmpegPostProcessor.__init__(self, downloader) FFmpegPostProcessor.__init__(self, downloader)
if preferredcodec is None: if preferredcodec is None:
@ -199,6 +201,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
class FFmpegVideoConvertor(FFmpegPostProcessor): class FFmpegVideoConvertor(FFmpegPostProcessor):
def __init__(self, downloader=None, preferedformat=None): def __init__(self, downloader=None, preferedformat=None):
super(FFmpegVideoConvertor, self).__init__(downloader) super(FFmpegVideoConvertor, self).__init__(downloader)
self._preferedformat = preferedformat self._preferedformat = preferedformat
@ -446,6 +449,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
class FFmpegMetadataPP(FFmpegPostProcessor): class FFmpegMetadataPP(FFmpegPostProcessor):
def run(self, info): def run(self, info):
metadata = {} metadata = {}
if info.get('title') is not None: if info.get('title') is not None:
@ -476,10 +480,10 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
class FFmpegMergerPP(FFmpegPostProcessor): class FFmpegMergerPP(FFmpegPostProcessor):
def run(self, info): def run(self, info):
filename = info['filepath'] filename = info['filepath']
args = ['-c', 'copy'] args = ['-c', 'copy']
self._downloader.to_screen(u'[ffmpeg] Merging formats into "%s"' % filename) self._downloader.to_screen(u'[ffmpeg] Merging formats into "%s"' % filename)
self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args) self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args)
return True, info return True, info

View File

@ -105,4 +105,3 @@ class XAttrMetadataPP(PostProcessor):
except (subprocess.CalledProcessError, OSError): except (subprocess.CalledProcessError, OSError):
self._downloader.report_error("This filesystem doesn't support extended attributes. (You may have to enable them in your /etc/fstab)") self._downloader.report_error("This filesystem doesn't support extended attributes. (You may have to enable them in your /etc/fstab)")
return False, info return False, info

View File

@ -13,14 +13,18 @@ from .utils import (
) )
from .version import __version__ from .version import __version__
def rsa_verify(message, signature, key): def rsa_verify(message, signature, key):
from struct import pack from struct import pack
from hashlib import sha256 from hashlib import sha256
from sys import version_info from sys import version_info
def b(x): def b(x):
if version_info[0] == 2: return x if version_info[0] == 2:
else: return x.encode('latin1') return x
assert(type(message) == type(b(''))) else:
return x.encode('latin1')
assert(isinstance(message, type(b(''))))
block_size = 0 block_size = 0
n = key[0] n = key[0]
while n: while n:
@ -32,13 +36,17 @@ def rsa_verify(message, signature, key):
raw_bytes.insert(0, pack("B", signature & 0xFF)) raw_bytes.insert(0, pack("B", signature & 0xFF))
signature >>= 8 signature >>= 8
signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes) signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
if signature[0:2] != b('\x00\x01'): return False if signature[0:2] != b('\x00\x01'):
return False
signature = signature[2:] signature = signature[2:]
if not b('\x00') in signature: return False if not b('\x00') in signature:
return False
signature = signature[signature.index(b('\x00')) + 1:] signature = signature[signature.index(b('\x00')) + 1:]
if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')):
return False
signature = signature[19:] signature = signature[19:]
if signature != sha256(message).digest(): return False if signature != sha256(message).digest():
return False
return True return True
@ -58,7 +66,8 @@ def update_self(to_screen, verbose):
try: try:
newversion = compat_urllib_request.urlopen(VERSION_URL).read().decode('utf-8').strip() newversion = compat_urllib_request.urlopen(VERSION_URL).read().decode('utf-8').strip()
except: except:
if verbose: to_screen(compat_str(traceback.format_exc())) if verbose:
to_screen(compat_str(traceback.format_exc()))
to_screen(u'ERROR: can\'t find the current version. Please try again later.') to_screen(u'ERROR: can\'t find the current version. Please try again later.')
return return
if newversion == __version__: if newversion == __version__:
@ -70,7 +79,8 @@ def update_self(to_screen, verbose):
versions_info = compat_urllib_request.urlopen(JSON_URL).read().decode('utf-8') versions_info = compat_urllib_request.urlopen(JSON_URL).read().decode('utf-8')
versions_info = json.loads(versions_info) versions_info = json.loads(versions_info)
except: except:
if verbose: to_screen(compat_str(traceback.format_exc())) if verbose:
to_screen(compat_str(traceback.format_exc()))
to_screen(u'ERROR: can\'t obtain versions info. Please try again later.') to_screen(u'ERROR: can\'t obtain versions info. Please try again later.')
return return
if not 'signature' in versions_info: if not 'signature' in versions_info:
@ -118,7 +128,8 @@ def update_self(to_screen, verbose):
newcontent = urlh.read() newcontent = urlh.read()
urlh.close() urlh.close()
except (IOError, OSError): except (IOError, OSError):
if verbose: to_screen(compat_str(traceback.format_exc())) if verbose:
to_screen(compat_str(traceback.format_exc()))
to_screen(u'ERROR: unable to download latest version') to_screen(u'ERROR: unable to download latest version')
return return
@ -131,7 +142,8 @@ def update_self(to_screen, verbose):
with open(exe + '.new', 'wb') as outf: with open(exe + '.new', 'wb') as outf:
outf.write(newcontent) outf.write(newcontent)
except (IOError, OSError): except (IOError, OSError):
if verbose: to_screen(compat_str(traceback.format_exc())) if verbose:
to_screen(compat_str(traceback.format_exc()))
to_screen(u'ERROR: unable to write the new version') to_screen(u'ERROR: unable to write the new version')
return return
@ -150,7 +162,8 @@ start /b "" cmd /c del "%%~f0"&exit /b"
subprocess.Popen([bat]) # Continues to run in the background subprocess.Popen([bat]) # Continues to run in the background
return # Do not show premature success messages return # Do not show premature success messages
except (IOError, OSError): except (IOError, OSError):
if verbose: to_screen(compat_str(traceback.format_exc())) if verbose:
to_screen(compat_str(traceback.format_exc()))
to_screen(u'ERROR: unable to overwrite current version') to_screen(u'ERROR: unable to overwrite current version')
return return
@ -161,7 +174,8 @@ start /b "" cmd /c del "%%~f0"&exit /b"
newcontent = urlh.read() newcontent = urlh.read()
urlh.close() urlh.close()
except (IOError, OSError): except (IOError, OSError):
if verbose: to_screen(compat_str(traceback.format_exc())) if verbose:
to_screen(compat_str(traceback.format_exc()))
to_screen(u'ERROR: unable to download latest version') to_screen(u'ERROR: unable to download latest version')
return return
@ -174,12 +188,14 @@ start /b "" cmd /c del "%%~f0"&exit /b"
with open(filename, 'wb') as outf: with open(filename, 'wb') as outf:
outf.write(newcontent) outf.write(newcontent)
except (IOError, OSError): except (IOError, OSError):
if verbose: to_screen(compat_str(traceback.format_exc())) if verbose:
to_screen(compat_str(traceback.format_exc()))
to_screen(u'ERROR: unable to overwrite current version') to_screen(u'ERROR: unable to overwrite current version')
return return
to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.') to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
def get_notes(versions, fromVersion): def get_notes(versions, fromVersion):
notes = [] notes = []
for v, vdata in sorted(versions.items()): for v, vdata in sorted(versions.items()):
@ -187,6 +203,7 @@ def get_notes(versions, fromVersion):
notes.extend(vdata.get('notes', [])) notes.extend(vdata.get('notes', []))
return notes return notes
def print_notes(to_screen, versions, fromVersion=__version__): def print_notes(to_screen, versions, fromVersion=__version__):
notes = get_notes(versions, fromVersion) notes = get_notes(versions, fromVersion)
if notes: if notes:

View File

@ -174,9 +174,12 @@ try:
except NameError: except NameError:
compat_chr = chr compat_chr = chr
def compat_ord(c): def compat_ord(c):
if type(c) is int: return c if isinstance(c, int):
else: return ord(c) return c
else:
return ord(c)
# This is not clearly defined otherwise # This is not clearly defined otherwise
compiled_regex_type = type(re.compile('')) compiled_regex_type = type(re.compile(''))
@ -189,6 +192,7 @@ std_headers = {
'Accept-Language': 'en-us,en;q=0.5', 'Accept-Language': 'en-us,en;q=0.5',
} }
def preferredencoding(): def preferredencoding():
"""Get preferred encoding. """Get preferred encoding.
@ -208,7 +212,7 @@ if sys.version_info < (3,0):
print(s.encode(preferredencoding(), 'xmlcharrefreplace')) print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
else: else:
def compat_print(s): def compat_print(s):
assert type(s) == type(u'') assert isinstance(s, type(u''))
print(s) print(s)
# In Python 2.x, json.dump expects a bytestream. # In Python 2.x, json.dump expects a bytestream.
@ -238,6 +242,8 @@ else:
# On python2.6 the xml.etree.ElementTree.Element methods don't support # On python2.6 the xml.etree.ElementTree.Element methods don't support
# the namespace parameter # the namespace parameter
def xpath_with_ns(path, ns_map): def xpath_with_ns(path, ns_map):
components = [c.split(':') for c in path.split('/')] components = [c.split(':') for c in path.split('/')]
replaced = [] replaced = []
@ -249,6 +255,7 @@ def xpath_with_ns(path, ns_map):
replaced.append('{%s}%s' % (ns_map[ns], tag)) replaced.append('{%s}%s' % (ns_map[ns], tag))
return '/'.join(replaced) return '/'.join(replaced)
def htmlentity_transform(matchobj): def htmlentity_transform(matchobj):
"""Transforms an HTML entity to a character. """Transforms an HTML entity to a character.
@ -275,7 +282,10 @@ def htmlentity_transform(matchobj):
return (u'&%s;' % entity) return (u'&%s;' % entity)
compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
class BaseHTMLParser(compat_html_parser.HTMLParser): class BaseHTMLParser(compat_html_parser.HTMLParser):
def __init(self): def __init(self):
compat_html_parser.HTMLParser.__init__(self) compat_html_parser.HTMLParser.__init__(self)
self.html = None self.html = None
@ -285,8 +295,11 @@ class BaseHTMLParser(compat_html_parser.HTMLParser):
self.feed(html) self.feed(html)
self.close() self.close()
class AttrParser(BaseHTMLParser): class AttrParser(BaseHTMLParser):
"""Modified HTMLParser that isolates a tag with the specified attribute""" """Modified HTMLParser that isolates a tag with the specified attribute"""
def __init__(self, attribute, value): def __init__(self, attribute, value):
self.attribute = attribute self.attribute = attribute
self.value = value self.value = value
@ -313,12 +326,14 @@ class AttrParser(BaseHTMLParser):
self.started = True self.started = True
self.watch_startpos = True self.watch_startpos = True
if self.started: if self.started:
if not tag in self.depth: self.depth[tag] = 0 if not tag in self.depth:
self.depth[tag] = 0
self.depth[tag] += 1 self.depth[tag] += 1
def handle_endtag(self, tag): def handle_endtag(self, tag):
if self.started: if self.started:
if tag in self.depth: self.depth[tag] -= 1 if tag in self.depth:
self.depth[tag] -= 1
if self.depth[self.result[0]] == 0: if self.depth[self.result[0]] == 0:
self.started = False self.started = False
self.result.append(self.getpos()) self.result.append(self.getpos())
@ -351,10 +366,12 @@ if sys.version_info < (2, 7, 3):
if self.rawdata[i:].startswith("</scr'+'ipt>") if self.rawdata[i:].startswith("</scr'+'ipt>")
else compat_html_parser.HTMLParser.parse_endtag(self, i)) else compat_html_parser.HTMLParser.parse_endtag(self, i))
def get_element_by_id(id, html): def get_element_by_id(id, html):
"""Return the content of the tag with the specified ID in the passed HTML document""" """Return the content of the tag with the specified ID in the passed HTML document"""
return get_element_by_attribute("id", id, html) return get_element_by_attribute("id", id, html)
def get_element_by_attribute(attribute, value, html): def get_element_by_attribute(attribute, value, html):
"""Return the content of the tag with the specified attribute in the passed HTML document""" """Return the content of the tag with the specified attribute in the passed HTML document"""
parser = AttrParser(attribute, value) parser = AttrParser(attribute, value)
@ -364,11 +381,14 @@ def get_element_by_attribute(attribute, value, html):
pass pass
return parser.get_result() return parser.get_result()
class MetaParser(BaseHTMLParser): class MetaParser(BaseHTMLParser):
""" """
Modified HTMLParser that isolates a meta tag with the specified name Modified HTMLParser that isolates a meta tag with the specified name
attribute. attribute.
""" """
def __init__(self, name): def __init__(self, name):
BaseHTMLParser.__init__(self) BaseHTMLParser.__init__(self)
self.name = name self.name = name
@ -385,6 +405,7 @@ class MetaParser(BaseHTMLParser):
def get_result(self): def get_result(self):
return self.result return self.result
def get_meta_content(name, html): def get_meta_content(name, html):
""" """
Return the content attribute from the meta tag with the given name attribute. Return the content attribute from the meta tag with the given name attribute.
@ -453,6 +474,7 @@ def timeconvert(timestr):
timestamp = email.utils.mktime_tz(timetuple) timestamp = email.utils.mktime_tz(timetuple)
return timestamp return timestamp
def sanitize_filename(s, restricted=False, is_id=False): def sanitize_filename(s, restricted=False, is_id=False):
"""Sanitizes a string so it could be used as part of a filename. """Sanitizes a string so it could be used as part of a filename.
If restricted is set, use a stricter subset of allowed characters. If restricted is set, use a stricter subset of allowed characters.
@ -485,6 +507,7 @@ def sanitize_filename(s, restricted=False, is_id=False):
result = '_' result = '_'
return result return result
def orderedSet(iterable): def orderedSet(iterable):
""" Remove all duplicates from the input iterable """ """ Remove all duplicates from the input iterable """
res = [] res = []
@ -493,11 +516,12 @@ def orderedSet(iterable):
res.append(el) res.append(el)
return res return res
def unescapeHTML(s): def unescapeHTML(s):
""" """
@param s a string @param s a string
""" """
assert type(s) == type(u'') assert isinstance(s, type(u''))
result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s) result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s)
return result return result
@ -508,7 +532,7 @@ def encodeFilename(s, for_subprocess=False):
@param s The name of the file @param s The name of the file
""" """
assert type(s) == compat_str assert isinstance(s, compat_str)
# Python 3 has a Unicode API # Python 3 has a Unicode API
if sys.version_info >= (3, 0): if sys.version_info >= (3, 0):
@ -540,6 +564,7 @@ def decodeOption(optval):
assert isinstance(optval, compat_str) assert isinstance(optval, compat_str)
return optval return optval
def formatSeconds(secs): def formatSeconds(secs):
if secs > 3600: if secs > 3600:
return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60) return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
@ -554,6 +579,7 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
import httplib import httplib
class HTTPSConnectionV3(httplib.HTTPSConnection): class HTTPSConnectionV3(httplib.HTTPSConnection):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
httplib.HTTPSConnection.__init__(self, *args, **kwargs) httplib.HTTPSConnection.__init__(self, *args, **kwargs)
@ -568,6 +594,7 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23) self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23)
class HTTPSHandlerV3(compat_urllib_request.HTTPSHandler): class HTTPSHandlerV3(compat_urllib_request.HTTPSHandler):
def https_open(self, req): def https_open(self, req):
return self.do_open(HTTPSConnectionV3, req) return self.do_open(HTTPSConnectionV3, req)
return HTTPSHandlerV3(**kwargs) return HTTPSHandlerV3(**kwargs)
@ -583,8 +610,11 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
pass # Python < 3.4 pass # Python < 3.4
return compat_urllib_request.HTTPSHandler(context=context, **kwargs) return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
class ExtractorError(Exception): class ExtractorError(Exception):
"""Error during info extraction.""" """Error during info extraction."""
def __init__(self, msg, tb=None, expected=False, cause=None): def __init__(self, msg, tb=None, expected=False, cause=None):
""" tb, if given, is the original traceback (so that it can be printed out). """ tb, if given, is the original traceback (so that it can be printed out).
If expected is set, this is a normal error message and most likely not a bug in youtube-dl. If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
@ -607,17 +637,20 @@ class ExtractorError(Exception):
class RegexNotFoundError(ExtractorError): class RegexNotFoundError(ExtractorError):
"""Error when a regex didn't match""" """Error when a regex didn't match"""
pass pass
class DownloadError(Exception): class DownloadError(Exception):
"""Download Error exception. """Download Error exception.
This exception may be thrown by FileDownloader objects if they are not This exception may be thrown by FileDownloader objects if they are not
configured to continue on errors. They will contain the appropriate configured to continue on errors. They will contain the appropriate
error message. error message.
""" """
def __init__(self, msg, exc_info=None): def __init__(self, msg, exc_info=None):
""" exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """ """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
super(DownloadError, self).__init__(msg) super(DownloadError, self).__init__(msg)
@ -625,6 +658,7 @@ class DownloadError(Exception):
class SameFileError(Exception): class SameFileError(Exception):
"""Same File exception. """Same File exception.
This exception will be thrown by FileDownloader objects if they detect This exception will be thrown by FileDownloader objects if they detect
@ -634,20 +668,25 @@ class SameFileError(Exception):
class PostProcessingError(Exception): class PostProcessingError(Exception):
"""Post Processing exception. """Post Processing exception.
This exception may be raised by PostProcessor's .run() method to This exception may be raised by PostProcessor's .run() method to
indicate an error in the postprocessing task. indicate an error in the postprocessing task.
""" """
def __init__(self, msg): def __init__(self, msg):
self.msg = msg self.msg = msg
class MaxDownloadsReached(Exception): class MaxDownloadsReached(Exception):
""" --max-downloads limit has been reached. """ """ --max-downloads limit has been reached. """
pass pass
class UnavailableVideoError(Exception): class UnavailableVideoError(Exception):
"""Unavailable Format exception. """Unavailable Format exception.
This exception will be thrown when a video is requested This exception will be thrown when a video is requested
@ -657,6 +696,7 @@ class UnavailableVideoError(Exception):
class ContentTooShortError(Exception): class ContentTooShortError(Exception):
"""Content Too Short exception. """Content Too Short exception.
This exception may be raised by FileDownloader objects when a file they This exception may be raised by FileDownloader objects when a file they
@ -671,7 +711,9 @@ class ContentTooShortError(Exception):
self.downloaded = downloaded self.downloaded = downloaded
self.expected = expected self.expected = expected
class YoutubeDLHandler(compat_urllib_request.HTTPHandler): class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
"""Handler for HTTP requests and responses. """Handler for HTTP requests and responses.
This class, when installed with an OpenerDirector, automatically adds This class, when installed with an OpenerDirector, automatically adds
@ -787,6 +829,7 @@ def unified_strdate(date_str):
upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d') upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
return upload_date return upload_date
def determine_ext(url, default_ext=u'unknown_video'): def determine_ext(url, default_ext=u'unknown_video'):
guess = url.partition(u'?')[0].rpartition(u'.')[2] guess = url.partition(u'?')[0].rpartition(u'.')[2]
if re.match(r'^[A-Za-z0-9]+$', guess): if re.match(r'^[A-Za-z0-9]+$', guess):
@ -794,9 +837,11 @@ def determine_ext(url, default_ext=u'unknown_video'):
else: else:
return default_ext return default_ext
def subtitles_filename(filename, sub_lang, sub_format): def subtitles_filename(filename, sub_lang, sub_format):
return filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format return filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
def date_from_str(date_str): def date_from_str(date_str):
""" """
Return a datetime object from a string in the format YYYYMMDD or Return a datetime object from a string in the format YYYYMMDD or
@ -823,6 +868,7 @@ def date_from_str(date_str):
return today + delta return today + delta
return datetime.datetime.strptime(date_str, "%Y%m%d").date() return datetime.datetime.strptime(date_str, "%Y%m%d").date()
def hyphenate_date(date_str): def hyphenate_date(date_str):
""" """
Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format""" Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
@ -832,8 +878,11 @@ def hyphenate_date(date_str):
else: else:
return date_str return date_str
class DateRange(object): class DateRange(object):
"""Represents a time interval between two dates""" """Represents a time interval between two dates"""
def __init__(self, start=None, end=None): def __init__(self, start=None, end=None):
"""start and end must be strings in the format accepted by date""" """start and end must be strings in the format accepted by date"""
if start is not None: if start is not None:
@ -846,15 +895,18 @@ class DateRange(object):
self.end = datetime.datetime.max.date() self.end = datetime.datetime.max.date()
if self.start > self.end: if self.start > self.end:
raise ValueError('Date range: "%s" , the start date must be before the end date' % self) raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
@classmethod @classmethod
def day(cls, day): def day(cls, day):
"""Returns a range that only contains the given day""" """Returns a range that only contains the given day"""
return cls(day, day) return cls(day, day)
def __contains__(self, date): def __contains__(self, date):
"""Check if the date is in the range""" """Check if the date is in the range"""
if not isinstance(date, datetime.date): if not isinstance(date, datetime.date):
date = date_from_str(date) date = date_from_str(date)
return self.start <= date <= self.end return self.start <= date <= self.end
def __str__(self): def __str__(self):
return '%s - %s' % (self.start.isoformat(), self.end.isoformat()) return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
@ -872,7 +924,7 @@ def platform_name():
def write_string(s, out=None): def write_string(s, out=None):
if out is None: if out is None:
out = sys.stderr out = sys.stderr
assert type(s) == compat_str assert isinstance(s, compat_str)
if ('b' in getattr(out, 'mode', '') or if ('b' in getattr(out, 'mode', '') or
sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
@ -981,6 +1033,7 @@ else:
class locked_file(object): class locked_file(object):
def __init__(self, filename, mode, encoding=None): def __init__(self, filename, mode, encoding=None):
assert mode in ['r', 'a', 'w'] assert mode in ['r', 'a', 'w']
self.f = io.open(filename, mode, encoding=encoding) self.f = io.open(filename, mode, encoding=encoding)
@ -1053,7 +1106,7 @@ def unsmuggle_url(smug_url, default=None):
def format_bytes(bytes): def format_bytes(bytes):
if bytes is None: if bytes is None:
return u'N/A' return u'N/A'
if type(bytes) is str: if isinstance(bytes, str):
bytes = float(bytes) bytes = float(bytes)
if bytes == 0.0: if bytes == 0.0:
exponent = 0 exponent = 0
@ -1132,6 +1185,7 @@ def url_basename(url):
class HEADRequest(compat_urllib_request.Request): class HEADRequest(compat_urllib_request.Request):
def get_method(self): def get_method(self):
return "HEAD" return "HEAD"
@ -1172,6 +1226,7 @@ def check_executable(exe, args=[]):
class PagedList(object): class PagedList(object):
def __init__(self, pagefunc, pagesize): def __init__(self, pagefunc, pagesize):
self._pagefunc = pagefunc self._pagefunc = pagefunc
self._pagesize = pagesize self._pagesize = pagesize