diff --git a/Makefile b/Makefile index e00f5e650..85dacfa4c 100644 --- a/Makefile +++ b/Makefile @@ -40,15 +40,15 @@ tar: youtube-dl.tar.gz pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 -youtube-dl: youtube_dl/*.py - zip --quiet youtube-dl youtube_dl/*.py +youtube-dl: youtube_dl/*.py youtube_dl/*/*.py + zip --quiet youtube-dl youtube_dl/*.py youtube_dl/*/*.py zip --quiet --junk-paths youtube-dl youtube_dl/__main__.py echo '#!$(PYTHON)' > youtube-dl cat youtube-dl.zip >> youtube-dl rm youtube-dl.zip chmod a+x youtube-dl -README.md: youtube_dl/*.py +README.md: youtube_dl/*.py youtube_dl/*/*.py COLUMNS=80 python -m youtube_dl --help | python devscripts/make_readme.py README.txt: README.md @@ -57,7 +57,7 @@ README.txt: README.md youtube-dl.1: README.md pandoc -s -f markdown -t man README.md -o youtube-dl.1 -youtube-dl.bash-completion: youtube_dl/*.py devscripts/bash-completion.in +youtube-dl.bash-completion: youtube_dl/*.py youtube_dl/*/*.py devscripts/bash-completion.in python devscripts/bash-completion.py bash-completion: youtube-dl.bash-completion diff --git a/README.md b/README.md index 2f3c81a7c..7d190249d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ % YOUTUBE-DL(1) # NAME -youtube-dl +youtube-dl - download videos from youtube.com or other video platforms # SYNOPSIS **youtube-dl** [OPTIONS] URL [URL...] @@ -121,7 +121,7 @@ which means you can modify it, redistribute it or use it however you like. video (currently youtube only) --list-subs lists all available subtitles for the video (currently youtube only) - --sub-format LANG subtitle format [srt/sbv] (default=srt) + --sub-format FORMAT subtitle format [srt/sbv] (default=srt) (currently youtube only) --sub-lang LANG language of the subtitles to download (optional) use IETF language tags like 'en' @@ -130,6 +130,7 @@ which means you can modify it, redistribute it or use it however you like. -u, --username USERNAME account username -p, --password PASSWORD account password -n, --netrc use .netrc authentication data + --video-password PASSWORD video password (vimeo only) ## Post-processing Options: -x, --extract-audio convert video files to audio-only files (requires diff --git a/devscripts/release.sh b/devscripts/release.sh index b8efdab47..46c31e437 100755 --- a/devscripts/release.sh +++ b/devscripts/release.sh @@ -14,6 +14,12 @@ set -e +skip_tests=false +if [ "$1" = '--skip-test' ]; then + skip_tests=true + shift +fi + if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.06"; exit 1; fi version="$1" if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi @@ -22,7 +28,11 @@ if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit /bin/echo -e "\n### First of all, testing..." make cleanall -nosetests --verbose --with-coverage --cover-package=youtube_dl --cover-html test --stop || exit 1 +if $skip_tests ; then + echo 'SKIPPING TESTS' +else + nosetests --verbose --with-coverage --cover-package=youtube_dl --cover-html test --stop || exit 1 +fi /bin/echo -e "\n### Changing version in version.py..." sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py @@ -59,7 +69,9 @@ git checkout HEAD -- youtube-dl youtube-dl.exe /bin/echo -e "\n### Signing and uploading the new binaries to youtube-dl.org..." for f in $RELEASE_FILES; do gpg --detach-sig "build/$version/$f"; done -scp -r "build/$version" ytdl@youtube-dl.org:html/downloads/ +scp -r "build/$version" ytdl@yt-dl.org:html/tmp/ +ssh ytdl@yt-dl.org "mv html/tmp/$version html/downloads/" +ssh ytdl@yt-dl.org "sh html/update_latest.sh $version" /bin/echo -e "\n### Now switching to gh-pages..." git clone --branch gh-pages --single-branch . build/gh-pages diff --git a/setup.py b/setup.py index eb7b1a212..61435fcb7 100644 --- a/setup.py +++ b/setup.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- from __future__ import print_function + import pkg_resources import sys @@ -23,7 +24,7 @@ py2exe_options = { "compressed": 1, "optimize": 2, "dist_dir": '.', - "dll_excludes": ['w9xpopen.exe'] + "dll_excludes": ['w9xpopen.exe'], } py2exe_console = [{ "script": "./youtube_dl/__main__.py", @@ -57,7 +58,7 @@ setup( author = 'Ricardo Garcia', maintainer = 'Philipp Hagemeister', maintainer_email = 'phihag@phihag.de', - packages = ['youtube_dl'], + packages = ['youtube_dl', 'youtube_dl.extractor'], # Provokes warning on most systems (why?!) #test_suite = 'nose.collector', diff --git a/test/test_all_urls.py b/test/test_all_urls.py index dd67286a7..d3ee296be 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -7,7 +7,7 @@ import unittest import os sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE, YoutubeChannelIE, JustinTVIE +from youtube_dl.extractor import YoutubeIE, YoutubePlaylistIE, YoutubeChannelIE, JustinTVIE class TestAllURLsMatching(unittest.TestCase): def test_youtube_playlist_matching(self): diff --git a/test/test_download.py b/test/test_download.py index 577bcdbf2..067bde4bb 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -13,8 +13,8 @@ import binascii # Allow direct execution sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import youtube_dl.FileDownloader -import youtube_dl.InfoExtractors +import youtube_dl.YoutubeDL +import youtube_dl.extractor from youtube_dl.utils import * DEF_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tests.json') @@ -40,17 +40,17 @@ def _try_rm(filename): md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest() -class FileDownloader(youtube_dl.FileDownloader): +class YoutubeDL(youtube_dl.YoutubeDL): def __init__(self, *args, **kwargs): self.to_stderr = self.to_screen self.processed_info_dicts = [] - return youtube_dl.FileDownloader.__init__(self, *args, **kwargs) + super(YoutubeDL, self).__init__(*args, **kwargs) def report_warning(self, message): # Don't accept warnings during tests raise ExtractorError(message) def process_info(self, info_dict): self.processed_info_dicts.append(info_dict) - return youtube_dl.FileDownloader.process_info(self, info_dict) + return super(YoutubeDL, self).process_info(info_dict) def _file_md5(fn): with open(fn, 'rb') as f: @@ -72,7 +72,7 @@ class TestDownload(unittest.TestCase): def generator(test_case): def test_template(self): - ie = youtube_dl.InfoExtractors.get_info_extractor(test_case['name']) + ie = youtube_dl.extractor.get_info_extractor(test_case['name']) if not ie._WORKING: print('Skipping: IE marked as not _WORKING') return @@ -86,14 +86,14 @@ def generator(test_case): params = self.parameters.copy() params.update(test_case.get('params', {})) - fd = FileDownloader(params) - for ie in youtube_dl.InfoExtractors.gen_extractors(): - fd.add_info_extractor(ie) + ydl = YoutubeDL(params) + for ie in youtube_dl.extractor.gen_extractors(): + ydl.add_info_extractor(ie) finished_hook_called = set() def _hook(status): if status['status'] == 'finished': finished_hook_called.add(status['filename']) - fd.add_progress_hook(_hook) + ydl.fd.add_progress_hook(_hook) test_cases = test_case.get('playlist', [test_case]) for tc in test_cases: @@ -103,7 +103,7 @@ def generator(test_case): try: for retry in range(1, RETRIES + 1): try: - fd.download([test_case['url']]) + ydl.download([test_case['url']]) except (DownloadError, ExtractorError) as err: if retry == RETRIES: raise @@ -125,11 +125,14 @@ def generator(test_case): self.assertEqual(md5_for_file, tc['md5']) with io.open(tc['file'] + '.info.json', encoding='utf-8') as infof: info_dict = json.load(infof) - for (info_field, value) in tc.get('info_dict', {}).items(): - if isinstance(value, compat_str) and value.startswith('md5:'): - self.assertEqual(value, 'md5:' + md5(info_dict.get(info_field))) + for (info_field, expected) in tc.get('info_dict', {}).items(): + if isinstance(expected, compat_str) and expected.startswith('md5:'): + self.assertEqual(expected, 'md5:' + md5(info_dict.get(info_field))) else: - self.assertEqual(value, info_dict.get(info_field)) + got = info_dict.get(info_field) + self.assertEqual( + expected, got, + u'invalid value for field %s, expected %r, got %r' % (info_field, expected, got)) # If checkable fields are missing from the test case, print the info_dict test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value)) @@ -150,9 +153,11 @@ def generator(test_case): return test_template ### And add them to TestDownload -for test_case in defs: +for n, test_case in enumerate(defs): test_method = generator(test_case) test_method.__name__ = "test_{0}".format(test_case["name"]) + if getattr(TestDownload, test_method.__name__, False): + test_method.__name__ = "test_{0}_{1}".format(test_case["name"], n) setattr(TestDownload, test_method.__name__, test_method) del test_method diff --git a/test/test_write_info_json.py b/test/test_write_info_json.py index 8134dda37..de6d5180f 100644 --- a/test/test_write_info_json.py +++ b/test/test_write_info_json.py @@ -9,8 +9,8 @@ import unittest # Allow direct execution sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import youtube_dl.FileDownloader -import youtube_dl.InfoExtractors +import youtube_dl.YoutubeDL +import youtube_dl.extractor from youtube_dl.utils import * PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json") @@ -22,9 +22,9 @@ proxy_handler = compat_urllib_request.ProxyHandler() opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler()) compat_urllib_request.install_opener(opener) -class FileDownloader(youtube_dl.FileDownloader): +class YoutubeDL(youtube_dl.YoutubeDL): def __init__(self, *args, **kwargs): - youtube_dl.FileDownloader.__init__(self, *args, **kwargs) + super(YoutubeDL, self).__init__(*args, **kwargs) self.to_stderr = self.to_screen with io.open(PARAMETERS_FILE, encoding='utf-8') as pf: @@ -48,10 +48,10 @@ class TestInfoJSON(unittest.TestCase): self.tearDown() def test_info_json(self): - ie = youtube_dl.InfoExtractors.YoutubeIE() - fd = FileDownloader(params) - fd.add_info_extractor(ie) - fd.download([TEST_ID]) + ie = youtube_dl.extractor.YoutubeIE() + ydl = YoutubeDL(params) + ydl.add_info_extractor(ie) + ydl.download([TEST_ID]) self.assertTrue(os.path.exists(INFO_JSON_FILE)) with io.open(INFO_JSON_FILE, 'r', encoding='utf-8') as jsonf: jd = json.load(jsonf) diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py index e8b49ff8e..320b44082 100644 --- a/test/test_youtube_lists.py +++ b/test/test_youtube_lists.py @@ -8,9 +8,9 @@ import json import os sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from youtube_dl.InfoExtractors import YoutubeUserIE, YoutubePlaylistIE, YoutubeIE, YoutubeChannelIE +from youtube_dl.extractor import YoutubeUserIE, YoutubePlaylistIE, YoutubeIE, YoutubeChannelIE from youtube_dl.utils import * -from youtube_dl.FileDownloader import FileDownloader +from youtube_dl import YoutubeDL PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json") with io.open(PARAMETERS_FILE, encoding='utf-8') as pf: @@ -23,7 +23,7 @@ proxy_handler = compat_urllib_request.ProxyHandler() opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler()) compat_urllib_request.install_opener(opener) -class FakeDownloader(FileDownloader): +class FakeYDL(YoutubeDL): def __init__(self): self.result = [] self.params = parameters @@ -41,7 +41,7 @@ class TestYoutubeLists(unittest.TestCase): self.assertEqual(info['_type'], 'playlist') def test_youtube_playlist(self): - dl = FakeDownloader() + dl = FakeYDL() ie = YoutubePlaylistIE(dl) result = ie.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')[0] self.assertIsPlaylist(result) @@ -50,13 +50,13 @@ class TestYoutubeLists(unittest.TestCase): self.assertEqual(ytie_results, [ 'bV9L5Ht9LgY', 'FXxLjLQi3Fg', 'tU3Bgo5qJZE']) def test_issue_673(self): - dl = FakeDownloader() + dl = FakeYDL() ie = YoutubePlaylistIE(dl) result = ie.extract('PLBB231211A4F62143')[0] self.assertTrue(len(result['entries']) > 25) def test_youtube_playlist_long(self): - dl = FakeDownloader() + dl = FakeYDL() ie = YoutubePlaylistIE(dl) result = ie.extract('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')[0] self.assertIsPlaylist(result) @@ -64,7 +64,7 @@ class TestYoutubeLists(unittest.TestCase): def test_youtube_playlist_with_deleted(self): #651 - dl = FakeDownloader() + dl = FakeYDL() ie = YoutubePlaylistIE(dl) result = ie.extract('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC')[0] ytie_results = [YoutubeIE()._extract_id(url['url']) for url in result['entries']] @@ -72,14 +72,14 @@ class TestYoutubeLists(unittest.TestCase): self.assertFalse('KdPEApIVdWM' in ytie_results) def test_youtube_playlist_empty(self): - dl = FakeDownloader() + dl = FakeYDL() ie = YoutubePlaylistIE(dl) result = ie.extract('https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx')[0] self.assertIsPlaylist(result) self.assertEqual(len(result['entries']), 0) def test_youtube_course(self): - dl = FakeDownloader() + dl = FakeYDL() ie = YoutubePlaylistIE(dl) # TODO find a > 100 (paginating?) videos course result = ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')[0] @@ -89,7 +89,7 @@ class TestYoutubeLists(unittest.TestCase): self.assertEqual(YoutubeIE()._extract_id(entries[-1]['url']), 'rYefUsYuEp0') def test_youtube_channel(self): - dl = FakeDownloader() + dl = FakeYDL() ie = YoutubeChannelIE(dl) #test paginated channel result = ie.extract('https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w')[0] @@ -99,13 +99,13 @@ class TestYoutubeLists(unittest.TestCase): self.assertTrue(len(result['entries']) >= 18) def test_youtube_user(self): - dl = FakeDownloader() + dl = FakeYDL() ie = YoutubeUserIE(dl) result = ie.extract('https://www.youtube.com/user/TheLinuxFoundation')[0] self.assertTrue(len(result['entries']) >= 320) def test_youtube_safe_search(self): - dl = FakeDownloader() + dl = FakeYDL() ie = YoutubePlaylistIE(dl) result = ie.extract('PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl')[0] self.assertEqual(len(result['entries']), 2) diff --git a/test/test_youtube_subtitles.py b/test/test_youtube_subtitles.py index c80c90cbe..dad15de37 100644 --- a/test/test_youtube_subtitles.py +++ b/test/test_youtube_subtitles.py @@ -10,9 +10,9 @@ import hashlib import os sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from youtube_dl.InfoExtractors import YoutubeIE +from youtube_dl.extractor import YoutubeIE from youtube_dl.utils import * -from youtube_dl import FileDownloader +from youtube_dl import YoutubeDL PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json") with io.open(PARAMETERS_FILE, encoding='utf-8') as pf: @@ -25,7 +25,7 @@ proxy_handler = compat_urllib_request.ProxyHandler() opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler()) compat_urllib_request.install_opener(opener) -class FakeDownloader(FileDownloader): +class FakeYDL(YoutubeDL): def __init__(self): self.result = [] # Different instances of the downloader can't share the same dictionary @@ -42,27 +42,27 @@ md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest() class TestYoutubeSubtitles(unittest.TestCase): def setUp(self): - DL = FakeDownloader() + DL = FakeYDL() DL.params['allsubtitles'] = False DL.params['writesubtitles'] = False DL.params['subtitlesformat'] = 'srt' DL.params['listsubtitles'] = False def test_youtube_no_subtitles(self): - DL = FakeDownloader() + DL = FakeYDL() DL.params['writesubtitles'] = False IE = YoutubeIE(DL) info_dict = IE.extract('QRS8MkLhQmM') subtitles = info_dict[0]['subtitles'] self.assertEqual(subtitles, None) def test_youtube_subtitles(self): - DL = FakeDownloader() + DL = FakeYDL() DL.params['writesubtitles'] = True IE = YoutubeIE(DL) info_dict = IE.extract('QRS8MkLhQmM') sub = info_dict[0]['subtitles'][0] self.assertEqual(md5(sub[2]), '4cd9278a35ba2305f47354ee13472260') def test_youtube_subtitles_it(self): - DL = FakeDownloader() + DL = FakeYDL() DL.params['writesubtitles'] = True DL.params['subtitleslang'] = 'it' IE = YoutubeIE(DL) @@ -70,7 +70,7 @@ class TestYoutubeSubtitles(unittest.TestCase): sub = info_dict[0]['subtitles'][0] self.assertEqual(md5(sub[2]), '164a51f16f260476a05b50fe4c2f161d') def test_youtube_onlysubtitles(self): - DL = FakeDownloader() + DL = FakeYDL() DL.params['writesubtitles'] = True DL.params['onlysubtitles'] = True IE = YoutubeIE(DL) @@ -78,14 +78,14 @@ class TestYoutubeSubtitles(unittest.TestCase): sub = info_dict[0]['subtitles'][0] self.assertEqual(md5(sub[2]), '4cd9278a35ba2305f47354ee13472260') def test_youtube_allsubtitles(self): - DL = FakeDownloader() + DL = FakeYDL() DL.params['allsubtitles'] = True IE = YoutubeIE(DL) info_dict = IE.extract('QRS8MkLhQmM') subtitles = info_dict[0]['subtitles'] self.assertEqual(len(subtitles), 13) def test_youtube_subtitles_format(self): - DL = FakeDownloader() + DL = FakeYDL() DL.params['writesubtitles'] = True DL.params['subtitlesformat'] = 'sbv' IE = YoutubeIE(DL) @@ -93,14 +93,14 @@ class TestYoutubeSubtitles(unittest.TestCase): sub = info_dict[0]['subtitles'][0] self.assertEqual(md5(sub[2]), '13aeaa0c245a8bed9a451cb643e3ad8b') def test_youtube_list_subtitles(self): - DL = FakeDownloader() + DL = FakeYDL() DL.params['listsubtitles'] = True IE = YoutubeIE(DL) info_dict = IE.extract('QRS8MkLhQmM') self.assertEqual(info_dict, None) def test_youtube_automatic_captions(self): - DL = FakeDownloader() - DL.params['writesubtitles'] = True + DL = FakeYDL() + DL.params['writeautomaticsub'] = True DL.params['subtitleslang'] = 'it' IE = YoutubeIE(DL) info_dict = IE.extract('8YoUxe5ncPo') diff --git a/test/tests.json b/test/tests.json index 3e0db297d..5f4f642e8 100644 --- a/test/tests.json +++ b/test/tests.json @@ -11,6 +11,32 @@ "description": "test chars: \"'/\\ä↭𝕐\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de ." } }, + { + "name": "Youtube", + "url": "http://www.youtube.com/watch?v=1ltcDfZMA3U", + "file": "1ltcDfZMA3U.flv", + "note": "Test VEVO video (#897)", + "info_dict": { + "upload_date": "20070518", + "title": "Maps - It Will Find You", + "description": "Music video by Maps performing It Will Find You.", + "uploader": "MuteUSA", + "uploader_id": "MuteUSA" + } + }, + { + "name": "Youtube", + "url": "http://www.youtube.com/watch?v=UxxajLWwzqY", + "file": "UxxajLWwzqY.mp4", + "note": "Test generic use_cipher_signature video (#897)", + "info_dict": { + "upload_date": "20120506", + "title": "Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]", + "description": "md5:b085c9804f5ab69f4adea963a2dceb3c", + "uploader": "IconaPop", + "uploader_id": "IconaPop" + } + }, { "name": "Dailymotion", "md5": "392c4b85a60a90dc4792da41ce3144eb", @@ -630,5 +656,44 @@ "title": "E3 2013: Debut Trailer" }, "skip": "Requires rtmpdump" + }, + { + "name": "Statigram", + "url": "http://statigr.am/p/484091715184808010_284179915", + "file": "484091715184808010_284179915.mp4", + "md5": "deda4ff333abe2e118740321e992605b", + "info_dict": { + "uploader_id": "videoseconds", + "title": "Instagram photo by @videoseconds (Videos)" + } + }, + { + "name": "Break", + "url": "http://www.break.com/video/when-girls-act-like-guys-2468056", + "file": "2468056.mp4", + "md5": "a3513fb1547fba4fb6cfac1bffc6c46b", + "info_dict": { + "title": "When Girls Act Like D-Bags" + } + }, + { + "name": "Vevo", + "url": "http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280", + "file": "GB1101300280.mp4", + "md5": "06bea460acb744eab74a9d7dcb4bfd61", + "info_dict": { + "title": "Somebody To Die For", + "upload_date": "20130624", + "uploader": "Hurts" + } + }, + { + "name": "Tudou", + "url": "http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html", + "file": "159447792.f4v", + "md5": "ad7c358a01541e926a1e413612c6b10a", + "info_dict": { + "title": "卡马乔国足开大脚长传冲吊集锦" + } } ] diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 702cf0e53..57d8b139f 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -1,14 +1,6 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - import math -import io import os import re -import shutil -import socket import subprocess import sys import time @@ -18,103 +10,45 @@ if os.name == 'nt': import ctypes from .utils import * -from .InfoExtractors import get_info_extractor class FileDownloader(object): """File Downloader class. File downloader objects are the ones responsible of downloading the - actual video file and writing it to disk if the user has requested - it, among some other tasks. In most cases there should be one per - program. As, given a video URL, the downloader doesn't know how to - extract all the needed information, task that InfoExtractors do, it - has to pass the URL to one of them. - - For this, file downloader objects have a method that allows - InfoExtractors to be registered in a given order. When it is passed - a URL, the file downloader handles it to the first InfoExtractor it - finds that reports being able to handle it. The InfoExtractor extracts - all the information about the video or videos the URL refers to, and - asks the FileDownloader to process the video information, possibly - downloading the video. + actual video file and writing it to disk. File downloaders accept a lot of parameters. In order not to saturate the object constructor with arguments, it receives a dictionary of - options instead. These options are available through the params - attribute for the InfoExtractors to use. The FileDownloader also - registers itself as the downloader in charge for the InfoExtractors - that are added to it, so this is a "mutual registration". + options instead. Available options: - username: Username for authentication purposes. - password: Password for authentication purposes. - usenetrc: Use netrc for authentication instead. + verbose: Print additional info to stdout. quiet: Do not print messages to stdout. - forceurl: Force printing final URL. - forcetitle: Force printing title. - forceid: Force printing ID. - forcethumbnail: Force printing thumbnail URL. - forcedescription: Force printing description. - forcefilename: Force printing final filename. - simulate: Do not download the video files. - format: Video format code. - format_limit: Highest quality format to try. - outtmpl: Template for output names. - restrictfilenames: Do not allow "&" and spaces in file names - ignoreerrors: Do not stop on download errors. ratelimit: Download speed limit, in bytes/sec. - nooverwrites: Prevent overwriting files. retries: Number of times to retry for HTTP error 5xx buffersize: Size of download buffer in bytes. noresizebuffer: Do not automatically resize the download buffer. continuedl: Try to continue downloads if possible. noprogress: Do not print the progress bar. - playliststart: Playlist item to start at. - playlistend: Playlist item to end at. - matchtitle: Download only matching titles. - rejecttitle: Reject downloads for matching titles. logtostderr: Log messages to stderr instead of stdout. consoletitle: Display progress in console window's titlebar. nopart: Do not use temporary .part files. updatetime: Use the Last-modified header to set output file timestamps. - writedescription: Write the video description to a .description file - writeinfojson: Write the video description to a .info.json file - writethumbnail: Write the thumbnail image to a file - writesubtitles: Write the video subtitles to a file - allsubtitles: Downloads all the subtitles of the video - listsubtitles: Lists all available subtitles for the video - subtitlesformat: Subtitle format [sbv/srt] (default=srt) - subtitleslang: Language of the subtitles to download test: Download only first bytes to test the downloader. - keepvideo: Keep the video file after post-processing min_filesize: Skip files smaller than this size max_filesize: Skip files larger than this size - daterange: A DateRange object, download only if the upload_date is in the range. - skip_download: Skip the actual download of the video file """ params = None - _ies = [] - _pps = [] - _download_retcode = None - _num_downloads = None - _screen_file = None - def __init__(self, params): + def __init__(self, ydl, params): """Create a FileDownloader object with the given options.""" - self._ies = [] - self._pps = [] + self.ydl = ydl self._progress_hooks = [] - self._download_retcode = 0 - self._num_downloads = 0 - self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] self.params = params - if '%(stitle)s' in self.params['outtmpl']: - self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.') - @staticmethod def format_bytes(bytes): if bytes is None: @@ -179,34 +113,11 @@ class FileDownloader(object): multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower()) return int(round(number * multiplier)) - def add_info_extractor(self, ie): - """Add an InfoExtractor object to the end of the list.""" - self._ies.append(ie) - ie.set_downloader(self) - - def add_post_processor(self, pp): - """Add a PostProcessor object to the end of the chain.""" - self._pps.append(pp) - pp.set_downloader(self) - - def to_screen(self, message, skip_eol=False): - """Print message to stdout if not in quiet mode.""" - assert type(message) == type(u'') - if not self.params.get('quiet', False): - terminator = [u'\n', u''][skip_eol] - output = message + terminator - if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr - output = output.encode(preferredencoding(), 'ignore') - self._screen_file.write(output) - self._screen_file.flush() + def to_screen(self, *args, **kargs): + self.ydl.to_screen(*args, **kargs) def to_stderr(self, message): - """Print message to stderr.""" - assert type(message) == type(u'') - output = message + u'\n' - if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr - output = output.encode(preferredencoding()) - sys.stderr.write(output) + self.ydl.to_screen(message) def to_cons_title(self, message): """Set console/terminal window title to message.""" @@ -219,63 +130,14 @@ class FileDownloader(object): elif 'TERM' in os.environ: self.to_screen('\033]0;%s\007' % message, skip_eol=True) - def fixed_template(self): - """Checks if the output template is fixed.""" - return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None) + def trouble(self, *args, **kargs): + self.ydl.trouble(*args, **kargs) - def trouble(self, message=None, tb=None): - """Determine action to take when a download problem appears. + def report_warning(self, *args, **kargs): + self.ydl.report_warning(*args, **kargs) - Depending on if the downloader has been configured to ignore - download errors or not, this method may throw an exception or - not when errors are found, after printing the message. - - tb, if given, is additional traceback information. - """ - if message is not None: - self.to_stderr(message) - if self.params.get('verbose'): - if tb is None: - if sys.exc_info()[0]: # if .trouble has been called from an except block - tb = u'' - if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]: - tb += u''.join(traceback.format_exception(*sys.exc_info()[1].exc_info)) - tb += compat_str(traceback.format_exc()) - else: - tb_data = traceback.format_list(traceback.extract_stack()) - tb = u''.join(tb_data) - self.to_stderr(tb) - if not self.params.get('ignoreerrors', False): - if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]: - exc_info = sys.exc_info()[1].exc_info - else: - exc_info = sys.exc_info() - raise DownloadError(message, exc_info) - self._download_retcode = 1 - - def report_warning(self, message): - ''' - Print the message to stderr, it will be prefixed with 'WARNING:' - If stderr is a tty file the 'WARNING:' will be colored - ''' - if sys.stderr.isatty() and os.name != 'nt': - _msg_header=u'\033[0;33mWARNING:\033[0m' - else: - _msg_header=u'WARNING:' - warning_message=u'%s %s' % (_msg_header,message) - self.to_stderr(warning_message) - - def report_error(self, message, tb=None): - ''' - Do the same as trouble, but prefixes the message with 'ERROR:', colored - in red if stderr is a tty file. - ''' - if sys.stderr.isatty() and os.name != 'nt': - _msg_header = u'\033[0;31mERROR:\033[0m' - else: - _msg_header = u'ERROR:' - error_message = u'%s %s' % (_msg_header, message) - self.trouble(error_message, tb) + def report_error(self, *args, **kargs): + self.ydl.error(*args, **kargs) def slow_down(self, start_time, byte_counter): """Sleep if the download speed is over the rate limit.""" @@ -331,18 +193,6 @@ class FileDownloader(object): pass return filetime - def report_writedescription(self, descfn): - """ Report that the description file is being written """ - self.to_screen(u'[info] Writing video description to: ' + descfn) - - def report_writesubtitles(self, sub_filename): - """ Report that the subtitles file is being written """ - self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename) - - def report_writeinfojson(self, infofn): - """ Report that the metadata file has been written """ - self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn) - def report_destination(self, filename): """Report destination filename.""" self.to_screen(u'[download] Destination: ' + filename) @@ -387,375 +237,6 @@ class FileDownloader(object): else: self.to_screen(u'') - def increment_downloads(self): - """Increment the ordinal that assigns a number to each file.""" - self._num_downloads += 1 - - def prepare_filename(self, info_dict): - """Generate the output filename.""" - try: - template_dict = dict(info_dict) - - template_dict['epoch'] = int(time.time()) - autonumber_size = self.params.get('autonumber_size') - if autonumber_size is None: - autonumber_size = 5 - autonumber_templ = u'%0' + str(autonumber_size) + u'd' - template_dict['autonumber'] = autonumber_templ % self._num_downloads - if template_dict['playlist_index'] is not None: - template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index'] - - sanitize = lambda k,v: sanitize_filename( - u'NA' if v is None else compat_str(v), - restricted=self.params.get('restrictfilenames'), - is_id=(k==u'id')) - template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items()) - - filename = self.params['outtmpl'] % template_dict - return filename - except KeyError as err: - self.report_error(u'Erroneous output template') - return None - except ValueError as err: - self.report_error(u'Insufficient system charset ' + repr(preferredencoding())) - return None - - def _match_entry(self, info_dict): - """ Returns None iff the file should be downloaded """ - - title = info_dict['title'] - matchtitle = self.params.get('matchtitle', False) - if matchtitle: - if not re.search(matchtitle, title, re.IGNORECASE): - return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"' - rejecttitle = self.params.get('rejecttitle', False) - if rejecttitle: - if re.search(rejecttitle, title, re.IGNORECASE): - return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"' - date = info_dict.get('upload_date', None) - if date is not None: - dateRange = self.params.get('daterange', DateRange()) - if date not in dateRange: - return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange) - return None - - def extract_info(self, url, download=True, ie_key=None, extra_info={}): - ''' - Returns a list with a dictionary for each video we find. - If 'download', also downloads the videos. - extra_info is a dict containing the extra values to add to each result - ''' - - if ie_key: - ie = get_info_extractor(ie_key)() - ie.set_downloader(self) - ies = [ie] - else: - ies = self._ies - - for ie in ies: - if not ie.suitable(url): - continue - - if not ie.working(): - self.report_warning(u'The program functionality for this site has been marked as broken, ' - u'and will probably not work.') - - try: - ie_result = ie.extract(url) - if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here) - break - if isinstance(ie_result, list): - # Backwards compatibility: old IE result format - for result in ie_result: - result.update(extra_info) - ie_result = { - '_type': 'compat_list', - 'entries': ie_result, - } - else: - ie_result.update(extra_info) - if 'extractor' not in ie_result: - ie_result['extractor'] = ie.IE_NAME - return self.process_ie_result(ie_result, download=download) - except ExtractorError as de: # An error we somewhat expected - self.report_error(compat_str(de), de.format_traceback()) - break - except Exception as e: - if self.params.get('ignoreerrors', False): - self.report_error(compat_str(e), tb=compat_str(traceback.format_exc())) - break - else: - raise - else: - self.report_error(u'no suitable InfoExtractor: %s' % url) - - def process_ie_result(self, ie_result, download=True, extra_info={}): - """ - Take the result of the ie(may be modified) and resolve all unresolved - references (URLs, playlist items). - - It will also download the videos if 'download'. - Returns the resolved ie_result. - """ - - result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system - if result_type == 'video': - if 'playlist' not in ie_result: - # It isn't part of a playlist - ie_result['playlist'] = None - ie_result['playlist_index'] = None - if download: - self.process_info(ie_result) - return ie_result - elif result_type == 'url': - # We have to add extra_info to the results because it may be - # contained in a playlist - return self.extract_info(ie_result['url'], - download, - ie_key=ie_result.get('ie_key'), - extra_info=extra_info) - elif result_type == 'playlist': - # We process each entry in the playlist - playlist = ie_result.get('title', None) or ie_result.get('id', None) - self.to_screen(u'[download] Downloading playlist: %s' % playlist) - - playlist_results = [] - - n_all_entries = len(ie_result['entries']) - playliststart = self.params.get('playliststart', 1) - 1 - playlistend = self.params.get('playlistend', -1) - - if playlistend == -1: - entries = ie_result['entries'][playliststart:] - else: - entries = ie_result['entries'][playliststart:playlistend] - - n_entries = len(entries) - - self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" % - (ie_result['extractor'], playlist, n_all_entries, n_entries)) - - for i,entry in enumerate(entries,1): - self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries)) - extra = { - 'playlist': playlist, - 'playlist_index': i + playliststart, - } - if not 'extractor' in entry: - # We set the extractor, if it's an url it will be set then to - # the new extractor, but if it's already a video we must make - # sure it's present: see issue #877 - entry['extractor'] = ie_result['extractor'] - entry_result = self.process_ie_result(entry, - download=download, - extra_info=extra) - playlist_results.append(entry_result) - ie_result['entries'] = playlist_results - return ie_result - elif result_type == 'compat_list': - def _fixup(r): - r.setdefault('extractor', ie_result['extractor']) - return r - ie_result['entries'] = [ - self.process_ie_result(_fixup(r), download=download) - for r in ie_result['entries'] - ] - return ie_result - else: - raise Exception('Invalid result type: %s' % result_type) - - def process_info(self, info_dict): - """Process a single resolved IE result.""" - - assert info_dict.get('_type', 'video') == 'video' - #We increment the download the download count here to match the previous behaviour. - self.increment_downloads() - - info_dict['fulltitle'] = info_dict['title'] - if len(info_dict['title']) > 200: - info_dict['title'] = info_dict['title'][:197] + u'...' - - # Keep for backwards compatibility - info_dict['stitle'] = info_dict['title'] - - if not 'format' in info_dict: - info_dict['format'] = info_dict['ext'] - - reason = self._match_entry(info_dict) - if reason is not None: - self.to_screen(u'[download] ' + reason) - return - - max_downloads = self.params.get('max_downloads') - if max_downloads is not None: - if self._num_downloads > int(max_downloads): - raise MaxDownloadsReached() - - filename = self.prepare_filename(info_dict) - - # Forced printings - if self.params.get('forcetitle', False): - compat_print(info_dict['title']) - if self.params.get('forceid', False): - compat_print(info_dict['id']) - if self.params.get('forceurl', False): - compat_print(info_dict['url']) - if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict: - compat_print(info_dict['thumbnail']) - if self.params.get('forcedescription', False) and 'description' in info_dict: - compat_print(info_dict['description']) - if self.params.get('forcefilename', False) and filename is not None: - compat_print(filename) - if self.params.get('forceformat', False): - compat_print(info_dict['format']) - - # Do nothing else if in simulate mode - if self.params.get('simulate', False): - return - - if filename is None: - return - - try: - dn = os.path.dirname(encodeFilename(filename)) - if dn != '' and not os.path.exists(dn): - os.makedirs(dn) - except (OSError, IOError) as err: - self.report_error(u'unable to create directory ' + compat_str(err)) - return - - if self.params.get('writedescription', False): - try: - descfn = filename + u'.description' - self.report_writedescription(descfn) - with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile: - descfile.write(info_dict['description']) - except (OSError, IOError): - self.report_error(u'Cannot write description file ' + descfn) - return - - if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']: - # subtitles download errors are already managed as troubles in relevant IE - # that way it will silently go on when used with unsupporting IE - subtitle = info_dict['subtitles'][0] - (sub_error, sub_lang, sub) = subtitle - sub_format = self.params.get('subtitlesformat') - if sub_error: - self.report_warning("Some error while getting the subtitles") - else: - try: - sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format - self.report_writesubtitles(sub_filename) - with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile: - subfile.write(sub) - except (OSError, IOError): - self.report_error(u'Cannot write subtitles file ' + descfn) - return - - if self.params.get('allsubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']: - subtitles = info_dict['subtitles'] - sub_format = self.params.get('subtitlesformat') - for subtitle in subtitles: - (sub_error, sub_lang, sub) = subtitle - if sub_error: - self.report_warning("Some error while getting the subtitles") - else: - try: - sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format - self.report_writesubtitles(sub_filename) - with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile: - subfile.write(sub) - except (OSError, IOError): - self.report_error(u'Cannot write subtitles file ' + descfn) - return - - if self.params.get('writeinfojson', False): - infofn = filename + u'.info.json' - self.report_writeinfojson(infofn) - try: - json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle']) - write_json_file(json_info_dict, encodeFilename(infofn)) - except (OSError, IOError): - self.report_error(u'Cannot write metadata to JSON file ' + infofn) - return - - if self.params.get('writethumbnail', False): - if 'thumbnail' in info_dict: - thumb_format = info_dict['thumbnail'].rpartition(u'/')[2].rpartition(u'.')[2] - if not thumb_format: - thumb_format = 'jpg' - thumb_filename = filename.rpartition('.')[0] + u'.' + thumb_format - self.to_screen(u'[%s] %s: Downloading thumbnail ...' % - (info_dict['extractor'], info_dict['id'])) - uf = compat_urllib_request.urlopen(info_dict['thumbnail']) - with open(thumb_filename, 'wb') as thumbf: - shutil.copyfileobj(uf, thumbf) - self.to_screen(u'[%s] %s: Writing thumbnail to: %s' % - (info_dict['extractor'], info_dict['id'], thumb_filename)) - - if not self.params.get('skip_download', False): - if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)): - success = True - else: - try: - success = self._do_download(filename, info_dict) - except (OSError, IOError) as err: - raise UnavailableVideoError() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self.report_error(u'unable to download video data: %s' % str(err)) - return - except (ContentTooShortError, ) as err: - self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) - return - - if success: - try: - self.post_process(filename, info_dict) - except (PostProcessingError) as err: - self.report_error(u'postprocessing: %s' % str(err)) - return - - def download(self, url_list): - """Download a given list of URLs.""" - if len(url_list) > 1 and self.fixed_template(): - raise SameFileError(self.params['outtmpl']) - - for url in url_list: - try: - #It also downloads the videos - videos = self.extract_info(url) - except UnavailableVideoError: - self.report_error(u'unable to download video') - except MaxDownloadsReached: - self.to_screen(u'[info] Maximum number of downloaded files reached.') - raise - - return self._download_retcode - - def post_process(self, filename, ie_info): - """Run all the postprocessors on the given file.""" - info = dict(ie_info) - info['filepath'] = filename - keep_video = None - for pp in self._pps: - try: - keep_video_wish,new_info = pp.run(info) - if keep_video_wish is not None: - if keep_video_wish: - keep_video = keep_video_wish - elif keep_video is None: - # No clear decision yet, let IE decide - keep_video = keep_video_wish - except PostProcessingError as e: - self.to_stderr(u'ERROR: ' + e.msg) - if keep_video is False and not self.params.get('keepvideo', False): - try: - self.to_screen(u'Deleting original file %s (pass -k to keep)' % filename) - os.remove(encodeFilename(filename)) - except (IOError, OSError): - self.report_warning(u'Unable to remove downloaded video file') - def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url): self.report_destination(filename) tmpfilename = self.temp_name(filename) diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 619ddeba1..672ef9eed 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -1,4619 +1,4 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- +# Legacy file for backwards compatibility, use youtube_dl.extractor instead! -from __future__ import absolute_import - -import base64 -import datetime -import itertools -import netrc -import os -import re -import socket -import time -import email.utils -import xml.etree.ElementTree -import random -import math -import operator -import hashlib -import binascii -import urllib - -from .utils import * - - -class InfoExtractor(object): - """Information Extractor class. - - Information extractors are the classes that, given a URL, extract - information about the video (or videos) the URL refers to. This - information includes the real video URL, the video title, author and - others. The information is stored in a dictionary which is then - passed to the FileDownloader. The FileDownloader processes this - information possibly downloading the video to the file system, among - other possible outcomes. - - The dictionaries must include the following fields: - - id: Video identifier. - url: Final video URL. - title: Video title, unescaped. - ext: Video filename extension. - - The following fields are optional: - - format: The video format, defaults to ext (used for --get-format) - thumbnail: Full URL to a video thumbnail image. - description: One-line video description. - uploader: Full name of the video uploader. - upload_date: Video upload date (YYYYMMDD). - uploader_id: Nickname or id of the video uploader. - location: Physical location of the video. - player_url: SWF Player URL (used for rtmpdump). - subtitles: The subtitle file contents. - urlhandle: [internal] The urlHandle to be used to download the file, - like returned by urllib.request.urlopen - - The fields should all be Unicode strings. - - Subclasses of this one should re-define the _real_initialize() and - _real_extract() methods and define a _VALID_URL regexp. - Probably, they should also be added to the list of extractors. - - _real_extract() must return a *list* of information dictionaries as - described above. - - Finally, the _WORKING attribute should be set to False for broken IEs - in order to warn the users and skip the tests. - """ - - _ready = False - _downloader = None - _WORKING = True - - def __init__(self, downloader=None): - """Constructor. Receives an optional downloader.""" - self._ready = False - self.set_downloader(downloader) - - @classmethod - def suitable(cls, url): - """Receives a URL and returns True if suitable for this IE.""" - return re.match(cls._VALID_URL, url) is not None - - @classmethod - def working(cls): - """Getter method for _WORKING.""" - return cls._WORKING - - def initialize(self): - """Initializes an instance (authentication, etc).""" - if not self._ready: - self._real_initialize() - self._ready = True - - def extract(self, url): - """Extracts URL information and returns it in list of dicts.""" - self.initialize() - return self._real_extract(url) - - def set_downloader(self, downloader): - """Sets the downloader for this IE.""" - self._downloader = downloader - - def _real_initialize(self): - """Real initialization process. Redefine in subclasses.""" - pass - - def _real_extract(self, url): - """Real extraction process. Redefine in subclasses.""" - pass - - @property - def IE_NAME(self): - return type(self).__name__[:-2] - - def _request_webpage(self, url_or_request, video_id, note=None, errnote=None): - """ Returns the response handle """ - if note is None: - self.report_download_webpage(video_id) - elif note is not False: - self.to_screen(u'%s: %s' % (video_id, note)) - try: - return compat_urllib_request.urlopen(url_or_request) - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - if errnote is None: - errnote = u'Unable to download webpage' - raise ExtractorError(u'%s: %s' % (errnote, compat_str(err)), sys.exc_info()[2]) - - def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None): - """ Returns a tuple (page content as string, URL handle) """ - urlh = self._request_webpage(url_or_request, video_id, note, errnote) - content_type = urlh.headers.get('Content-Type', '') - m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type) - if m: - encoding = m.group(1) - else: - encoding = 'utf-8' - webpage_bytes = urlh.read() - if self._downloader.params.get('dump_intermediate_pages', False): - try: - url = url_or_request.get_full_url() - except AttributeError: - url = url_or_request - self.to_screen(u'Dumping request to ' + url) - dump = base64.b64encode(webpage_bytes).decode('ascii') - self._downloader.to_screen(dump) - content = webpage_bytes.decode(encoding, 'replace') - return (content, urlh) - - def _download_webpage(self, url_or_request, video_id, note=None, errnote=None): - """ Returns the data of the page as a string """ - return self._download_webpage_handle(url_or_request, video_id, note, errnote)[0] - - def to_screen(self, msg): - """Print msg to screen, prefixing it with '[ie_name]'""" - self._downloader.to_screen(u'[%s] %s' % (self.IE_NAME, msg)) - - def report_extraction(self, id_or_name): - """Report information extraction.""" - self.to_screen(u'%s: Extracting information' % id_or_name) - - def report_download_webpage(self, video_id): - """Report webpage download.""" - self.to_screen(u'%s: Downloading webpage' % video_id) - - def report_age_confirmation(self): - """Report attempt to confirm age.""" - self.to_screen(u'Confirming age') - - #Methods for following #608 - #They set the correct value of the '_type' key - def video_result(self, video_info): - """Returns a video""" - video_info['_type'] = 'video' - return video_info - def url_result(self, url, ie=None): - """Returns a url that points to a page that should be processed""" - #TODO: ie should be the class used for getting the info - video_info = {'_type': 'url', - 'url': url, - 'ie_key': ie} - return video_info - def playlist_result(self, entries, playlist_id=None, playlist_title=None): - """Returns a playlist""" - video_info = {'_type': 'playlist', - 'entries': entries} - if playlist_id: - video_info['id'] = playlist_id - if playlist_title: - video_info['title'] = playlist_title - return video_info - - def _search_regex(self, pattern, string, name, default=None, fatal=True, flags=0): - """ - Perform a regex search on the given string, using a single or a list of - patterns returning the first matching group. - In case of failure return a default value or raise a WARNING or a - ExtractorError, depending on fatal, specifying the field name. - """ - if isinstance(pattern, (str, compat_str, compiled_regex_type)): - mobj = re.search(pattern, string, flags) - else: - for p in pattern: - mobj = re.search(p, string, flags) - if mobj: break - - if sys.stderr.isatty() and os.name != 'nt': - _name = u'\033[0;34m%s\033[0m' % name - else: - _name = name - - if mobj: - # return the first matching group - return next(g for g in mobj.groups() if g is not None) - elif default is not None: - return default - elif fatal: - raise ExtractorError(u'Unable to extract %s' % _name) - else: - self._downloader.report_warning(u'unable to extract %s; ' - u'please report this issue on GitHub.' % _name) - return None - - def _html_search_regex(self, pattern, string, name, default=None, fatal=True, flags=0): - """ - Like _search_regex, but strips HTML tags and unescapes entities. - """ - res = self._search_regex(pattern, string, name, default, fatal, flags) - if res: - return clean_html(res).strip() - else: - return res - -class SearchInfoExtractor(InfoExtractor): - """ - Base class for paged search queries extractors. - They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query} - Instances should define _SEARCH_KEY and _MAX_RESULTS. - """ - - @classmethod - def _make_valid_url(cls): - return r'%s(?P|[1-9][0-9]*|all):(?P[\s\S]+)' % cls._SEARCH_KEY - - @classmethod - def suitable(cls, url): - return re.match(cls._make_valid_url(), url) is not None - - def _real_extract(self, query): - mobj = re.match(self._make_valid_url(), query) - if mobj is None: - raise ExtractorError(u'Invalid search query "%s"' % query) - - prefix = mobj.group('prefix') - query = mobj.group('query') - if prefix == '': - return self._get_n_results(query, 1) - elif prefix == 'all': - return self._get_n_results(query, self._MAX_RESULTS) - else: - n = int(prefix) - if n <= 0: - raise ExtractorError(u'invalid download number %s for query "%s"' % (n, query)) - elif n > self._MAX_RESULTS: - self._downloader.report_warning(u'%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n)) - n = self._MAX_RESULTS - return self._get_n_results(query, n) - - def _get_n_results(self, query, n): - """Get a specified number of results for a query""" - raise NotImplementedError("This method must be implemented by sublclasses") - - -class YoutubeIE(InfoExtractor): - """Information extractor for youtube.com.""" - - _VALID_URL = r"""^ - ( - (?:https?://)? # http(s):// (optional) - (?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/| - tube\.majestyc\.net/) # the various hostnames, with wildcard subdomains - (?:.*?\#/)? # handle anchor (#/) redirect urls - (?: # the various things that can precede the ID: - (?:(?:v|embed|e)/) # v/ or embed/ or e/ - |(?: # or the v= param in all its forms - (?:watch(?:_popup)?(?:\.php)?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx) - (?:\?|\#!?) # the params delimiter ? or # or #! - (?:.*?&)? # any other preceding param (like /?s=tuff&v=xxxx) - v= - ) - )? # optional -> youtube.com/xxxx is OK - )? # all until now is optional -> you can pass the naked ID - ([0-9A-Za-z_-]+) # here is it! the YouTube video ID - (?(1).+)? # if we found the ID, everything can follow - $""" - _LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1' - _LOGIN_URL = 'https://accounts.google.com/ServiceLogin' - _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en' - _NEXT_URL_RE = r'[\?&]next_url=([^&]+)' - _NETRC_MACHINE = 'youtube' - # Listed in order of quality - _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13'] - _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13'] - _video_extensions = { - '13': '3gp', - '17': 'mp4', - '18': 'mp4', - '22': 'mp4', - '37': 'mp4', - '38': 'video', # You actually don't know if this will be MOV, AVI or whatever - '43': 'webm', - '44': 'webm', - '45': 'webm', - '46': 'webm', - } - _video_dimensions = { - '5': '240x400', - '6': '???', - '13': '???', - '17': '144x176', - '18': '360x640', - '22': '720x1280', - '34': '360x640', - '35': '480x854', - '37': '1080x1920', - '38': '3072x4096', - '43': '360x640', - '44': '480x854', - '45': '720x1280', - '46': '1080x1920', - } - IE_NAME = u'youtube' - - @classmethod - def suitable(cls, url): - """Receives a URL and returns True if suitable for this IE.""" - if YoutubePlaylistIE.suitable(url): return False - return re.match(cls._VALID_URL, url, re.VERBOSE) is not None - - def report_lang(self): - """Report attempt to set language.""" - self.to_screen(u'Setting language') - - def report_login(self): - """Report attempt to log in.""" - self.to_screen(u'Logging in') - - def report_video_webpage_download(self, video_id): - """Report attempt to download video webpage.""" - self.to_screen(u'%s: Downloading video webpage' % video_id) - - def report_video_info_webpage_download(self, video_id): - """Report attempt to download video info webpage.""" - self.to_screen(u'%s: Downloading video info webpage' % video_id) - - def report_video_subtitles_download(self, video_id): - """Report attempt to download video info webpage.""" - self.to_screen(u'%s: Checking available subtitles' % video_id) - - def report_video_subtitles_request(self, video_id, sub_lang, format): - """Report attempt to download video info webpage.""" - self.to_screen(u'%s: Downloading video subtitles for %s.%s' % (video_id, sub_lang, format)) - - def report_video_subtitles_available(self, video_id, sub_lang_list): - """Report available subtitles.""" - sub_lang = ",".join(list(sub_lang_list.keys())) - self.to_screen(u'%s: Available subtitles for video: %s' % (video_id, sub_lang)) - - def report_information_extraction(self, video_id): - """Report attempt to extract video information.""" - self.to_screen(u'%s: Extracting video information' % video_id) - - def report_unavailable_format(self, video_id, format): - """Report extracted video URL.""" - self.to_screen(u'%s: Format %s not available' % (video_id, format)) - - def report_rtmp_download(self): - """Indicate the download will use the RTMP protocol.""" - self.to_screen(u'RTMP download detected') - - def _get_available_subtitles(self, video_id): - self.report_video_subtitles_download(video_id) - request = compat_urllib_request.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id) - try: - sub_list = compat_urllib_request.urlopen(request).read().decode('utf-8') - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - return (u'unable to download video subtitles: %s' % compat_str(err), None) - sub_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list) - sub_lang_list = dict((l[1], l[0]) for l in sub_lang_list) - if not sub_lang_list: - return (u'video doesn\'t have subtitles', None) - return sub_lang_list - - def _list_available_subtitles(self, video_id): - sub_lang_list = self._get_available_subtitles(video_id) - self.report_video_subtitles_available(video_id, sub_lang_list) - - def _request_subtitle(self, sub_lang, sub_name, video_id, format): - """ - Return tuple: - (error_message, sub_lang, sub) - """ - self.report_video_subtitles_request(video_id, sub_lang, format) - params = compat_urllib_parse.urlencode({ - 'lang': sub_lang, - 'name': sub_name, - 'v': video_id, - 'fmt': format, - }) - url = 'http://www.youtube.com/api/timedtext?' + params - try: - sub = compat_urllib_request.urlopen(url).read().decode('utf-8') - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - return (u'unable to download video subtitles: %s' % compat_str(err), None, None) - if not sub: - return (u'Did not fetch video subtitles', None, None) - return (None, sub_lang, sub) - - def _request_automatic_caption(self, video_id, webpage): - """We need the webpage for getting the captions url, pass it as an - argument to speed up the process.""" - sub_lang = self._downloader.params.get('subtitleslang') or 'en' - sub_format = self._downloader.params.get('subtitlesformat') - self.to_screen(u'%s: Looking for automatic captions' % video_id) - mobj = re.search(r';ytplayer.config = ({.*?});', webpage) - err_msg = u'Couldn\'t find automatic captions for "%s"' % sub_lang - if mobj is None: - return [(err_msg, None, None)] - player_config = json.loads(mobj.group(1)) - try: - args = player_config[u'args'] - caption_url = args[u'ttsurl'] - timestamp = args[u'timestamp'] - params = compat_urllib_parse.urlencode({ - 'lang': 'en', - 'tlang': sub_lang, - 'fmt': sub_format, - 'ts': timestamp, - 'kind': 'asr', - }) - subtitles_url = caption_url + '&' + params - sub = self._download_webpage(subtitles_url, video_id, u'Downloading automatic captions') - return [(None, sub_lang, sub)] - except KeyError: - return [(err_msg, None, None)] - - def _extract_subtitle(self, video_id): - """ - Return a list with a tuple: - [(error_message, sub_lang, sub)] - """ - sub_lang_list = self._get_available_subtitles(video_id) - sub_format = self._downloader.params.get('subtitlesformat') - if isinstance(sub_lang_list,tuple): #There was some error, it didn't get the available subtitles - return [(sub_lang_list[0], None, None)] - if self._downloader.params.get('subtitleslang', False): - sub_lang = self._downloader.params.get('subtitleslang') - elif 'en' in sub_lang_list: - sub_lang = 'en' - else: - sub_lang = list(sub_lang_list.keys())[0] - if not sub_lang in sub_lang_list: - return [(u'no closed captions found in the specified language "%s"' % sub_lang, None, None)] - - subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format) - return [subtitle] - - def _extract_all_subtitles(self, video_id): - sub_lang_list = self._get_available_subtitles(video_id) - sub_format = self._downloader.params.get('subtitlesformat') - if isinstance(sub_lang_list,tuple): #There was some error, it didn't get the available subtitles - return [(sub_lang_list[0], None, None)] - subtitles = [] - for sub_lang in sub_lang_list: - subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format) - subtitles.append(subtitle) - return subtitles - - def _print_formats(self, formats): - print('Available formats:') - for x in formats: - print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???'))) - - def _real_initialize(self): - if self._downloader is None: - return - - username = None - password = None - downloader_params = self._downloader.params - - # Attempt to use provided username and password or .netrc data - if downloader_params.get('username', None) is not None: - username = downloader_params['username'] - password = downloader_params['password'] - elif downloader_params.get('usenetrc', False): - try: - info = netrc.netrc().authenticators(self._NETRC_MACHINE) - if info is not None: - username = info[0] - password = info[2] - else: - raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE) - except (IOError, netrc.NetrcParseError) as err: - self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err)) - return - - # Set language - request = compat_urllib_request.Request(self._LANG_URL) - try: - self.report_lang() - compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.report_warning(u'unable to set language: %s' % compat_str(err)) - return - - # No authentication to be performed - if username is None: - return - - request = compat_urllib_request.Request(self._LOGIN_URL) - try: - login_page = compat_urllib_request.urlopen(request).read().decode('utf-8') - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.report_warning(u'unable to fetch login page: %s' % compat_str(err)) - return - - galx = None - dsh = None - match = re.search(re.compile(r']* id="gaia_loginform"', login_results) is not None: - self._downloader.report_warning(u'unable to log in: bad username or password') - return - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.report_warning(u'unable to log in: %s' % compat_str(err)) - return - - # Confirm age - age_form = { - 'next_url': '/', - 'action_confirm': 'Confirm', - } - request = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form)) - try: - self.report_age_confirmation() - age_results = compat_urllib_request.urlopen(request).read().decode('utf-8') - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to confirm age: %s' % compat_str(err)) - - def _extract_id(self, url): - mobj = re.match(self._VALID_URL, url, re.VERBOSE) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - video_id = mobj.group(2) - return video_id - - def _real_extract(self, url): - # Extract original video URL from URL with redirection, like age verification, using next_url parameter - mobj = re.search(self._NEXT_URL_RE, url) - if mobj: - url = 'https://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/') - video_id = self._extract_id(url) - - # Get video webpage - self.report_video_webpage_download(video_id) - url = 'https://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id - request = compat_urllib_request.Request(url) - try: - video_webpage_bytes = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to download video webpage: %s' % compat_str(err)) - - video_webpage = video_webpage_bytes.decode('utf-8', 'ignore') - - # Attempt to extract SWF player URL - mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage) - if mobj is not None: - player_url = re.sub(r'\\(.)', r'\1', mobj.group(1)) - else: - player_url = None - - # Get video info - self.report_video_info_webpage_download(video_id) - for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']: - video_info_url = ('https://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en' - % (video_id, el_type)) - video_info_webpage = self._download_webpage(video_info_url, video_id, - note=False, - errnote='unable to download video info webpage') - video_info = compat_parse_qs(video_info_webpage) - if 'token' in video_info: - break - if 'token' not in video_info: - if 'reason' in video_info: - raise ExtractorError(u'YouTube said: %s' % video_info['reason'][0]) - else: - raise ExtractorError(u'"token" parameter not in video info for unknown reason') - - # Check for "rental" videos - if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info: - raise ExtractorError(u'"rental" videos not supported') - - # Start extracting information - self.report_information_extraction(video_id) - - # uploader - if 'author' not in video_info: - raise ExtractorError(u'Unable to extract uploader name') - video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0]) - - # uploader_id - video_uploader_id = None - mobj = re.search(r'', video_webpage) - if mobj is not None: - video_uploader_id = mobj.group(1) - else: - self._downloader.report_warning(u'unable to extract uploader nickname') - - # title - if 'title' not in video_info: - raise ExtractorError(u'Unable to extract video title') - video_title = compat_urllib_parse.unquote_plus(video_info['title'][0]) - - # thumbnail image - if 'thumbnail_url' not in video_info: - self._downloader.report_warning(u'unable to extract video thumbnail') - video_thumbnail = '' - else: # don't panic if we can't find it - video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0]) - - # upload date - upload_date = None - mobj = re.search(r'id="eow-date.*?>(.*?)', video_webpage, re.DOTALL) - if mobj is not None: - upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split()) - upload_date = unified_strdate(upload_date) - - # description - video_description = get_element_by_id("eow-description", video_webpage) - if video_description: - video_description = clean_html(video_description) - else: - fd_mobj = re.search(r'= 1: - url_map = {} - for url_data_str in video_info['url_encoded_fmt_stream_map'][0].split(','): - url_data = compat_parse_qs(url_data_str) - if 'itag' in url_data and 'url' in url_data: - url = url_data['url'][0] - if 'sig' in url_data: - url += '&signature=' + url_data['sig'][0] - if 'ratebypass' not in url: - url += '&ratebypass=yes' - url_map[url_data['itag'][0]] = url - - format_limit = self._downloader.params.get('format_limit', None) - available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats - if format_limit is not None and format_limit in available_formats: - format_list = available_formats[available_formats.index(format_limit):] - else: - format_list = available_formats - existing_formats = [x for x in format_list if x in url_map] - if len(existing_formats) == 0: - raise ExtractorError(u'no known formats available for video') - if self._downloader.params.get('listformats', None): - self._print_formats(existing_formats) - return - if req_format is None or req_format == 'best': - video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality - elif req_format == 'worst': - video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality - elif req_format in ('-1', 'all'): - video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats - else: - # Specific formats. We pick the first in a slash-delimeted sequence. - # For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'. - req_formats = req_format.split('/') - video_url_list = None - for rf in req_formats: - if rf in url_map: - video_url_list = [(rf, url_map[rf])] - break - if video_url_list is None: - raise ExtractorError(u'requested format not available') - else: - raise ExtractorError(u'no conn or url_encoded_fmt_stream_map information found in video info') - - results = [] - for format_param, video_real_url in video_url_list: - # Extension - video_extension = self._video_extensions.get(format_param, 'flv') - - video_format = '{0} - {1}'.format(format_param if format_param else video_extension, - self._video_dimensions.get(format_param, '???')) - - results.append({ - 'id': video_id, - 'url': video_real_url, - 'uploader': video_uploader, - 'uploader_id': video_uploader_id, - 'upload_date': upload_date, - 'title': video_title, - 'ext': video_extension, - 'format': video_format, - 'thumbnail': video_thumbnail, - 'description': video_description, - 'player_url': player_url, - 'subtitles': video_subtitles, - 'duration': video_duration - }) - return results - - -class MetacafeIE(InfoExtractor): - """Information Extractor for metacafe.com.""" - - _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*' - _DISCLAIMER = 'http://www.metacafe.com/family_filter/' - _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user' - IE_NAME = u'metacafe' - - def report_disclaimer(self): - """Report disclaimer retrieval.""" - self.to_screen(u'Retrieving disclaimer') - - def _real_initialize(self): - # Retrieve disclaimer - request = compat_urllib_request.Request(self._DISCLAIMER) - try: - self.report_disclaimer() - disclaimer = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to retrieve disclaimer: %s' % compat_str(err)) - - # Confirm age - disclaimer_form = { - 'filters': '0', - 'submit': "Continue - I'm over 18", - } - request = compat_urllib_request.Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form)) - try: - self.report_age_confirmation() - disclaimer = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to confirm age: %s' % compat_str(err)) - - def _real_extract(self, url): - # Extract id and simplified title from URL - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - - video_id = mobj.group(1) - - # Check if video comes from YouTube - mobj2 = re.match(r'^yt-(.*)$', video_id) - if mobj2 is not None: - return [self.url_result('http://www.youtube.com/watch?v=%s' % mobj2.group(1), 'Youtube')] - - # Retrieve video webpage to extract further information - webpage = self._download_webpage('http://www.metacafe.com/watch/%s/' % video_id, video_id) - - # Extract URL, uploader and title from webpage - self.report_extraction(video_id) - mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage) - if mobj is not None: - mediaURL = compat_urllib_parse.unquote(mobj.group(1)) - video_extension = mediaURL[-3:] - - # Extract gdaKey if available - mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage) - if mobj is None: - video_url = mediaURL - else: - gdaKey = mobj.group(1) - video_url = '%s?__gda__=%s' % (mediaURL, gdaKey) - else: - mobj = re.search(r' name="flashvars" value="(.*?)"', webpage) - if mobj is None: - raise ExtractorError(u'Unable to extract media URL') - vardict = compat_parse_qs(mobj.group(1)) - if 'mediaData' not in vardict: - raise ExtractorError(u'Unable to extract media URL') - mobj = re.search(r'"mediaURL":"(?Phttp.*?)",(.*?)"key":"(?P.*?)"', vardict['mediaData'][0]) - if mobj is None: - raise ExtractorError(u'Unable to extract media URL') - mediaURL = mobj.group('mediaURL').replace('\\/', '/') - video_extension = mediaURL[-3:] - video_url = '%s?__gda__=%s' % (mediaURL, mobj.group('key')) - - mobj = re.search(r'(?im)(.*) - Video', webpage) - if mobj is None: - raise ExtractorError(u'Unable to extract title') - video_title = mobj.group(1).decode('utf-8') - - mobj = re.search(r'submitter=(.*?);', webpage) - if mobj is None: - raise ExtractorError(u'Unable to extract uploader nickname') - video_uploader = mobj.group(1) - - return [{ - 'id': video_id.decode('utf-8'), - 'url': video_url.decode('utf-8'), - 'uploader': video_uploader.decode('utf-8'), - 'upload_date': None, - 'title': video_title, - 'ext': video_extension.decode('utf-8'), - }] - -class DailymotionIE(InfoExtractor): - """Information Extractor for Dailymotion""" - - _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^/]+)' - IE_NAME = u'dailymotion' - - def _real_extract(self, url): - # Extract id and simplified title from URL - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - - video_id = mobj.group(1).split('_')[0].split('?')[0] - - video_extension = 'mp4' - - # Retrieve video webpage to extract further information - request = compat_urllib_request.Request(url) - request.add_header('Cookie', 'family_filter=off') - webpage = self._download_webpage(request, video_id) - - # Extract URL, uploader and title from webpage - self.report_extraction(video_id) - mobj = re.search(r'\s*var flashvars = (.*)', webpage) - if mobj is None: - raise ExtractorError(u'Unable to extract media URL') - flashvars = compat_urllib_parse.unquote(mobj.group(1)) - - for key in ['hd1080URL', 'hd720URL', 'hqURL', 'sdURL', 'ldURL', 'video_url']: - if key in flashvars: - max_quality = key - self.to_screen(u'Using %s' % key) - break - else: - raise ExtractorError(u'Unable to extract video URL') - - mobj = re.search(r'"' + max_quality + r'":"(.+?)"', flashvars) - if mobj is None: - raise ExtractorError(u'Unable to extract video URL') - - video_url = compat_urllib_parse.unquote(mobj.group(1)).replace('\\/', '/') - - # TODO: support choosing qualities - - mobj = re.search(r'', webpage) - if mobj is None: - raise ExtractorError(u'Unable to extract title') - video_title = unescapeHTML(mobj.group('title')) - - video_uploader = None - video_uploader = self._search_regex([r'(?im)[^<]+?]+?>([^<]+?)', - # Looking for official user - r'<(?:span|a) .*?rel="author".*?>([^<]+?)([0-9]{2})-([0-9]{2})-([0-9]{4})', webpage) - if mobj is not None: - video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1) - - return [{ - 'id': video_id, - 'url': video_url, - 'uploader': video_uploader, - 'upload_date': video_upload_date, - 'title': video_title, - 'ext': video_extension, - }] - - -class PhotobucketIE(InfoExtractor): - """Information extractor for photobucket.com.""" - - # TODO: the original _VALID_URL was: - # r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)' - # Check if it's necessary to keep the old extracion process - _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*(([\?\&]current=)|_)(?P.*)\.(?P(flv)|(mp4))' - IE_NAME = u'photobucket' - - def _real_extract(self, url): - # Extract id from URL - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - - video_id = mobj.group('id') - - video_extension = mobj.group('ext') - - # Retrieve video webpage to extract further information - webpage = self._download_webpage(url, video_id) - - # Extract URL, uploader, and title from webpage - self.report_extraction(video_id) - # We try first by looking the javascript code: - mobj = re.search(r'Pb\.Data\.Shared\.put\(Pb\.Data\.Shared\.MEDIA, (?P.*?)\);', webpage) - if mobj is not None: - info = json.loads(mobj.group('json')) - return [{ - 'id': video_id, - 'url': info[u'downloadUrl'], - 'uploader': info[u'username'], - 'upload_date': datetime.date.fromtimestamp(info[u'creationDate']).strftime('%Y%m%d'), - 'title': info[u'title'], - 'ext': video_extension, - 'thumbnail': info[u'thumbUrl'], - }] - - # We try looking in other parts of the webpage - video_url = self._search_regex(r'', - webpage, u'video URL') - - mobj = re.search(r'(.*) video by (.*) - Photobucket', webpage) - if mobj is None: - raise ExtractorError(u'Unable to extract title') - video_title = mobj.group(1).decode('utf-8') - video_uploader = mobj.group(2).decode('utf-8') - - return [{ - 'id': video_id.decode('utf-8'), - 'url': video_url.decode('utf-8'), - 'uploader': video_uploader, - 'upload_date': None, - 'title': video_title, - 'ext': video_extension.decode('utf-8'), - }] - - -class YahooIE(InfoExtractor): - """Information extractor for screen.yahoo.com.""" - _VALID_URL = r'http://screen\.yahoo\.com/.*?-(?P\d*?)\.html' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - video_id = mobj.group('id') - webpage = self._download_webpage(url, video_id) - m_id = re.search(r'YUI\.namespace\("Media"\)\.CONTENT_ID = "(?P.+?)";', webpage) - - if m_id is None: - # TODO: Check which url parameters are required - info_url = 'http://cosmos.bcst.yahoo.com/rest/v2/pops;lmsoverride=1;outputformat=mrss;cb=974419660;id=%s;rd=news.yahoo.com;datacontext=mdb;lg=KCa2IihxG3qE60vQ7HtyUy' % video_id - webpage = self._download_webpage(info_url, video_id, u'Downloading info webpage') - info_re = r'''<!\[CDATA\[(?P<title>.*?)\]\]>.* - .*?)\]\]>.* - .*?)\ .*\]\]>.* - https?://)?(?:(?:www|player)\.)?vimeo(?Ppro)?\.com/(?:(?:(?:groups|album)/[^/]+)|(?:.*?)/)?(?Pplay_redirect_hls\?clip_id=)?(?:videos?/)?(?P[0-9]+)' - IE_NAME = u'vimeo' - - def _real_extract(self, url, new_video=True): - # Extract ID from URL - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - - video_id = mobj.group('id') - if not mobj.group('proto'): - url = 'https://' + url - if mobj.group('direct_link') or mobj.group('pro'): - url = 'https://vimeo.com/' + video_id - - # Retrieve video webpage to extract further information - request = compat_urllib_request.Request(url, None, std_headers) - webpage = self._download_webpage(request, video_id) - - # Now we begin extracting as much information as we can from what we - # retrieved. First we extract the information common to all extractors, - # and latter we extract those that are Vimeo specific. - self.report_extraction(video_id) - - # Extract the config JSON - try: - config = webpage.split(' = {config:')[1].split(',assets:')[0] - config = json.loads(config) - except: - if re.search('The creator of this video has not given you permission to embed it on this domain.', webpage): - raise ExtractorError(u'The author has restricted the access to this video, try with the "--referer" option') - else: - raise ExtractorError(u'Unable to extract info section') - - # Extract title - video_title = config["video"]["title"] - - # Extract uploader and uploader_id - video_uploader = config["video"]["owner"]["name"] - video_uploader_id = config["video"]["owner"]["url"].split('/')[-1] if config["video"]["owner"]["url"] else None - - # Extract video thumbnail - video_thumbnail = config["video"]["thumbnail"] - - # Extract video description - video_description = get_element_by_attribute("itemprop", "description", webpage) - if video_description: video_description = clean_html(video_description) - else: video_description = u'' - - # Extract upload date - video_upload_date = None - mobj = re.search(r' 0: - video_quality = files[quality][0][2] - video_codec = files[quality][0][0] - video_extension = files[quality][0][1] - self.to_screen(u'%s: Downloading %s file at %s quality' % (video_id, video_codec.upper(), video_quality)) - break - else: - raise ExtractorError(u'No known codec found') - - video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \ - %(video_id, sig, timestamp, video_quality, video_codec.upper()) - - return [{ - 'id': video_id, - 'url': video_url, - 'uploader': video_uploader, - 'uploader_id': video_uploader_id, - 'upload_date': video_upload_date, - 'title': video_title, - 'ext': video_extension, - 'thumbnail': video_thumbnail, - 'description': video_description, - }] - - -class ArteTvIE(InfoExtractor): - """arte.tv information extractor.""" - - _VALID_URL = r'(?:http://)?videos\.arte\.tv/(?:fr|de)/videos/.*' - _LIVE_URL = r'index-[0-9]+\.html$' - - IE_NAME = u'arte.tv' - - def fetch_webpage(self, url): - request = compat_urllib_request.Request(url) - try: - self.report_download_webpage(url) - webpage = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to retrieve video webpage: %s' % compat_str(err)) - except ValueError as err: - raise ExtractorError(u'Invalid URL: %s' % url) - return webpage - - def grep_webpage(self, url, regex, regexFlags, matchTuples): - page = self.fetch_webpage(url) - mobj = re.search(regex, page, regexFlags) - info = {} - - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - - for (i, key, err) in matchTuples: - if mobj.group(i) is None: - raise ExtractorError(err) - else: - info[key] = mobj.group(i) - - return info - - def extractLiveStream(self, url): - video_lang = url.split('/')[-4] - info = self.grep_webpage( - url, - r'src="(.*?/videothek_js.*?\.js)', - 0, - [ - (1, 'url', u'Invalid URL: %s' % url) - ] - ) - http_host = url.split('/')[2] - next_url = 'http://%s%s' % (http_host, compat_urllib_parse.unquote(info.get('url'))) - info = self.grep_webpage( - next_url, - r'(s_artestras_scst_geoFRDE_' + video_lang + '.*?)\'.*?' + - '(http://.*?\.swf).*?' + - '(rtmp://.*?)\'', - re.DOTALL, - [ - (1, 'path', u'could not extract video path: %s' % url), - (2, 'player', u'could not extract video player: %s' % url), - (3, 'url', u'could not extract video url: %s' % url) - ] - ) - video_url = u'%s/%s' % (info.get('url'), info.get('path')) - - def extractPlus7Stream(self, url): - video_lang = url.split('/')[-3] - info = self.grep_webpage( - url, - r'param name="movie".*?videorefFileUrl=(http[^\'"&]*)', - 0, - [ - (1, 'url', u'Invalid URL: %s' % url) - ] - ) - next_url = compat_urllib_parse.unquote(info.get('url')) - info = self.grep_webpage( - next_url, - r'""" - mobj = re.search(_title, webpage_src) - if mobj is not None: - video_title = mobj.group(1) - - results = [{ - 'id': video_id, - 'url' : video_url, - 'title' : video_title, - 'thumbnail' : thumbnail, - 'ext' : ext, - }] - return results - -class RBMARadioIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?rbmaradio\.com/shows/(?P[^/]+)$' - - def _real_extract(self, url): - m = re.match(self._VALID_URL, url) - video_id = m.group('videoID') - - webpage = self._download_webpage(url, video_id) - - json_data = self._search_regex(r'window\.gon.*?gon\.show=(.+?);$', - webpage, u'json data', flags=re.MULTILINE) - - try: - data = json.loads(json_data) - except ValueError as e: - raise ExtractorError(u'Invalid JSON: ' + str(e)) - - video_url = data['akamai_url'] + '&cbr=256' - url_parts = compat_urllib_parse_urlparse(video_url) - video_ext = url_parts.path.rpartition('.')[2] - info = { - 'id': video_id, - 'url': video_url, - 'ext': video_ext, - 'title': data['title'], - 'description': data.get('teaser_text'), - 'location': data.get('country_of_origin'), - 'uploader': data.get('host', {}).get('name'), - 'uploader_id': data.get('host', {}).get('slug'), - 'thumbnail': data.get('image', {}).get('large_url_2x'), - 'duration': data.get('duration'), - } - return [info] - - -class YouPornIE(InfoExtractor): - """Information extractor for youporn.com.""" - _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youporn\.com/watch/(?P[0-9]+)/(?P[^/]+)' - - def _print_formats(self, formats): - """Print all available formats""" - print(u'Available formats:') - print(u'ext\t\tformat') - print(u'---------------------------------') - for format in formats: - print(u'%s\t\t%s' % (format['ext'], format['format'])) - - def _specific(self, req_format, formats): - for x in formats: - if(x["format"]==req_format): - return x - return None - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - video_id = mobj.group('videoid') - - req = compat_urllib_request.Request(url) - req.add_header('Cookie', 'age_verified=1') - webpage = self._download_webpage(req, video_id) - - # Get JSON parameters - json_params = self._search_regex(r'var currentVideo = new Video\((.*)\);', webpage, u'JSON parameters') - try: - params = json.loads(json_params) - except: - raise ExtractorError(u'Invalid JSON') - - self.report_extraction(video_id) - try: - video_title = params['title'] - upload_date = unified_strdate(params['release_date_f']) - video_description = params['description'] - video_uploader = params['submitted_by'] - thumbnail = params['thumbnails'][0]['image'] - except KeyError: - raise ExtractorError('Missing JSON parameter: ' + sys.exc_info()[1]) - - # Get all of the formats available - DOWNLOAD_LIST_RE = r'(?s)<ul class="downloadList">(?P<download_list>.*?)</ul>' - download_list_html = self._search_regex(DOWNLOAD_LIST_RE, - webpage, u'download list').strip() - - # Get all of the links from the page - LINK_RE = r'(?s)<a href="(?P<url>[^"]+)">' - links = re.findall(LINK_RE, download_list_html) - if(len(links) == 0): - raise ExtractorError(u'ERROR: no known formats available for video') - - self.to_screen(u'Links found: %d' % len(links)) - - formats = [] - for link in links: - - # A link looks like this: - # http://cdn1.download.youporn.phncdn.com/201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4?nvb=20121113051249&nva=20121114051249&ir=1200&sr=1200&hash=014b882080310e95fb6a0 - # A path looks like this: - # /201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4 - video_url = unescapeHTML( link ) - path = compat_urllib_parse_urlparse( video_url ).path - extension = os.path.splitext( path )[1][1:] - format = path.split('/')[4].split('_')[:2] - size = format[0] - bitrate = format[1] - format = "-".join( format ) - # title = u'%s-%s-%s' % (video_title, size, bitrate) - - formats.append({ - 'id': video_id, - 'url': video_url, - 'uploader': video_uploader, - 'upload_date': upload_date, - 'title': video_title, - 'ext': extension, - 'format': format, - 'thumbnail': thumbnail, - 'description': video_description - }) - - if self._downloader.params.get('listformats', None): - self._print_formats(formats) - return - - req_format = self._downloader.params.get('format', None) - self.to_screen(u'Format: %s' % req_format) - - if req_format is None or req_format == 'best': - return [formats[0]] - elif req_format == 'worst': - return [formats[-1]] - elif req_format in ('-1', 'all'): - return formats - else: - format = self._specific( req_format, formats ) - if result is None: - raise ExtractorError(u'Requested format not available') - return [format] - - - -class PornotubeIE(InfoExtractor): - """Information extractor for pornotube.com.""" - _VALID_URL = r'^(?:https?://)?(?:\w+\.)?pornotube\.com(/c/(?P<channel>[0-9]+))?(/m/(?P<videoid>[0-9]+))(/(?P<title>.+))$' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - - video_id = mobj.group('videoid') - video_title = mobj.group('title') - - # Get webpage content - webpage = self._download_webpage(url, video_id) - - # Get the video URL - VIDEO_URL_RE = r'url: "(?P<url>http://video[0-9].pornotube.com/.+\.flv)",' - video_url = self._search_regex(VIDEO_URL_RE, webpage, u'video url') - video_url = compat_urllib_parse.unquote(video_url) - - #Get the uploaded date - VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by' - upload_date = self._html_search_regex(VIDEO_UPLOADED_RE, webpage, u'upload date', fatal=False) - if upload_date: upload_date = unified_strdate(upload_date) - - info = {'id': video_id, - 'url': video_url, - 'uploader': None, - 'upload_date': upload_date, - 'title': video_title, - 'ext': 'flv', - 'format': 'flv'} - - return [info] - -class YouJizzIE(InfoExtractor): - """Information extractor for youjizz.com.""" - _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youjizz\.com/videos/(?P<videoid>[^.]+).html$' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - - video_id = mobj.group('videoid') - - # Get webpage content - webpage = self._download_webpage(url, video_id) - - # Get the video title - video_title = self._html_search_regex(r'<title>(?P<title>.*)', - webpage, u'title').strip() - - # Get the embed page - result = re.search(r'https?://www.youjizz.com/videos/embed/(?P[0-9]+)', webpage) - if result is None: - raise ExtractorError(u'ERROR: unable to extract embed page') - - embed_page_url = result.group(0).strip() - video_id = result.group('videoid') - - webpage = self._download_webpage(embed_page_url, video_id) - - # Get the video URL - video_url = self._search_regex(r'so.addVariable\("file",encodeURIComponent\("(?P[^"]+)"\)\);', - webpage, u'video URL') - - info = {'id': video_id, - 'url': video_url, - 'title': video_title, - 'ext': 'flv', - 'format': 'flv', - 'player_url': embed_page_url} - - return [info] - -class EightTracksIE(InfoExtractor): - IE_NAME = '8tracks' - _VALID_URL = r'https?://8tracks.com/(?P[^/]+)/(?P[^/#]+)(?:#.*)?$' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - playlist_id = mobj.group('id') - - webpage = self._download_webpage(url, playlist_id) - - json_like = self._search_regex(r"PAGE.mix = (.*?);\n", webpage, u'trax information', flags=re.DOTALL) - data = json.loads(json_like) - - session = str(random.randint(0, 1000000000)) - mix_id = data['id'] - track_count = data['tracks_count'] - first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id) - next_url = first_url - res = [] - for i in itertools.count(): - api_json = self._download_webpage(next_url, playlist_id, - note=u'Downloading song information %s/%s' % (str(i+1), track_count), - errnote=u'Failed to download song information') - api_data = json.loads(api_json) - track_data = api_data[u'set']['track'] - info = { - 'id': track_data['id'], - 'url': track_data['track_file_stream_url'], - 'title': track_data['performer'] + u' - ' + track_data['name'], - 'raw_title': track_data['name'], - 'uploader_id': data['user']['login'], - 'ext': 'm4a', - } - res.append(info) - if api_data['set']['at_last_track']: - break - next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % (session, mix_id, track_data['id']) - return res - -class KeekIE(InfoExtractor): - _VALID_URL = r'http://(?:www\.)?keek\.com/(?:!|\w+/keeks/)(?P\w+)' - IE_NAME = u'keek' - - def _real_extract(self, url): - m = re.match(self._VALID_URL, url) - video_id = m.group('videoID') - - video_url = u'http://cdn.keek.com/keek/video/%s' % video_id - thumbnail = u'http://cdn.keek.com/keek/thumbnail/%s/w100/h75' % video_id - webpage = self._download_webpage(url, video_id) - - video_title = self._html_search_regex(r'[\S\s]+?

(?P.+?)

', - webpage, u'uploader', fatal=False) - - info = { - 'id': video_id, - 'url': video_url, - 'ext': 'mp4', - 'title': video_title, - 'thumbnail': thumbnail, - 'uploader': uploader - } - return [info] - -class TEDIE(InfoExtractor): - _VALID_URL=r'''http://www\.ted\.com/ - ( - ((?Pplaylists)/(?P\d+)) # We have a playlist - | - ((?Ptalks)) # We have a simple talk - ) - (/lang/(.*?))? # The url may contain the language - /(?P\w+) # Here goes the name and then ".html" - ''' - - @classmethod - def suitable(cls, url): - """Receives a URL and returns True if suitable for this IE.""" - return re.match(cls._VALID_URL, url, re.VERBOSE) is not None - - def _real_extract(self, url): - m=re.match(self._VALID_URL, url, re.VERBOSE) - if m.group('type_talk'): - return [self._talk_info(url)] - else : - playlist_id=m.group('playlist_id') - name=m.group('name') - self.to_screen(u'Getting info of playlist %s: "%s"' % (playlist_id,name)) - return [self._playlist_videos_info(url,name,playlist_id)] - - def _playlist_videos_info(self,url,name,playlist_id=0): - '''Returns the videos of the playlist''' - video_RE=r''' - (?P.+?)

' - webpage=self._download_webpage(url, playlist_id, 'Downloading playlist webpage') - m_videos=re.finditer(video_RE,webpage,re.VERBOSE) - m_names=re.finditer(video_name_RE,webpage) - - playlist_title = self._html_search_regex(r'div class="headline">\s*?

\s*?(.*?)', - webpage, 'playlist title') - - playlist_entries = [] - for m_video, m_name in zip(m_videos,m_names): - video_id=m_video.group('video_id') - talk_url='http://www.ted.com%s' % m_name.group('talk_url') - playlist_entries.append(self.url_result(talk_url, 'TED')) - return self.playlist_result(playlist_entries, playlist_id = playlist_id, playlist_title = playlist_title) - - def _talk_info(self, url, video_id=0): - """Return the video for the talk in the url""" - m = re.match(self._VALID_URL, url,re.VERBOSE) - video_name = m.group('name') - webpage = self._download_webpage(url, video_id, 'Downloading \"%s\" page' % video_name) - self.report_extraction(video_name) - # If the url includes the language we get the title translated - title = self._html_search_regex(r'(?P.*)</span>', - webpage, 'title') - json_data = self._search_regex(r'<script.*?>var talkDetails = ({.*?})</script>', - webpage, 'json data') - info = json.loads(json_data) - desc = self._html_search_regex(r'<div class="talk-intro">.*?<p.*?>(.*?)</p>', - webpage, 'description', flags = re.DOTALL) - - thumbnail = self._search_regex(r'</span>[\s.]*</div>[\s.]*<img src="(.*?)"', - webpage, 'thumbnail') - info = { - 'id': info['id'], - 'url': info['htmlStreams'][-1]['file'], - 'ext': 'mp4', - 'title': title, - 'thumbnail': thumbnail, - 'description': desc, - } - return info - -class MySpassIE(InfoExtractor): - _VALID_URL = r'http://www.myspass.de/.*' - - def _real_extract(self, url): - META_DATA_URL_TEMPLATE = 'http://www.myspass.de/myspass/includes/apps/video/getvideometadataxml.php?id=%s' - - # video id is the last path element of the URL - # usually there is a trailing slash, so also try the second but last - url_path = compat_urllib_parse_urlparse(url).path - url_parent_path, video_id = os.path.split(url_path) - if not video_id: - _, video_id = os.path.split(url_parent_path) - - # get metadata - metadata_url = META_DATA_URL_TEMPLATE % video_id - metadata_text = self._download_webpage(metadata_url, video_id) - metadata = xml.etree.ElementTree.fromstring(metadata_text.encode('utf-8')) - - # extract values from metadata - url_flv_el = metadata.find('url_flv') - if url_flv_el is None: - raise ExtractorError(u'Unable to extract download url') - video_url = url_flv_el.text - extension = os.path.splitext(video_url)[1][1:] - title_el = metadata.find('title') - if title_el is None: - raise ExtractorError(u'Unable to extract title') - title = title_el.text - format_id_el = metadata.find('format_id') - if format_id_el is None: - format = ext - else: - format = format_id_el.text - description_el = metadata.find('description') - if description_el is not None: - description = description_el.text - else: - description = None - imagePreview_el = metadata.find('imagePreview') - if imagePreview_el is not None: - thumbnail = imagePreview_el.text - else: - thumbnail = None - info = { - 'id': video_id, - 'url': video_url, - 'title': title, - 'ext': extension, - 'format': format, - 'thumbnail': thumbnail, - 'description': description - } - return [info] - -class SpiegelIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<videoID>[0-9]+)(?:\.html)?(?:#.*)?$' - - def _real_extract(self, url): - m = re.match(self._VALID_URL, url) - video_id = m.group('videoID') - - webpage = self._download_webpage(url, video_id) - - video_title = self._html_search_regex(r'<div class="module-title">(.*?)</div>', - webpage, u'title') - - xml_url = u'http://video2.spiegel.de/flash/' + video_id + u'.xml' - xml_code = self._download_webpage(xml_url, video_id, - note=u'Downloading XML', errnote=u'Failed to download XML') - - idoc = xml.etree.ElementTree.fromstring(xml_code) - last_type = idoc[-1] - filename = last_type.findall('./filename')[0].text - duration = float(last_type.findall('./duration')[0].text) - - video_url = 'http://video2.spiegel.de/flash/' + filename - video_ext = filename.rpartition('.')[2] - info = { - 'id': video_id, - 'url': video_url, - 'ext': video_ext, - 'title': video_title, - 'duration': duration, - } - return [info] - -class LiveLeakIE(InfoExtractor): - - _VALID_URL = r'^(?:http?://)?(?:\w+\.)?liveleak\.com/view\?(?:.*?)i=(?P<video_id>[\w_]+)(?:.*)' - IE_NAME = u'liveleak' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - - video_id = mobj.group('video_id') - - webpage = self._download_webpage(url, video_id) - - video_url = self._search_regex(r'file: "(.*?)",', - webpage, u'video URL') - - video_title = self._html_search_regex(r'<meta property="og:title" content="(?P<title>.*?)"', - webpage, u'title').replace('LiveLeak.com -', '').strip() - - video_description = self._html_search_regex(r'<meta property="og:description" content="(?P<desc>.*?)"', - webpage, u'description', fatal=False) - - video_uploader = self._html_search_regex(r'By:.*?(\w+)</a>', - webpage, u'uploader', fatal=False) - - info = { - 'id': video_id, - 'url': video_url, - 'ext': 'mp4', - 'title': video_title, - 'description': video_description, - 'uploader': video_uploader - } - - return [info] - -class ARDIE(InfoExtractor): - _VALID_URL = r'^(?:https?://)?(?:(?:www\.)?ardmediathek\.de|mediathek\.daserste\.de)/(?:.*/)(?P<video_id>[^/\?]+)(?:\?.*)?' - _TITLE = r'<h1(?: class="boxTopHeadline")?>(?P<title>.*)</h1>' - _MEDIA_STREAM = r'mediaCollection\.addMediaStream\((?P<media_type>\d+), (?P<quality>\d+), "(?P<rtmp_url>[^"]*)", "(?P<video_url>[^"]*)", "[^"]*"\)' - - def _real_extract(self, url): - # determine video id from url - m = re.match(self._VALID_URL, url) - - numid = re.search(r'documentId=([0-9]+)', url) - if numid: - video_id = numid.group(1) - else: - video_id = m.group('video_id') - - # determine title and media streams from webpage - html = self._download_webpage(url, video_id) - title = re.search(self._TITLE, html).group('title') - streams = [m.groupdict() for m in re.finditer(self._MEDIA_STREAM, html)] - if not streams: - assert '"fsk"' in html - raise ExtractorError(u'This video is only available after 8:00 pm') - - # choose default media type and highest quality for now - stream = max([s for s in streams if int(s["media_type"]) == 0], - key=lambda s: int(s["quality"])) - - # there's two possibilities: RTMP stream or HTTP download - info = {'id': video_id, 'title': title, 'ext': 'mp4'} - if stream['rtmp_url']: - self.to_screen(u'RTMP download detected') - assert stream['video_url'].startswith('mp4:') - info["url"] = stream["rtmp_url"] - info["play_path"] = stream['video_url'] - else: - assert stream["video_url"].endswith('.mp4') - info["url"] = stream["video_url"] - return [info] - -class ZDFIE(InfoExtractor): - _VALID_URL = r'^http://www\.zdf\.de\/ZDFmediathek\/(.*beitrag\/video\/)(?P<video_id>[^/\?]+)(?:\?.*)?' - _TITLE = r'<h1(?: class="beitragHeadline")?>(?P<title>.*)</h1>' - _MEDIA_STREAM = r'<a href="(?P<video_url>.+(?P<media_type>.streaming).+/zdf/(?P<quality>[^\/]+)/[^"]*)".+class="play".+>' - _MMS_STREAM = r'href="(?P<video_url>mms://[^"]*)"' - _RTSP_STREAM = r'(?P<video_url>rtsp://[^"]*.mp4)' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - video_id = mobj.group('video_id') - - html = self._download_webpage(url, video_id) - streams = [m.groupdict() for m in re.finditer(self._MEDIA_STREAM, html)] - if streams is None: - raise ExtractorError(u'No media url found.') - - # s['media_type'] == 'wstreaming' -> use 'Windows Media Player' and mms url - # s['media_type'] == 'hstreaming' -> use 'Quicktime' and rtsp url - # choose first/default media type and highest quality for now - for s in streams: #find 300 - dsl1000mbit - if s['quality'] == '300' and s['media_type'] == 'wstreaming': - stream_=s - break - for s in streams: #find veryhigh - dsl2000mbit - if s['quality'] == 'veryhigh' and s['media_type'] == 'wstreaming': # 'hstreaming' - rtsp is not working - stream_=s - break - if stream_ is None: - raise ExtractorError(u'No stream found.') - - media_link = self._download_webpage(stream_['video_url'], video_id,'Get stream URL') - - self.report_extraction(video_id) - mobj = re.search(self._TITLE, html) - if mobj is None: - raise ExtractorError(u'Cannot extract title') - title = unescapeHTML(mobj.group('title')) - - mobj = re.search(self._MMS_STREAM, media_link) - if mobj is None: - mobj = re.search(self._RTSP_STREAM, media_link) - if mobj is None: - raise ExtractorError(u'Cannot extract mms:// or rtsp:// URL') - mms_url = mobj.group('video_url') - - mobj = re.search('(.*)[.](?P<ext>[^.]+)', mms_url) - if mobj is None: - raise ExtractorError(u'Cannot extract extention') - ext = mobj.group('ext') - - return [{'id': video_id, - 'url': mms_url, - 'title': title, - 'ext': ext - }] - -class TumblrIE(InfoExtractor): - _VALID_URL = r'http://(?P<blog_name>.*?)\.tumblr\.com/((post)|(video))/(?P<id>\d*)/(.*?)' - - def _real_extract(self, url): - m_url = re.match(self._VALID_URL, url) - video_id = m_url.group('id') - blog = m_url.group('blog_name') - - url = 'http://%s.tumblr.com/post/%s/' % (blog, video_id) - webpage = self._download_webpage(url, video_id) - - re_video = r'src=\\x22(?P<video_url>http://%s\.tumblr\.com/video_file/%s/(.*?))\\x22 type=\\x22video/(?P<ext>.*?)\\x22' % (blog, video_id) - video = re.search(re_video, webpage) - if video is None: - raise ExtractorError(u'Unable to extract video') - video_url = video.group('video_url') - ext = video.group('ext') - - video_thumbnail = self._search_regex(r'posters(.*?)\[\\x22(?P<thumb>.*?)\\x22', - webpage, u'thumbnail', fatal=False) # We pick the first poster - if video_thumbnail: video_thumbnail = video_thumbnail.replace('\\', '') - - # The only place where you can get a title, it's not complete, - # but searching in other places doesn't work for all videos - video_title = self._html_search_regex(r'<title>(?P<title>.*?)', - webpage, u'title', flags=re.DOTALL) - - return [{'id': video_id, - 'url': video_url, - 'title': video_title, - 'thumbnail': video_thumbnail, - 'ext': ext - }] - -class BandcampIE(InfoExtractor): - _VALID_URL = r'http://.*?\.bandcamp\.com/track/(?P.*)' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - title = mobj.group('title') - webpage = self._download_webpage(url, title) - # We get the link to the free download page - m_download = re.search(r'freeDownloadPage: "(.*?)"', webpage) - if m_download is None: - raise ExtractorError(u'No free songs found') - - download_link = m_download.group(1) - id = re.search(r'var TralbumData = {(.*?)id: (?P<id>\d*?)$', - webpage, re.MULTILINE|re.DOTALL).group('id') - - download_webpage = self._download_webpage(download_link, id, - 'Downloading free downloads page') - # We get the dictionary of the track from some javascrip code - info = re.search(r'items: (.*?),$', - download_webpage, re.MULTILINE).group(1) - info = json.loads(info)[0] - # We pick mp3-320 for now, until format selection can be easily implemented. - mp3_info = info[u'downloads'][u'mp3-320'] - # If we try to use this url it says the link has expired - initial_url = mp3_info[u'url'] - re_url = r'(?P<server>http://(.*?)\.bandcamp\.com)/download/track\?enc=mp3-320&fsig=(?P<fsig>.*?)&id=(?P<id>.*?)&ts=(?P<ts>.*)$' - m_url = re.match(re_url, initial_url) - #We build the url we will use to get the final track url - # This url is build in Bandcamp in the script download_bunde_*.js - request_url = '%s/statdownload/track?enc=mp3-320&fsig=%s&id=%s&ts=%s&.rand=665028774616&.vrs=1' % (m_url.group('server'), m_url.group('fsig'), id, m_url.group('ts')) - final_url_webpage = self._download_webpage(request_url, id, 'Requesting download url') - # If we could correctly generate the .rand field the url would be - #in the "download_url" key - final_url = re.search(r'"retry_url":"(.*?)"', final_url_webpage).group(1) - - track_info = {'id':id, - 'title' : info[u'title'], - 'ext' : 'mp3', - 'url' : final_url, - 'thumbnail' : info[u'thumb_url'], - 'uploader' : info[u'artist'] - } - - return [track_info] - -class RedTubeIE(InfoExtractor): - """Information Extractor for redtube""" - _VALID_URL = r'(?:http://)?(?:www\.)?redtube\.com/(?P<id>[0-9]+)' - - def _real_extract(self,url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - - video_id = mobj.group('id') - video_extension = 'mp4' - webpage = self._download_webpage(url, video_id) - - self.report_extraction(video_id) - - video_url = self._html_search_regex(r'<source src="(.+?)" type="video/mp4">', - webpage, u'video URL') - - video_title = self._html_search_regex('<h1 class="videoTitle slidePanelMovable">(.+?)</h1>', - webpage, u'title') - - return [{ - 'id': video_id, - 'url': video_url, - 'ext': video_extension, - 'title': video_title, - }] - -class InaIE(InfoExtractor): - """Information Extractor for Ina.fr""" - _VALID_URL = r'(?:http://)?(?:www\.)?ina\.fr/video/(?P<id>I[0-9]+)/.*' - - def _real_extract(self,url): - mobj = re.match(self._VALID_URL, url) - - video_id = mobj.group('id') - mrss_url='http://player.ina.fr/notices/%s.mrss' % video_id - video_extension = 'mp4' - webpage = self._download_webpage(mrss_url, video_id) - - self.report_extraction(video_id) - - video_url = self._html_search_regex(r'<media:player url="(?P<mp4url>http://mp4.ina.fr/[^"]+\.mp4)', - webpage, u'video URL') - - video_title = self._search_regex(r'<title><!\[CDATA\[(?P<titre>.*?)]]>', - webpage, u'title') - - return [{ - 'id': video_id, - 'url': video_url, - 'ext': video_extension, - 'title': video_title, - }] - -class HowcastIE(InfoExtractor): - """Information Extractor for Howcast.com""" - _VALID_URL = r'(?:https?://)?(?:www\.)?howcast\.com/videos/(?P\d+)' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - - video_id = mobj.group('id') - webpage_url = 'http://www.howcast.com/videos/' + video_id - webpage = self._download_webpage(webpage_url, video_id) - - self.report_extraction(video_id) - - video_url = self._search_regex(r'\'?file\'?: "(http://mobile-media\.howcast\.com/[0-9]+\.mp4)', - webpage, u'video URL') - - video_title = self._html_search_regex(r'\w+)' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - - video_id = mobj.group('id') - webpage_url = 'https://vine.co/v/' + video_id - webpage = self._download_webpage(webpage_url, video_id) - - self.report_extraction(video_id) - - video_url = self._html_search_regex(r'.*?

(.+?)

', - webpage, u'uploader', fatal=False, flags=re.DOTALL) - - return [{ - 'id': video_id, - 'url': video_url, - 'ext': 'mp4', - 'title': video_title, - 'thumbnail': thumbnail, - 'uploader': uploader, - }] - -class FlickrIE(InfoExtractor): - """Information Extractor for Flickr videos""" - _VALID_URL = r'(?:https?://)?(?:www\.)?flickr\.com/photos/(?P[\w\-_@]+)/(?P\d+).*' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - - video_id = mobj.group('id') - video_uploader_id = mobj.group('uploader_id') - webpage_url = 'http://www.flickr.com/photos/' + video_uploader_id + '/' + video_id - webpage = self._download_webpage(webpage_url, video_id) - - secret = self._search_regex(r"photo_secret: '(\w+)'", webpage, u'secret') - - first_url = 'https://secure.flickr.com/apps/video/video_mtl_xml.gne?v=x&photo_id=' + video_id + '&secret=' + secret + '&bitrate=700&target=_self' - first_xml = self._download_webpage(first_url, video_id, 'Downloading first data webpage') - - node_id = self._html_search_regex(r'(\d+-\d+)', - first_xml, u'node_id') - - second_url = 'https://secure.flickr.com/video_playlist.gne?node_id=' + node_id + '&tech=flash&mode=playlist&bitrate=700&secret=' + secret + '&rd=video.yahoo.com&noad=1' - second_xml = self._download_webpage(second_url, video_id, 'Downloading second data webpage') - - self.report_extraction(video_id) - - mobj = re.search(r'.*)' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - url_title = mobj.group('url_title') - webpage = self._download_webpage(url, url_title) - - video_id = self._html_search_regex(r'
(.*?)', - data, u'video URL') - - return [{ - 'id': video_id, - 'url': video_url, - 'ext': 'mp4', - 'title': video_title, - 'thumbnail': thumbnail, - 'description': video_description, - }] - -class XHamsterIE(InfoExtractor): - """Information Extractor for xHamster""" - _VALID_URL = r'(?:http://)?(?:www.)?xhamster\.com/movies/(?P[0-9]+)/.*\.html' - - def _real_extract(self,url): - mobj = re.match(self._VALID_URL, url) - - video_id = mobj.group('id') - mrss_url = 'http://xhamster.com/movies/%s/.html' % video_id - webpage = self._download_webpage(mrss_url, video_id) - - mobj = re.search(r'\'srv\': \'(?P[^\']*)\',\s*\'file\': \'(?P[^\']+)\',', webpage) - if mobj is None: - raise ExtractorError(u'Unable to extract media URL') - if len(mobj.group('server')) == 0: - video_url = compat_urllib_parse.unquote(mobj.group('file')) - else: - video_url = mobj.group('server')+'/key='+mobj.group('file') - video_extension = video_url.split('.')[-1] - - video_title = self._html_search_regex(r'(?P<title>.+?) - xHamster\.com', - webpage, u'title') - - # Can't see the description anywhere in the UI - # video_description = self._html_search_regex(r'Description: (?P[^<]+)', - # webpage, u'description', fatal=False) - # if video_description: video_description = unescapeHTML(video_description) - - mobj = re.search(r'hint=\'(?P[0-9]{4})-(?P[0-9]{2})-(?P[0-9]{2}) [0-9]{2}:[0-9]{2}:[0-9]{2} [A-Z]{3,4}\'', webpage) - if mobj: - video_upload_date = mobj.group('upload_date_Y')+mobj.group('upload_date_m')+mobj.group('upload_date_d') - else: - video_upload_date = None - self._downloader.report_warning(u'Unable to extract upload date') - - video_uploader_id = self._html_search_regex(r']+>(?P[^<]+)', - webpage, u'uploader id', default=u'anonymous') - - video_thumbnail = self._search_regex(r'\'image\':\'(?P[^\']+)\'', - webpage, u'thumbnail', fatal=False) - - return [{ - 'id': video_id, - 'url': video_url, - 'ext': video_extension, - 'title': video_title, - # 'description': video_description, - 'upload_date': video_upload_date, - 'uploader_id': video_uploader_id, - 'thumbnail': video_thumbnail - }] - -class HypemIE(InfoExtractor): - """Information Extractor for hypem""" - _VALID_URL = r'(?:http://)?(?:www\.)?hypem\.com/track/([^/]+)/([^/]+)' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - track_id = mobj.group(1) - - data = { 'ax': 1, 'ts': time.time() } - data_encoded = compat_urllib_parse.urlencode(data) - complete_url = url + "?" + data_encoded - request = compat_urllib_request.Request(complete_url) - response, urlh = self._download_webpage_handle(request, track_id, u'Downloading webpage with the url') - cookie = urlh.headers.get('Set-Cookie', '') - - self.report_extraction(track_id) - - html_tracks = self._html_search_regex(r'', - response, u'tracks', flags=re.MULTILINE|re.DOTALL).strip() - try: - track_list = json.loads(html_tracks) - track = track_list[u'tracks'][0] - except ValueError: - raise ExtractorError(u'Hypemachine contained invalid JSON.') - - key = track[u"key"] - track_id = track[u"id"] - artist = track[u"artist"] - title = track[u"song"] - - serve_url = "http://hypem.com/serve/source/%s/%s" % (compat_str(track_id), compat_str(key)) - request = compat_urllib_request.Request(serve_url, "" , {'Content-Type': 'application/json'}) - request.add_header('cookie', cookie) - song_data_json = self._download_webpage(request, track_id, u'Downloading metadata') - try: - song_data = json.loads(song_data_json) - except ValueError: - raise ExtractorError(u'Hypemachine contained invalid JSON.') - final_url = song_data[u"url"] - - return [{ - 'id': track_id, - 'url': final_url, - 'ext': "mp3", - 'title': title, - 'artist': artist, - }] - -class Vbox7IE(InfoExtractor): - """Information Extractor for Vbox7""" - _VALID_URL = r'(?:http://)?(?:www\.)?vbox7\.com/play:([^/]+)' - - def _real_extract(self,url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - video_id = mobj.group(1) - - redirect_page, urlh = self._download_webpage_handle(url, video_id) - new_location = self._search_regex(r'window\.location = \'(.*)\';', redirect_page, u'redirect location') - redirect_url = urlh.geturl() + new_location - webpage = self._download_webpage(redirect_url, video_id, u'Downloading redirect page') - - title = self._html_search_regex(r'(.*)', - webpage, u'title').split('/')[0].strip() - - ext = "flv" - info_url = "http://vbox7.com/play/magare.do" - data = compat_urllib_parse.urlencode({'as3':'1','vid':video_id}) - info_request = compat_urllib_request.Request(info_url, data) - info_request.add_header('Content-Type', 'application/x-www-form-urlencoded') - info_response = self._download_webpage(info_request, video_id, u'Downloading info webpage') - if info_response is None: - raise ExtractorError(u'Unable to extract the media url') - (final_url, thumbnail_url) = map(lambda x: x.split('=')[1], info_response.split('&')) - - return [{ - 'id': video_id, - 'url': final_url, - 'ext': ext, - 'title': title, - 'thumbnail': thumbnail_url, - }] - -class GametrailersIE(InfoExtractor): - _VALID_URL = r'http://www.gametrailers.com/(?Pvideos|reviews|full-episodes)/(?P.*?)/(?P.*)' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - video_id = mobj.group('id') - video_type = mobj.group('type') - webpage = self._download_webpage(url, video_id) - if video_type == 'full-episodes': - mgid_re = r'data-video="(?P<mgid>mgid:.*?)"' - else: - mgid_re = r'data-contentId=\'(?P<mgid>mgid:.*?)\'' - mgid = self._search_regex(mgid_re, webpage, u'mgid') - data = compat_urllib_parse.urlencode({'uri': mgid, 'acceptMethods': 'fms'}) - - info_page = self._download_webpage('http://www.gametrailers.com/feeds/mrss?' + data, - video_id, u'Downloading video info') - links_webpage = self._download_webpage('http://www.gametrailers.com/feeds/mediagen/?' + data, - video_id, u'Downloading video urls info') - - self.report_extraction(video_id) - info_re = r'''<title><!\[CDATA\[(?P<title>.*?)\]\]>.* - .*?)\]\]>.* - .* - (?P.*?).* - ''' - - m_info = re.search(info_re, info_page, re.VERBOSE|re.DOTALL) - if m_info is None: - raise ExtractorError(u'Unable to extract video info') - video_title = m_info.group('title') - video_description = m_info.group('description') - video_thumb = m_info.group('thumb') - - m_urls = list(re.finditer(r'(?P.*)', links_webpage)) - if m_urls is None or len(m_urls) == 0: - raise ExtractError(u'Unable to extrat video url') - # They are sorted from worst to best quality - video_url = m_urls[-1].group('url') - - return {'url': video_url, - 'id': video_id, - 'title': video_title, - # Videos are actually flv not mp4 - 'ext': 'flv', - 'thumbnail': video_thumb, - 'description': video_description, - } - -def gen_extractors(): - """ Return a list of an instance of every supported extractor. - The order does matter; the first extractor matched is the one handling the URL. - """ - return [ - YoutubePlaylistIE(), - YoutubeChannelIE(), - YoutubeUserIE(), - YoutubeSearchIE(), - YoutubeIE(), - MetacafeIE(), - DailymotionIE(), - GoogleSearchIE(), - PhotobucketIE(), - YahooIE(), - YahooSearchIE(), - DepositFilesIE(), - FacebookIE(), - BlipTVIE(), - BlipTVUserIE(), - VimeoIE(), - MyVideoIE(), - ComedyCentralIE(), - EscapistIE(), - CollegeHumorIE(), - XVideosIE(), - SoundcloudSetIE(), - SoundcloudIE(), - InfoQIE(), - MixcloudIE(), - StanfordOpenClassroomIE(), - MTVIE(), - YoukuIE(), - XNXXIE(), - YouJizzIE(), - PornotubeIE(), - YouPornIE(), - GooglePlusIE(), - ArteTvIE(), - NBAIE(), - WorldStarHipHopIE(), - JustinTVIE(), - FunnyOrDieIE(), - SteamIE(), - UstreamIE(), - RBMARadioIE(), - EightTracksIE(), - KeekIE(), - TEDIE(), - MySpassIE(), - SpiegelIE(), - LiveLeakIE(), - ARDIE(), - ZDFIE(), - TumblrIE(), - BandcampIE(), - RedTubeIE(), - InaIE(), - HowcastIE(), - VineIE(), - FlickrIE(), - TeamcocoIE(), - XHamsterIE(), - HypemIE(), - Vbox7IE(), - GametrailersIE(), - GenericIE() - ] - -def get_info_extractor(ie_name): - """Returns the info extractor class with the given ie_name""" - return globals()[ie_name+'IE'] +from .extractor.common import InfoExtractor, SearchInfoExtractor +from .extractor import gen_extractors, get_info_extractor diff --git a/youtube_dl/PostProcessor.py b/youtube_dl/PostProcessor.py index 8868b37af..8c5e53991 100644 --- a/youtube_dl/PostProcessor.py +++ b/youtube_dl/PostProcessor.py @@ -1,8 +1,3 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - import os import subprocess import sys diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py new file mode 100644 index 000000000..e9a24a95a --- /dev/null +++ b/youtube_dl/YoutubeDL.py @@ -0,0 +1,597 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from __future__ import absolute_import + +import io +import os +import re +import shutil +import socket +import sys +import time +import traceback + +from .utils import * +from .extractor import get_info_extractor +from .FileDownloader import FileDownloader + + +class YoutubeDL(object): + """YoutubeDL class. + + YoutubeDL objects are the ones responsible of downloading the + actual video file and writing it to disk if the user has requested + it, among some other tasks. In most cases there should be one per + program. As, given a video URL, the downloader doesn't know how to + extract all the needed information, task that InfoExtractors do, it + has to pass the URL to one of them. + + For this, YoutubeDL objects have a method that allows + InfoExtractors to be registered in a given order. When it is passed + a URL, the YoutubeDL object handles it to the first InfoExtractor it + finds that reports being able to handle it. The InfoExtractor extracts + all the information about the video or videos the URL refers to, and + YoutubeDL process the extracted information, possibly using a File + Downloader to download the video. + + YoutubeDL objects accept a lot of parameters. In order not to saturate + the object constructor with arguments, it receives a dictionary of + options instead. These options are available through the params + attribute for the InfoExtractors to use. The YoutubeDL also + registers itself as the downloader in charge for the InfoExtractors + that are added to it, so this is a "mutual registration". + + Available options: + + username: Username for authentication purposes. + password: Password for authentication purposes. + videopassword: Password for acces a video. + usenetrc: Use netrc for authentication instead. + verbose: Print additional info to stdout. + quiet: Do not print messages to stdout. + forceurl: Force printing final URL. + forcetitle: Force printing title. + forceid: Force printing ID. + forcethumbnail: Force printing thumbnail URL. + forcedescription: Force printing description. + forcefilename: Force printing final filename. + simulate: Do not download the video files. + format: Video format code. + format_limit: Highest quality format to try. + outtmpl: Template for output names. + restrictfilenames: Do not allow "&" and spaces in file names + ignoreerrors: Do not stop on download errors. + nooverwrites: Prevent overwriting files. + playliststart: Playlist item to start at. + playlistend: Playlist item to end at. + matchtitle: Download only matching titles. + rejecttitle: Reject downloads for matching titles. + logtostderr: Log messages to stderr instead of stdout. + writedescription: Write the video description to a .description file + writeinfojson: Write the video description to a .info.json file + writethumbnail: Write the thumbnail image to a file + writesubtitles: Write the video subtitles to a file + writeautomaticsub: Write the automatic subtitles to a file + allsubtitles: Downloads all the subtitles of the video + listsubtitles: Lists all available subtitles for the video + subtitlesformat: Subtitle format [sbv/srt] (default=srt) + subtitleslang: Language of the subtitles to download + keepvideo: Keep the video file after post-processing + daterange: A DateRange object, download only if the upload_date is in the range. + skip_download: Skip the actual download of the video file + + The following parameters are not used by YoutubeDL itself, they are used by + the FileDownloader: + nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test, + noresizebuffer, retries, continuedl, noprogress, consoletitle + """ + + params = None + _ies = [] + _pps = [] + _download_retcode = None + _num_downloads = None + _screen_file = None + + def __init__(self, params): + """Create a FileDownloader object with the given options.""" + self._ies = [] + self._pps = [] + self._progress_hooks = [] + self._download_retcode = 0 + self._num_downloads = 0 + self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] + self.params = params + self.fd = FileDownloader(self, self.params) + + if '%(stitle)s' in self.params['outtmpl']: + self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.') + + def add_info_extractor(self, ie): + """Add an InfoExtractor object to the end of the list.""" + self._ies.append(ie) + ie.set_downloader(self) + + def add_post_processor(self, pp): + """Add a PostProcessor object to the end of the chain.""" + self._pps.append(pp) + pp.set_downloader(self) + + def to_screen(self, message, skip_eol=False): + """Print message to stdout if not in quiet mode.""" + assert type(message) == type(u'') + if not self.params.get('quiet', False): + terminator = [u'\n', u''][skip_eol] + output = message + terminator + if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr + output = output.encode(preferredencoding(), 'ignore') + self._screen_file.write(output) + self._screen_file.flush() + + def to_stderr(self, message): + """Print message to stderr.""" + assert type(message) == type(u'') + output = message + u'\n' + if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr + output = output.encode(preferredencoding()) + sys.stderr.write(output) + + def fixed_template(self): + """Checks if the output template is fixed.""" + return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None) + + def trouble(self, message=None, tb=None): + """Determine action to take when a download problem appears. + + Depending on if the downloader has been configured to ignore + download errors or not, this method may throw an exception or + not when errors are found, after printing the message. + + tb, if given, is additional traceback information. + """ + if message is not None: + self.to_stderr(message) + if self.params.get('verbose'): + if tb is None: + if sys.exc_info()[0]: # if .trouble has been called from an except block + tb = u'' + if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]: + tb += u''.join(traceback.format_exception(*sys.exc_info()[1].exc_info)) + tb += compat_str(traceback.format_exc()) + else: + tb_data = traceback.format_list(traceback.extract_stack()) + tb = u''.join(tb_data) + self.to_stderr(tb) + if not self.params.get('ignoreerrors', False): + if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]: + exc_info = sys.exc_info()[1].exc_info + else: + exc_info = sys.exc_info() + raise DownloadError(message, exc_info) + self._download_retcode = 1 + + def report_warning(self, message): + ''' + Print the message to stderr, it will be prefixed with 'WARNING:' + If stderr is a tty file the 'WARNING:' will be colored + ''' + if sys.stderr.isatty() and os.name != 'nt': + _msg_header=u'\033[0;33mWARNING:\033[0m' + else: + _msg_header=u'WARNING:' + warning_message=u'%s %s' % (_msg_header,message) + self.to_stderr(warning_message) + + def report_error(self, message, tb=None): + ''' + Do the same as trouble, but prefixes the message with 'ERROR:', colored + in red if stderr is a tty file. + ''' + if sys.stderr.isatty() and os.name != 'nt': + _msg_header = u'\033[0;31mERROR:\033[0m' + else: + _msg_header = u'ERROR:' + error_message = u'%s %s' % (_msg_header, message) + self.trouble(error_message, tb) + + def slow_down(self, start_time, byte_counter): + """Sleep if the download speed is over the rate limit.""" + rate_limit = self.params.get('ratelimit', None) + if rate_limit is None or byte_counter == 0: + return + now = time.time() + elapsed = now - start_time + if elapsed <= 0.0: + return + speed = float(byte_counter) / elapsed + if speed > rate_limit: + time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) + + def report_writedescription(self, descfn): + """ Report that the description file is being written """ + self.to_screen(u'[info] Writing video description to: ' + descfn) + + def report_writesubtitles(self, sub_filename): + """ Report that the subtitles file is being written """ + self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename) + + def report_writeinfojson(self, infofn): + """ Report that the metadata file has been written """ + self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn) + + def report_file_already_downloaded(self, file_name): + """Report file has already been fully downloaded.""" + try: + self.to_screen(u'[download] %s has already been downloaded' % file_name) + except (UnicodeEncodeError) as err: + self.to_screen(u'[download] The file has already been downloaded') + + def increment_downloads(self): + """Increment the ordinal that assigns a number to each file.""" + self._num_downloads += 1 + + def prepare_filename(self, info_dict): + """Generate the output filename.""" + try: + template_dict = dict(info_dict) + + template_dict['epoch'] = int(time.time()) + autonumber_size = self.params.get('autonumber_size') + if autonumber_size is None: + autonumber_size = 5 + autonumber_templ = u'%0' + str(autonumber_size) + u'd' + template_dict['autonumber'] = autonumber_templ % self._num_downloads + if template_dict['playlist_index'] is not None: + template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index'] + + sanitize = lambda k,v: sanitize_filename( + u'NA' if v is None else compat_str(v), + restricted=self.params.get('restrictfilenames'), + is_id=(k==u'id')) + template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items()) + + filename = self.params['outtmpl'] % template_dict + return filename + except KeyError as err: + self.report_error(u'Erroneous output template') + return None + except ValueError as err: + self.report_error(u'Insufficient system charset ' + repr(preferredencoding())) + return None + + def _match_entry(self, info_dict): + """ Returns None iff the file should be downloaded """ + + title = info_dict['title'] + matchtitle = self.params.get('matchtitle', False) + if matchtitle: + if not re.search(matchtitle, title, re.IGNORECASE): + return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"' + rejecttitle = self.params.get('rejecttitle', False) + if rejecttitle: + if re.search(rejecttitle, title, re.IGNORECASE): + return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"' + date = info_dict.get('upload_date', None) + if date is not None: + dateRange = self.params.get('daterange', DateRange()) + if date not in dateRange: + return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange) + return None + + def extract_info(self, url, download=True, ie_key=None, extra_info={}): + ''' + Returns a list with a dictionary for each video we find. + If 'download', also downloads the videos. + extra_info is a dict containing the extra values to add to each result + ''' + + if ie_key: + ie = get_info_extractor(ie_key)() + ie.set_downloader(self) + ies = [ie] + else: + ies = self._ies + + for ie in ies: + if not ie.suitable(url): + continue + + if not ie.working(): + self.report_warning(u'The program functionality for this site has been marked as broken, ' + u'and will probably not work.') + + try: + ie_result = ie.extract(url) + if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here) + break + if isinstance(ie_result, list): + # Backwards compatibility: old IE result format + for result in ie_result: + result.update(extra_info) + ie_result = { + '_type': 'compat_list', + 'entries': ie_result, + } + else: + ie_result.update(extra_info) + if 'extractor' not in ie_result: + ie_result['extractor'] = ie.IE_NAME + return self.process_ie_result(ie_result, download=download) + except ExtractorError as de: # An error we somewhat expected + self.report_error(compat_str(de), de.format_traceback()) + break + except Exception as e: + if self.params.get('ignoreerrors', False): + self.report_error(compat_str(e), tb=compat_str(traceback.format_exc())) + break + else: + raise + else: + self.report_error(u'no suitable InfoExtractor: %s' % url) + + def process_ie_result(self, ie_result, download=True, extra_info={}): + """ + Take the result of the ie(may be modified) and resolve all unresolved + references (URLs, playlist items). + + It will also download the videos if 'download'. + Returns the resolved ie_result. + """ + + result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system + if result_type == 'video': + if 'playlist' not in ie_result: + # It isn't part of a playlist + ie_result['playlist'] = None + ie_result['playlist_index'] = None + if download: + self.process_info(ie_result) + return ie_result + elif result_type == 'url': + # We have to add extra_info to the results because it may be + # contained in a playlist + return self.extract_info(ie_result['url'], + download, + ie_key=ie_result.get('ie_key'), + extra_info=extra_info) + elif result_type == 'playlist': + # We process each entry in the playlist + playlist = ie_result.get('title', None) or ie_result.get('id', None) + self.to_screen(u'[download] Downloading playlist: %s' % playlist) + + playlist_results = [] + + n_all_entries = len(ie_result['entries']) + playliststart = self.params.get('playliststart', 1) - 1 + playlistend = self.params.get('playlistend', -1) + + if playlistend == -1: + entries = ie_result['entries'][playliststart:] + else: + entries = ie_result['entries'][playliststart:playlistend] + + n_entries = len(entries) + + self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" % + (ie_result['extractor'], playlist, n_all_entries, n_entries)) + + for i,entry in enumerate(entries,1): + self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries)) + extra = { + 'playlist': playlist, + 'playlist_index': i + playliststart, + } + if not 'extractor' in entry: + # We set the extractor, if it's an url it will be set then to + # the new extractor, but if it's already a video we must make + # sure it's present: see issue #877 + entry['extractor'] = ie_result['extractor'] + entry_result = self.process_ie_result(entry, + download=download, + extra_info=extra) + playlist_results.append(entry_result) + ie_result['entries'] = playlist_results + return ie_result + elif result_type == 'compat_list': + def _fixup(r): + r.setdefault('extractor', ie_result['extractor']) + return r + ie_result['entries'] = [ + self.process_ie_result(_fixup(r), download=download) + for r in ie_result['entries'] + ] + return ie_result + else: + raise Exception('Invalid result type: %s' % result_type) + + def process_info(self, info_dict): + """Process a single resolved IE result.""" + + assert info_dict.get('_type', 'video') == 'video' + #We increment the download the download count here to match the previous behaviour. + self.increment_downloads() + + info_dict['fulltitle'] = info_dict['title'] + if len(info_dict['title']) > 200: + info_dict['title'] = info_dict['title'][:197] + u'...' + + # Keep for backwards compatibility + info_dict['stitle'] = info_dict['title'] + + if not 'format' in info_dict: + info_dict['format'] = info_dict['ext'] + + reason = self._match_entry(info_dict) + if reason is not None: + self.to_screen(u'[download] ' + reason) + return + + max_downloads = self.params.get('max_downloads') + if max_downloads is not None: + if self._num_downloads > int(max_downloads): + raise MaxDownloadsReached() + + filename = self.prepare_filename(info_dict) + + # Forced printings + if self.params.get('forcetitle', False): + compat_print(info_dict['title']) + if self.params.get('forceid', False): + compat_print(info_dict['id']) + if self.params.get('forceurl', False): + compat_print(info_dict['url']) + if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict: + compat_print(info_dict['thumbnail']) + if self.params.get('forcedescription', False) and 'description' in info_dict: + compat_print(info_dict['description']) + if self.params.get('forcefilename', False) and filename is not None: + compat_print(filename) + if self.params.get('forceformat', False): + compat_print(info_dict['format']) + + # Do nothing else if in simulate mode + if self.params.get('simulate', False): + return + + if filename is None: + return + + try: + dn = os.path.dirname(encodeFilename(filename)) + if dn != '' and not os.path.exists(dn): + os.makedirs(dn) + except (OSError, IOError) as err: + self.report_error(u'unable to create directory ' + compat_str(err)) + return + + if self.params.get('writedescription', False): + try: + descfn = filename + u'.description' + self.report_writedescription(descfn) + with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile: + descfile.write(info_dict['description']) + except (OSError, IOError): + self.report_error(u'Cannot write description file ' + descfn) + return + + if (self.params.get('writesubtitles', False) or self.params.get('writeautomaticsub')) and 'subtitles' in info_dict and info_dict['subtitles']: + # subtitles download errors are already managed as troubles in relevant IE + # that way it will silently go on when used with unsupporting IE + subtitle = info_dict['subtitles'][0] + (sub_error, sub_lang, sub) = subtitle + sub_format = self.params.get('subtitlesformat') + if sub_error: + self.report_warning("Some error while getting the subtitles") + else: + try: + sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format + self.report_writesubtitles(sub_filename) + with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile: + subfile.write(sub) + except (OSError, IOError): + self.report_error(u'Cannot write subtitles file ' + descfn) + return + + if self.params.get('allsubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']: + subtitles = info_dict['subtitles'] + sub_format = self.params.get('subtitlesformat') + for subtitle in subtitles: + (sub_error, sub_lang, sub) = subtitle + if sub_error: + self.report_warning("Some error while getting the subtitles") + else: + try: + sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format + self.report_writesubtitles(sub_filename) + with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile: + subfile.write(sub) + except (OSError, IOError): + self.report_error(u'Cannot write subtitles file ' + descfn) + return + + if self.params.get('writeinfojson', False): + infofn = filename + u'.info.json' + self.report_writeinfojson(infofn) + try: + json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle']) + write_json_file(json_info_dict, encodeFilename(infofn)) + except (OSError, IOError): + self.report_error(u'Cannot write metadata to JSON file ' + infofn) + return + + if self.params.get('writethumbnail', False): + if 'thumbnail' in info_dict: + thumb_format = info_dict['thumbnail'].rpartition(u'/')[2].rpartition(u'.')[2] + if not thumb_format: + thumb_format = 'jpg' + thumb_filename = filename.rpartition('.')[0] + u'.' + thumb_format + self.to_screen(u'[%s] %s: Downloading thumbnail ...' % + (info_dict['extractor'], info_dict['id'])) + uf = compat_urllib_request.urlopen(info_dict['thumbnail']) + with open(thumb_filename, 'wb') as thumbf: + shutil.copyfileobj(uf, thumbf) + self.to_screen(u'[%s] %s: Writing thumbnail to: %s' % + (info_dict['extractor'], info_dict['id'], thumb_filename)) + + if not self.params.get('skip_download', False): + if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)): + success = True + else: + try: + success = self.fd._do_download(filename, info_dict) + except (OSError, IOError) as err: + raise UnavailableVideoError() + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + self.report_error(u'unable to download video data: %s' % str(err)) + return + except (ContentTooShortError, ) as err: + self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) + return + + if success: + try: + self.post_process(filename, info_dict) + except (PostProcessingError) as err: + self.report_error(u'postprocessing: %s' % str(err)) + return + + def download(self, url_list): + """Download a given list of URLs.""" + if len(url_list) > 1 and self.fixed_template(): + raise SameFileError(self.params['outtmpl']) + + for url in url_list: + try: + #It also downloads the videos + videos = self.extract_info(url) + except UnavailableVideoError: + self.report_error(u'unable to download video') + except MaxDownloadsReached: + self.to_screen(u'[info] Maximum number of downloaded files reached.') + raise + + return self._download_retcode + + def post_process(self, filename, ie_info): + """Run all the postprocessors on the given file.""" + info = dict(ie_info) + info['filepath'] = filename + keep_video = None + for pp in self._pps: + try: + keep_video_wish,new_info = pp.run(info) + if keep_video_wish is not None: + if keep_video_wish: + keep_video = keep_video_wish + elif keep_video is None: + # No clear decision yet, let IE decide + keep_video = keep_video_wish + except PostProcessingError as e: + self.to_stderr(u'ERROR: ' + e.msg) + if keep_video is False and not self.params.get('keepvideo', False): + try: + self.to_screen(u'Deleting original file %s (pass -k to keep)' % filename) + os.remove(encodeFilename(filename)) + except (IOError, OSError): + self.report_warning(u'Unable to remove downloaded video file') diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 02442a531..3fd70a750 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -1,9 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import with_statement -from __future__ import absolute_import - __authors__ = ( 'Ricardo Garcia Gonzalez', 'Danny Colligan', @@ -28,6 +25,7 @@ __authors__ = ( 'M. Yasoob Ullah Khalid', 'Julien Fraichard', 'Johny Mo Swag', + 'Axel Noack', ) __license__ = 'Public Domain' @@ -48,7 +46,8 @@ from .utils import * from .update import update_self from .version import __version__ from .FileDownloader import * -from .InfoExtractors import gen_extractors +from .extractor import gen_extractors +from .YoutubeDL import YoutubeDL from .PostProcessor import * def parseOpts(overrideArguments=None): @@ -174,6 +173,8 @@ def parseOpts(overrideArguments=None): dest='password', metavar='PASSWORD', help='account password') authentication.add_option('-n', '--netrc', action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False) + authentication.add_option('--video-password', + dest='videopassword', metavar='PASSWORD', help='video password (vimeo only)') video_format.add_option('-f', '--format', @@ -190,6 +191,9 @@ def parseOpts(overrideArguments=None): video_format.add_option('--write-sub', '--write-srt', action='store_true', dest='writesubtitles', help='write subtitle file (currently youtube only)', default=False) + video_format.add_option('--write-auto-sub', '--write-automatic-sub', + action='store_true', dest='writeautomaticsub', + help='write automatic subtitle file (currently youtube only)', default=False) video_format.add_option('--only-sub', action='store_true', dest='skip_download', help='[deprecated] alias of --skip-download', default=False) @@ -200,7 +204,7 @@ def parseOpts(overrideArguments=None): action='store_true', dest='listsubtitles', help='lists all available subtitles for the video (currently youtube only)', default=False) video_format.add_option('--sub-format', - action='store', dest='subtitlesformat', metavar='LANG', + action='store', dest='subtitlesformat', metavar='FORMAT', help='subtitle format [srt/sbv] (default=srt) (currently youtube only)', default='srt') video_format.add_option('--sub-lang', '--srt-lang', action='store', dest='subtitleslang', metavar='LANG', @@ -321,7 +325,7 @@ def parseOpts(overrideArguments=None): if overrideArguments is not None: opts, args = parser.parse_args(overrideArguments) if opts.verbose: - print(u'[debug] Override config: ' + repr(overrideArguments)) + sys.stderr.write(u'[debug] Override config: ' + repr(overrideArguments) + '\n') else: xdg_config_home = os.environ.get('XDG_CONFIG_HOME') if xdg_config_home: @@ -334,9 +338,9 @@ def parseOpts(overrideArguments=None): argv = systemConf + userConf + commandLineConf opts, args = parser.parse_args(argv) if opts.verbose: - print(u'[debug] System config: ' + repr(systemConf)) - print(u'[debug] User config: ' + repr(userConf)) - print(u'[debug] Command-line args: ' + repr(commandLineConf)) + sys.stderr.write(u'[debug] System config: ' + repr(systemConf) + '\n') + sys.stderr.write(u'[debug] User config: ' + repr(userConf) + '\n') + sys.stderr.write(u'[debug] Command-line args: ' + repr(commandLineConf) + '\n') return parser, opts, args @@ -375,7 +379,7 @@ def _real_main(argv=None): # Dump user agent if opts.dump_user_agent: - print(std_headers['User-Agent']) + compat_print(std_headers['User-Agent']) sys.exit(0) # Batch file verification @@ -416,18 +420,18 @@ def _real_main(argv=None): if opts.list_extractors: for ie in extractors: - print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '')) + compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '')) matchedUrls = [url for url in all_urls if ie.suitable(url)] all_urls = [url for url in all_urls if url not in matchedUrls] for mu in matchedUrls: - print(u' ' + mu) + compat_print(u' ' + mu) sys.exit(0) # Conflicting, missing and erroneous options if opts.usenetrc and (opts.username is not None or opts.password is not None): parser.error(u'using .netrc conflicts with giving username/password') if opts.password is not None and opts.username is None: - parser.error(u'account username missing') + parser.error(u' account username missing\n') if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid): parser.error(u'using output template conflicts with using title, video ID or auto number') if opts.usetitle and opts.useid: @@ -499,11 +503,12 @@ def _real_main(argv=None): or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s') or u'%(title)s-%(id)s.%(ext)s') - # File downloader - fd = FileDownloader({ + # YoutubeDL + ydl = YoutubeDL({ 'usenetrc': opts.usenetrc, 'username': opts.username, 'password': opts.password, + 'videopassword': opts.videopassword, 'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat), 'forceurl': opts.geturl, 'forcetitle': opts.gettitle, @@ -539,6 +544,7 @@ def _real_main(argv=None): 'writeinfojson': opts.writeinfojson, 'writethumbnail': opts.writethumbnail, 'writesubtitles': opts.writesubtitles, + 'writeautomaticsub': opts.writeautomaticsub, 'allsubtitles': opts.allsubtitles, 'listsubtitles': opts.listsubtitles, 'subtitlesformat': opts.subtitlesformat, @@ -557,31 +563,31 @@ def _real_main(argv=None): }) if opts.verbose: - fd.to_screen(u'[debug] youtube-dl version ' + __version__) + ydl.to_screen(u'[debug] youtube-dl version ' + __version__) try: sp = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=os.path.dirname(os.path.abspath(__file__))) out, err = sp.communicate() out = out.decode().strip() if re.match('[0-9a-f]+', out): - fd.to_screen(u'[debug] Git HEAD: ' + out) + ydl.to_screen(u'[debug] Git HEAD: ' + out) except: pass - fd.to_screen(u'[debug] Python version %s - %s' %(platform.python_version(), platform.platform())) - fd.to_screen(u'[debug] Proxy map: ' + str(proxy_handler.proxies)) + ydl.to_screen(u'[debug] Python version %s - %s' %(platform.python_version(), platform.platform())) + ydl.to_screen(u'[debug] Proxy map: ' + str(proxy_handler.proxies)) for extractor in extractors: - fd.add_info_extractor(extractor) + ydl.add_info_extractor(extractor) # PostProcessors if opts.extractaudio: - fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites)) + ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites)) if opts.recodevideo: - fd.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo)) + ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo)) # Update version if opts.update_self: - update_self(fd.to_screen, opts.verbose, sys.argv[0]) + update_self(ydl.to_screen, opts.verbose, sys.argv[0]) # Maybe do nothing if len(all_urls) < 1: @@ -591,9 +597,9 @@ def _real_main(argv=None): sys.exit() try: - retcode = fd.download(all_urls) + retcode = ydl.download(all_urls) except MaxDownloadsReached: - fd.to_screen(u'--max-download limit reached, aborting.') + ydl.to_screen(u'--max-download limit reached, aborting.') retcode = 101 # Dump cookie jar if requested diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py new file mode 100644 index 000000000..0ea990860 --- /dev/null +++ b/youtube_dl/extractor/__init__.py @@ -0,0 +1,140 @@ + +from .ard import ARDIE +from .arte import ArteTvIE +from .bandcamp import BandcampIE +from .bliptv import BlipTVIE, BlipTVUserIE +from .breakcom import BreakIE +from .collegehumor import CollegeHumorIE +from .comedycentral import ComedyCentralIE +from .dailymotion import DailymotionIE +from .depositfiles import DepositFilesIE +from .eighttracks import EightTracksIE +from .escapist import EscapistIE +from .facebook import FacebookIE +from .flickr import FlickrIE +from .funnyordie import FunnyOrDieIE +from .gametrailers import GametrailersIE +from .generic import GenericIE +from .googleplus import GooglePlusIE +from .googlesearch import GoogleSearchIE +from .howcast import HowcastIE +from .hypem import HypemIE +from .ina import InaIE +from .infoq import InfoQIE +from .jukebox import JukeboxIE +from .justintv import JustinTVIE +from .keek import KeekIE +from .liveleak import LiveLeakIE +from .metacafe import MetacafeIE +from .mixcloud import MixcloudIE +from .mtv import MTVIE +from .myspass import MySpassIE +from .myvideo import MyVideoIE +from .nba import NBAIE +from .photobucket import PhotobucketIE +from .pornotube import PornotubeIE +from .rbmaradio import RBMARadioIE +from .redtube import RedTubeIE +from .soundcloud import SoundcloudIE, SoundcloudSetIE +from .spiegel import SpiegelIE +from .stanfordoc import StanfordOpenClassroomIE +from .statigram import StatigramIE +from .steam import SteamIE +from .teamcoco import TeamcocoIE +from .ted import TEDIE +from .tudou import TudouIE +from .tumblr import TumblrIE +from .ustream import UstreamIE +from .vbox7 import Vbox7IE +from .vevo import VevoIE +from .vimeo import VimeoIE +from .vine import VineIE +from .worldstarhiphop import WorldStarHipHopIE +from .xhamster import XHamsterIE +from .xnxx import XNXXIE +from .xvideos import XVideosIE +from .yahoo import YahooIE, YahooSearchIE +from .youjizz import YouJizzIE +from .youku import YoukuIE +from .youporn import YouPornIE +from .youtube import YoutubeIE, YoutubePlaylistIE, YoutubeSearchIE, YoutubeUserIE, YoutubeChannelIE +from .zdf import ZDFIE + + +def gen_extractors(): + """ Return a list of an instance of every supported extractor. + The order does matter; the first extractor matched is the one handling the URL. + """ + return [ + YoutubePlaylistIE(), + YoutubeChannelIE(), + YoutubeUserIE(), + YoutubeSearchIE(), + YoutubeIE(), + MetacafeIE(), + DailymotionIE(), + GoogleSearchIE(), + PhotobucketIE(), + YahooIE(), + YahooSearchIE(), + DepositFilesIE(), + FacebookIE(), + BlipTVIE(), + BlipTVUserIE(), + VimeoIE(), + MyVideoIE(), + ComedyCentralIE(), + EscapistIE(), + CollegeHumorIE(), + XVideosIE(), + SoundcloudSetIE(), + SoundcloudIE(), + InfoQIE(), + MixcloudIE(), + StanfordOpenClassroomIE(), + MTVIE(), + YoukuIE(), + XNXXIE(), + YouJizzIE(), + PornotubeIE(), + YouPornIE(), + GooglePlusIE(), + ArteTvIE(), + NBAIE(), + WorldStarHipHopIE(), + JustinTVIE(), + FunnyOrDieIE(), + SteamIE(), + UstreamIE(), + RBMARadioIE(), + EightTracksIE(), + KeekIE(), + TEDIE(), + MySpassIE(), + SpiegelIE(), + LiveLeakIE(), + ARDIE(), + ZDFIE(), + TumblrIE(), + BandcampIE(), + RedTubeIE(), + InaIE(), + HowcastIE(), + VineIE(), + FlickrIE(), + TeamcocoIE(), + XHamsterIE(), + HypemIE(), + Vbox7IE(), + GametrailersIE(), + StatigramIE(), + BreakIE(), + VevoIE(), + JukeboxIE(), + TudouIE(), + GenericIE() + ] + +def get_info_extractor(ie_name): + """Returns the info extractor class with the given ie_name""" + return globals()[ie_name+'IE'] diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py new file mode 100644 index 000000000..e1ecdf4d3 --- /dev/null +++ b/youtube_dl/extractor/ard.py @@ -0,0 +1,45 @@ +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, +) + +class ARDIE(InfoExtractor): + _VALID_URL = r'^(?:https?://)?(?:(?:www\.)?ardmediathek\.de|mediathek\.daserste\.de)/(?:.*/)(?P[^/\?]+)(?:\?.*)?' + _TITLE = r'(?P.*)</h1>' + _MEDIA_STREAM = r'mediaCollection\.addMediaStream\((?P<media_type>\d+), (?P<quality>\d+), "(?P<rtmp_url>[^"]*)", "(?P<video_url>[^"]*)", "[^"]*"\)' + + def _real_extract(self, url): + # determine video id from url + m = re.match(self._VALID_URL, url) + + numid = re.search(r'documentId=([0-9]+)', url) + if numid: + video_id = numid.group(1) + else: + video_id = m.group('video_id') + + # determine title and media streams from webpage + html = self._download_webpage(url, video_id) + title = re.search(self._TITLE, html).group('title') + streams = [m.groupdict() for m in re.finditer(self._MEDIA_STREAM, html)] + if not streams: + assert '"fsk"' in html + raise ExtractorError(u'This video is only available after 8:00 pm') + + # choose default media type and highest quality for now + stream = max([s for s in streams if int(s["media_type"]) == 0], + key=lambda s: int(s["quality"])) + + # there's two possibilities: RTMP stream or HTTP download + info = {'id': video_id, 'title': title, 'ext': 'mp4'} + if stream['rtmp_url']: + self.to_screen(u'RTMP download detected') + assert stream['video_url'].startswith('mp4:') + info["url"] = stream["rtmp_url"] + info["play_path"] = stream['video_url'] + else: + assert stream["video_url"].endswith('.mp4') + info["url"] = stream["video_url"] + return [info] diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py new file mode 100644 index 000000000..82e3ffe04 --- /dev/null +++ b/youtube_dl/extractor/arte.py @@ -0,0 +1,136 @@ +import re +import socket + +from .common import InfoExtractor +from ..utils import ( + compat_http_client, + compat_str, + compat_urllib_error, + compat_urllib_parse, + compat_urllib_request, + + ExtractorError, + unified_strdate, +) + +class ArteTvIE(InfoExtractor): + """arte.tv information extractor.""" + + _VALID_URL = r'(?:http://)?videos\.arte\.tv/(?:fr|de)/videos/.*' + _LIVE_URL = r'index-[0-9]+\.html$' + + IE_NAME = u'arte.tv' + + def fetch_webpage(self, url): + request = compat_urllib_request.Request(url) + try: + self.report_download_webpage(url) + webpage = compat_urllib_request.urlopen(request).read() + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + raise ExtractorError(u'Unable to retrieve video webpage: %s' % compat_str(err)) + except ValueError as err: + raise ExtractorError(u'Invalid URL: %s' % url) + return webpage + + def grep_webpage(self, url, regex, regexFlags, matchTuples): + page = self.fetch_webpage(url) + mobj = re.search(regex, page, regexFlags) + info = {} + + if mobj is None: + raise ExtractorError(u'Invalid URL: %s' % url) + + for (i, key, err) in matchTuples: + if mobj.group(i) is None: + raise ExtractorError(err) + else: + info[key] = mobj.group(i) + + return info + + # TODO implement Live Stream + # def extractLiveStream(self, url): + # video_lang = url.split('/')[-4] + # info = self.grep_webpage( + # url, + # r'src="(.*?/videothek_js.*?\.js)', + # 0, + # [ + # (1, 'url', u'Invalid URL: %s' % url) + # ] + # ) + # http_host = url.split('/')[2] + # next_url = 'http://%s%s' % (http_host, compat_urllib_parse.unquote(info.get('url'))) + # info = self.grep_webpage( + # next_url, + # r'(s_artestras_scst_geoFRDE_' + video_lang + '.*?)\'.*?' + + # '(http://.*?\.swf).*?' + + # '(rtmp://.*?)\'', + # re.DOTALL, + # [ + # (1, 'path', u'could not extract video path: %s' % url), + # (2, 'player', u'could not extract video player: %s' % url), + # (3, 'url', u'could not extract video url: %s' % url) + # ] + # ) + # video_url = u'%s/%s' % (info.get('url'), info.get('path')) + + def extractPlus7Stream(self, url): + video_lang = url.split('/')[-3] + info = self.grep_webpage( + url, + r'param name="movie".*?videorefFileUrl=(http[^\'"&]*)', + 0, + [ + (1, 'url', u'Invalid URL: %s' % url) + ] + ) + next_url = compat_urllib_parse.unquote(info.get('url')) + info = self.grep_webpage( + next_url, + r'<video lang="%s" ref="(http[^\'"&]*)' % video_lang, + 0, + [ + (1, 'url', u'Could not find <video> tag: %s' % url) + ] + ) + next_url = compat_urllib_parse.unquote(info.get('url')) + + info = self.grep_webpage( + next_url, + r'<video id="(.*?)".*?>.*?' + + '<name>(.*?)</name>.*?' + + '<dateVideo>(.*?)</dateVideo>.*?' + + '<url quality="hd">(.*?)</url>', + re.DOTALL, + [ + (1, 'id', u'could not extract video id: %s' % url), + (2, 'title', u'could not extract video title: %s' % url), + (3, 'date', u'could not extract video date: %s' % url), + (4, 'url', u'could not extract video url: %s' % url) + ] + ) + + return { + 'id': info.get('id'), + 'url': compat_urllib_parse.unquote(info.get('url')), + 'uploader': u'arte.tv', + 'upload_date': unified_strdate(info.get('date')), + 'title': info.get('title').decode('utf-8'), + 'ext': u'mp4', + 'format': u'NA', + 'player_url': None, + } + + def _real_extract(self, url): + video_id = url.split('/')[-1] + self.report_extraction(video_id) + + if re.search(self._LIVE_URL, video_id) is not None: + raise ExtractorError(u'Arte live streams are not yet supported, sorry') + # self.extractLiveStream(url) + # return + else: + info = self.extractPlus7Stream(url) + + return [info] diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py new file mode 100644 index 000000000..dcf6721ee --- /dev/null +++ b/youtube_dl/extractor/bandcamp.py @@ -0,0 +1,54 @@ +import json +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, +) + + +class BandcampIE(InfoExtractor): + _VALID_URL = r'http://.*?\.bandcamp\.com/track/(?P<title>.*)' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + title = mobj.group('title') + webpage = self._download_webpage(url, title) + # We get the link to the free download page + m_download = re.search(r'freeDownloadPage: "(.*?)"', webpage) + if m_download is None: + raise ExtractorError(u'No free songs found') + + download_link = m_download.group(1) + id = re.search(r'var TralbumData = {(.*?)id: (?P<id>\d*?)$', + webpage, re.MULTILINE|re.DOTALL).group('id') + + download_webpage = self._download_webpage(download_link, id, + 'Downloading free downloads page') + # We get the dictionary of the track from some javascrip code + info = re.search(r'items: (.*?),$', + download_webpage, re.MULTILINE).group(1) + info = json.loads(info)[0] + # We pick mp3-320 for now, until format selection can be easily implemented. + mp3_info = info[u'downloads'][u'mp3-320'] + # If we try to use this url it says the link has expired + initial_url = mp3_info[u'url'] + re_url = r'(?P<server>http://(.*?)\.bandcamp\.com)/download/track\?enc=mp3-320&fsig=(?P<fsig>.*?)&id=(?P<id>.*?)&ts=(?P<ts>.*)$' + m_url = re.match(re_url, initial_url) + #We build the url we will use to get the final track url + # This url is build in Bandcamp in the script download_bunde_*.js + request_url = '%s/statdownload/track?enc=mp3-320&fsig=%s&id=%s&ts=%s&.rand=665028774616&.vrs=1' % (m_url.group('server'), m_url.group('fsig'), id, m_url.group('ts')) + final_url_webpage = self._download_webpage(request_url, id, 'Requesting download url') + # If we could correctly generate the .rand field the url would be + #in the "download_url" key + final_url = re.search(r'"retry_url":"(.*?)"', final_url_webpage).group(1) + + track_info = {'id':id, + 'title' : info[u'title'], + 'ext' : 'mp3', + 'url' : final_url, + 'thumbnail' : info[u'thumb_url'], + 'uploader' : info[u'artist'] + } + + return [track_info] diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py new file mode 100644 index 000000000..df2ad4be2 --- /dev/null +++ b/youtube_dl/extractor/bliptv.py @@ -0,0 +1,177 @@ +import datetime +import json +import os +import re +import socket + +from .common import InfoExtractor +from ..utils import ( + compat_http_client, + compat_parse_qs, + compat_str, + compat_urllib_error, + compat_urllib_parse_urlparse, + compat_urllib_request, + + ExtractorError, + unescapeHTML, +) + + +class BlipTVIE(InfoExtractor): + """Information extractor for blip.tv""" + + _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv/((.+/)|(play/)|(api\.swf#))(.+)$' + _URL_EXT = r'^.*\.([a-z0-9]+)$' + IE_NAME = u'blip.tv' + + def report_direct_download(self, title): + """Report information extraction.""" + self.to_screen(u'%s: Direct download detected' % title) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + raise ExtractorError(u'Invalid URL: %s' % url) + + # See https://github.com/rg3/youtube-dl/issues/857 + api_mobj = re.match(r'http://a\.blip\.tv/api\.swf#(?P<video_id>[\d\w]+)', url) + if api_mobj is not None: + url = 'http://blip.tv/play/g_%s' % api_mobj.group('video_id') + urlp = compat_urllib_parse_urlparse(url) + if urlp.path.startswith('/play/'): + request = compat_urllib_request.Request(url) + response = compat_urllib_request.urlopen(request) + redirecturl = response.geturl() + rurlp = compat_urllib_parse_urlparse(redirecturl) + file_id = compat_parse_qs(rurlp.fragment)['file'][0].rpartition('/')[2] + url = 'http://blip.tv/a/a-' + file_id + return self._real_extract(url) + + + if '?' in url: + cchar = '&' + else: + cchar = '?' + json_url = url + cchar + 'skin=json&version=2&no_wrap=1' + request = compat_urllib_request.Request(json_url) + request.add_header('User-Agent', 'iTunes/10.6.1') + self.report_extraction(mobj.group(1)) + info = None + try: + urlh = compat_urllib_request.urlopen(request) + if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download + basename = url.split('/')[-1] + title,ext = os.path.splitext(basename) + title = title.decode('UTF-8') + ext = ext.replace('.', '') + self.report_direct_download(title) + info = { + 'id': title, + 'url': url, + 'uploader': None, + 'upload_date': None, + 'title': title, + 'ext': ext, + 'urlhandle': urlh + } + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + raise ExtractorError(u'ERROR: unable to download video info webpage: %s' % compat_str(err)) + if info is None: # Regular URL + try: + json_code_bytes = urlh.read() + json_code = json_code_bytes.decode('utf-8') + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + raise ExtractorError(u'Unable to read video info webpage: %s' % compat_str(err)) + + try: + json_data = json.loads(json_code) + if 'Post' in json_data: + data = json_data['Post'] + else: + data = json_data + + upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d') + video_url = data['media']['url'] + umobj = re.match(self._URL_EXT, video_url) + if umobj is None: + raise ValueError('Can not determine filename extension') + ext = umobj.group(1) + + info = { + 'id': data['item_id'], + 'url': video_url, + 'uploader': data['display_name'], + 'upload_date': upload_date, + 'title': data['title'], + 'ext': ext, + 'format': data['media']['mimeType'], + 'thumbnail': data['thumbnailUrl'], + 'description': data['description'], + 'player_url': data['embedUrl'], + 'user_agent': 'iTunes/10.6.1', + } + except (ValueError,KeyError) as err: + raise ExtractorError(u'Unable to parse video information: %s' % repr(err)) + + return [info] + + +class BlipTVUserIE(InfoExtractor): + """Information Extractor for blip.tv users.""" + + _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)([^/]+)/*$' + _PAGE_SIZE = 12 + IE_NAME = u'blip.tv:user' + + def _real_extract(self, url): + # Extract username + mobj = re.match(self._VALID_URL, url) + if mobj is None: + raise ExtractorError(u'Invalid URL: %s' % url) + + username = mobj.group(1) + + page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1' + + page = self._download_webpage(url, username, u'Downloading user page') + mobj = re.search(r'data-users-id="([^"]+)"', page) + page_base = page_base % mobj.group(1) + + + # Download video ids using BlipTV Ajax calls. Result size per + # query is limited (currently to 12 videos) so we need to query + # page by page until there are no video ids - it means we got + # all of them. + + video_ids = [] + pagenum = 1 + + while True: + url = page_base + "&page=" + str(pagenum) + page = self._download_webpage(url, username, + u'Downloading video ids from page %d' % pagenum) + + # Extract video identifiers + ids_in_page = [] + + for mobj in re.finditer(r'href="/([^"]+)"', page): + if mobj.group(1) not in ids_in_page: + ids_in_page.append(unescapeHTML(mobj.group(1))) + + video_ids.extend(ids_in_page) + + # A little optimization - if current page is not + # "full", ie. does not contain PAGE_SIZE video ids then + # we can assume that this page is the last one - there + # are no more ids on further pages - no need to query + # again. + + if len(ids_in_page) < self._PAGE_SIZE: + break + + pagenum += 1 + + urls = [u'http://blip.tv/%s' % video_id for video_id in video_ids] + url_entries = [self.url_result(url, 'BlipTV') for url in urls] + return [self.playlist_result(url_entries, playlist_title = username)] diff --git a/youtube_dl/extractor/breakcom.py b/youtube_dl/extractor/breakcom.py new file mode 100644 index 000000000..1f6620d91 --- /dev/null +++ b/youtube_dl/extractor/breakcom.py @@ -0,0 +1,25 @@ +import re + +from .common import InfoExtractor + + +class BreakIE(InfoExtractor): + _VALID_URL = r'(?:http://)?(?:www\.)?break\.com/video/([^/]+)' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group(1).split("-")[-1] + webpage = self._download_webpage(url, video_id) + video_url = re.search(r"videoPath: '(.+?)',",webpage).group(1) + key = re.search(r"icon: '(.+?)',",webpage).group(1) + final_url = str(video_url)+"?"+str(key) + thumbnail_url = re.search(r"thumbnailURL: '(.+?)'",webpage).group(1) + title = re.search(r"sVidTitle: '(.+)',",webpage).group(1) + ext = video_url.split('.')[-1] + return [{ + 'id': video_id, + 'url': final_url, + 'ext': ext, + 'title': title, + 'thumbnail': thumbnail_url, + }] diff --git a/youtube_dl/extractor/collegehumor.py b/youtube_dl/extractor/collegehumor.py new file mode 100644 index 000000000..7ae0972e5 --- /dev/null +++ b/youtube_dl/extractor/collegehumor.py @@ -0,0 +1,74 @@ +import re +import socket +import xml.etree.ElementTree + +from .common import InfoExtractor +from ..utils import ( + compat_http_client, + compat_str, + compat_urllib_error, + compat_urllib_parse_urlparse, + compat_urllib_request, + + ExtractorError, +) + + +class CollegeHumorIE(InfoExtractor): + _WORKING = False + _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/video/(?P<videoid>[0-9]+)/(?P<shorttitle>.*)$' + + def report_manifest(self, video_id): + """Report information extraction.""" + self.to_screen(u'%s: Downloading XML manifest' % video_id) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + raise ExtractorError(u'Invalid URL: %s' % url) + video_id = mobj.group('videoid') + + info = { + 'id': video_id, + 'uploader': None, + 'upload_date': None, + } + + self.report_extraction(video_id) + xmlUrl = 'http://www.collegehumor.com/moogaloop/video/' + video_id + try: + metaXml = compat_urllib_request.urlopen(xmlUrl).read() + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + raise ExtractorError(u'Unable to download video info XML: %s' % compat_str(err)) + + mdoc = xml.etree.ElementTree.fromstring(metaXml) + try: + videoNode = mdoc.findall('./video')[0] + info['description'] = videoNode.findall('./description')[0].text + info['title'] = videoNode.findall('./caption')[0].text + info['thumbnail'] = videoNode.findall('./thumbnail')[0].text + manifest_url = videoNode.findall('./file')[0].text + except IndexError: + raise ExtractorError(u'Invalid metadata XML file') + + manifest_url += '?hdcore=2.10.3' + self.report_manifest(video_id) + try: + manifestXml = compat_urllib_request.urlopen(manifest_url).read() + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + raise ExtractorError(u'Unable to download video info XML: %s' % compat_str(err)) + + adoc = xml.etree.ElementTree.fromstring(manifestXml) + try: + media_node = adoc.findall('./{http://ns.adobe.com/f4m/1.0}media')[0] + node_id = media_node.attrib['url'] + video_id = adoc.findall('./{http://ns.adobe.com/f4m/1.0}id')[0].text + except IndexError as err: + raise ExtractorError(u'Invalid manifest file') + + url_pr = compat_urllib_parse_urlparse(manifest_url) + url = url_pr.scheme + '://' + url_pr.netloc + '/z' + video_id[:-2] + '/' + node_id + 'Seg1-Frag1' + + info['url'] = url + info['ext'] = 'f4f' + return [info] diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py new file mode 100644 index 000000000..1bb359046 --- /dev/null +++ b/youtube_dl/extractor/comedycentral.py @@ -0,0 +1,179 @@ +import re +import xml.etree.ElementTree + +from .common import InfoExtractor +from ..utils import ( + compat_str, + compat_urllib_parse, + + ExtractorError, + unified_strdate, +) + + +class ComedyCentralIE(InfoExtractor): + """Information extractor for The Daily Show and Colbert Report """ + + # urls can be abbreviations like :thedailyshow or :colbert + # urls for episodes like: + # or urls for clips like: http://www.thedailyshow.com/watch/mon-december-10-2012/any-given-gun-day + # or: http://www.colbertnation.com/the-colbert-report-videos/421667/november-29-2012/moon-shattering-news + # or: http://www.colbertnation.com/the-colbert-report-collections/422008/festival-of-lights/79524 + _VALID_URL = r"""^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport) + |(https?://)?(www\.)? + (?P<showname>thedailyshow|colbertnation)\.com/ + (full-episodes/(?P<episode>.*)| + (?P<clip> + (the-colbert-report-(videos|collections)/(?P<clipID>[0-9]+)/[^/]*/(?P<cntitle>.*?)) + |(watch/(?P<date>[^/]*)/(?P<tdstitle>.*))))) + $""" + + _available_formats = ['3500', '2200', '1700', '1200', '750', '400'] + + _video_extensions = { + '3500': 'mp4', + '2200': 'mp4', + '1700': 'mp4', + '1200': 'mp4', + '750': 'mp4', + '400': 'mp4', + } + _video_dimensions = { + '3500': '1280x720', + '2200': '960x540', + '1700': '768x432', + '1200': '640x360', + '750': '512x288', + '400': '384x216', + } + + @classmethod + def suitable(cls, url): + """Receives a URL and returns True if suitable for this IE.""" + return re.match(cls._VALID_URL, url, re.VERBOSE) is not None + + def _print_formats(self, formats): + print('Available formats:') + for x in formats: + print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'mp4'), self._video_dimensions.get(x, '???'))) + + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url, re.VERBOSE) + if mobj is None: + raise ExtractorError(u'Invalid URL: %s' % url) + + if mobj.group('shortname'): + if mobj.group('shortname') in ('tds', 'thedailyshow'): + url = u'http://www.thedailyshow.com/full-episodes/' + else: + url = u'http://www.colbertnation.com/full-episodes/' + mobj = re.match(self._VALID_URL, url, re.VERBOSE) + assert mobj is not None + + if mobj.group('clip'): + if mobj.group('showname') == 'thedailyshow': + epTitle = mobj.group('tdstitle') + else: + epTitle = mobj.group('cntitle') + dlNewest = False + else: + dlNewest = not mobj.group('episode') + if dlNewest: + epTitle = mobj.group('showname') + else: + epTitle = mobj.group('episode') + + self.report_extraction(epTitle) + webpage,htmlHandle = self._download_webpage_handle(url, epTitle) + if dlNewest: + url = htmlHandle.geturl() + mobj = re.match(self._VALID_URL, url, re.VERBOSE) + if mobj is None: + raise ExtractorError(u'Invalid redirected URL: ' + url) + if mobj.group('episode') == '': + raise ExtractorError(u'Redirected URL is still not specific: ' + url) + epTitle = mobj.group('episode') + + mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*(?:episode|video).*?:.*?))"', webpage) + + if len(mMovieParams) == 0: + # The Colbert Report embeds the information in a without + # a URL prefix; so extract the alternate reference + # and then add the URL prefix manually. + + altMovieParams = re.findall('data-mgid="([^"]*(?:episode|video).*?:.*?)"', webpage) + if len(altMovieParams) == 0: + raise ExtractorError(u'unable to find Flash URL in webpage ' + url) + else: + mMovieParams = [("http://media.mtvnservices.com/" + altMovieParams[0], altMovieParams[0])] + + uri = mMovieParams[0][1] + indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + compat_urllib_parse.urlencode({'uri': uri}) + indexXml = self._download_webpage(indexUrl, epTitle, + u'Downloading show index', + u'unable to download episode index') + + results = [] + + idoc = xml.etree.ElementTree.fromstring(indexXml) + itemEls = idoc.findall('.//item') + for partNum,itemEl in enumerate(itemEls): + mediaId = itemEl.findall('./guid')[0].text + shortMediaId = mediaId.split(':')[-1] + showId = mediaId.split(':')[-2].replace('.com', '') + officialTitle = itemEl.findall('./title')[0].text + officialDate = unified_strdate(itemEl.findall('./pubDate')[0].text) + + configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' + + compat_urllib_parse.urlencode({'uri': mediaId})) + configXml = self._download_webpage(configUrl, epTitle, + u'Downloading configuration for %s' % shortMediaId) + + cdoc = xml.etree.ElementTree.fromstring(configXml) + turls = [] + for rendition in cdoc.findall('.//rendition'): + finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text) + turls.append(finfo) + + if len(turls) == 0: + self._downloader.report_error(u'unable to download ' + mediaId + ': No videos found') + continue + + if self._downloader.params.get('listformats', None): + self._print_formats([i[0] for i in turls]) + return + + # For now, just pick the highest bitrate + format,rtmp_video_url = turls[-1] + + # Get the format arg from the arg stream + req_format = self._downloader.params.get('format', None) + + # Select format if we can find one + for f,v in turls: + if f == req_format: + format, rtmp_video_url = f, v + break + + m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp.comedystor/.*)$', rtmp_video_url) + if not m: + raise ExtractorError(u'Cannot transform RTMP url') + base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/' + video_url = base + m.group('finalid') + + effTitle = showId + u'-' + epTitle + u' part ' + compat_str(partNum+1) + info = { + 'id': shortMediaId, + 'url': video_url, + 'uploader': showId, + 'upload_date': officialDate, + 'title': effTitle, + 'ext': 'mp4', + 'format': format, + 'thumbnail': None, + 'description': officialTitle, + } + results.append(info) + + return results diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py new file mode 100644 index 000000000..64d63e109 --- /dev/null +++ b/youtube_dl/extractor/common.py @@ -0,0 +1,264 @@ +import base64 +import os +import re +import socket +import sys + +from ..utils import ( + compat_http_client, + compat_urllib_error, + compat_urllib_request, + compat_str, + + clean_html, + compiled_regex_type, + ExtractorError, +) + +class InfoExtractor(object): + """Information Extractor class. + + Information extractors are the classes that, given a URL, extract + information about the video (or videos) the URL refers to. This + information includes the real video URL, the video title, author and + others. The information is stored in a dictionary which is then + passed to the FileDownloader. The FileDownloader processes this + information possibly downloading the video to the file system, among + other possible outcomes. + + The dictionaries must include the following fields: + + id: Video identifier. + url: Final video URL. + title: Video title, unescaped. + ext: Video filename extension. + + The following fields are optional: + + format: The video format, defaults to ext (used for --get-format) + thumbnail: Full URL to a video thumbnail image. + description: One-line video description. + uploader: Full name of the video uploader. + upload_date: Video upload date (YYYYMMDD). + uploader_id: Nickname or id of the video uploader. + location: Physical location of the video. + player_url: SWF Player URL (used for rtmpdump). + subtitles: The subtitle file contents. + urlhandle: [internal] The urlHandle to be used to download the file, + like returned by urllib.request.urlopen + + The fields should all be Unicode strings. + + Subclasses of this one should re-define the _real_initialize() and + _real_extract() methods and define a _VALID_URL regexp. + Probably, they should also be added to the list of extractors. + + _real_extract() must return a *list* of information dictionaries as + described above. + + Finally, the _WORKING attribute should be set to False for broken IEs + in order to warn the users and skip the tests. + """ + + _ready = False + _downloader = None + _WORKING = True + + def __init__(self, downloader=None): + """Constructor. Receives an optional downloader.""" + self._ready = False + self.set_downloader(downloader) + + @classmethod + def suitable(cls, url): + """Receives a URL and returns True if suitable for this IE.""" + return re.match(cls._VALID_URL, url) is not None + + @classmethod + def working(cls): + """Getter method for _WORKING.""" + return cls._WORKING + + def initialize(self): + """Initializes an instance (authentication, etc).""" + if not self._ready: + self._real_initialize() + self._ready = True + + def extract(self, url): + """Extracts URL information and returns it in list of dicts.""" + self.initialize() + return self._real_extract(url) + + def set_downloader(self, downloader): + """Sets the downloader for this IE.""" + self._downloader = downloader + + def _real_initialize(self): + """Real initialization process. Redefine in subclasses.""" + pass + + def _real_extract(self, url): + """Real extraction process. Redefine in subclasses.""" + pass + + @property + def IE_NAME(self): + return type(self).__name__[:-2] + + def _request_webpage(self, url_or_request, video_id, note=None, errnote=None): + """ Returns the response handle """ + if note is None: + self.report_download_webpage(video_id) + elif note is not False: + self.to_screen(u'%s: %s' % (video_id, note)) + try: + return compat_urllib_request.urlopen(url_or_request) + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + if errnote is None: + errnote = u'Unable to download webpage' + raise ExtractorError(u'%s: %s' % (errnote, compat_str(err)), sys.exc_info()[2]) + + def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None): + """ Returns a tuple (page content as string, URL handle) """ + urlh = self._request_webpage(url_or_request, video_id, note, errnote) + content_type = urlh.headers.get('Content-Type', '') + m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type) + if m: + encoding = m.group(1) + else: + encoding = 'utf-8' + webpage_bytes = urlh.read() + if self._downloader.params.get('dump_intermediate_pages', False): + try: + url = url_or_request.get_full_url() + except AttributeError: + url = url_or_request + self.to_screen(u'Dumping request to ' + url) + dump = base64.b64encode(webpage_bytes).decode('ascii') + self._downloader.to_screen(dump) + content = webpage_bytes.decode(encoding, 'replace') + return (content, urlh) + + def _download_webpage(self, url_or_request, video_id, note=None, errnote=None): + """ Returns the data of the page as a string """ + return self._download_webpage_handle(url_or_request, video_id, note, errnote)[0] + + def to_screen(self, msg): + """Print msg to screen, prefixing it with '[ie_name]'""" + self._downloader.to_screen(u'[%s] %s' % (self.IE_NAME, msg)) + + def report_extraction(self, id_or_name): + """Report information extraction.""" + self.to_screen(u'%s: Extracting information' % id_or_name) + + def report_download_webpage(self, video_id): + """Report webpage download.""" + self.to_screen(u'%s: Downloading webpage' % video_id) + + def report_age_confirmation(self): + """Report attempt to confirm age.""" + self.to_screen(u'Confirming age') + + #Methods for following #608 + #They set the correct value of the '_type' key + def video_result(self, video_info): + """Returns a video""" + video_info['_type'] = 'video' + return video_info + def url_result(self, url, ie=None): + """Returns a url that points to a page that should be processed""" + #TODO: ie should be the class used for getting the info + video_info = {'_type': 'url', + 'url': url, + 'ie_key': ie} + return video_info + def playlist_result(self, entries, playlist_id=None, playlist_title=None): + """Returns a playlist""" + video_info = {'_type': 'playlist', + 'entries': entries} + if playlist_id: + video_info['id'] = playlist_id + if playlist_title: + video_info['title'] = playlist_title + return video_info + + def _search_regex(self, pattern, string, name, default=None, fatal=True, flags=0): + """ + Perform a regex search on the given string, using a single or a list of + patterns returning the first matching group. + In case of failure return a default value or raise a WARNING or a + ExtractorError, depending on fatal, specifying the field name. + """ + if isinstance(pattern, (str, compat_str, compiled_regex_type)): + mobj = re.search(pattern, string, flags) + else: + for p in pattern: + mobj = re.search(p, string, flags) + if mobj: break + + if sys.stderr.isatty() and os.name != 'nt': + _name = u'\033[0;34m%s\033[0m' % name + else: + _name = name + + if mobj: + # return the first matching group + return next(g for g in mobj.groups() if g is not None) + elif default is not None: + return default + elif fatal: + raise ExtractorError(u'Unable to extract %s' % _name) + else: + self._downloader.report_warning(u'unable to extract %s; ' + u'please report this issue on http://yt-dl.org/bug' % _name) + return None + + def _html_search_regex(self, pattern, string, name, default=None, fatal=True, flags=0): + """ + Like _search_regex, but strips HTML tags and unescapes entities. + """ + res = self._search_regex(pattern, string, name, default, fatal, flags) + if res: + return clean_html(res).strip() + else: + return res + +class SearchInfoExtractor(InfoExtractor): + """ + Base class for paged search queries extractors. + They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query} + Instances should define _SEARCH_KEY and _MAX_RESULTS. + """ + + @classmethod + def _make_valid_url(cls): + return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY + + @classmethod + def suitable(cls, url): + return re.match(cls._make_valid_url(), url) is not None + + def _real_extract(self, query): + mobj = re.match(self._make_valid_url(), query) + if mobj is None: + raise ExtractorError(u'Invalid search query "%s"' % query) + + prefix = mobj.group('prefix') + query = mobj.group('query') + if prefix == '': + return self._get_n_results(query, 1) + elif prefix == 'all': + return self._get_n_results(query, self._MAX_RESULTS) + else: + n = int(prefix) + if n <= 0: + raise ExtractorError(u'invalid download number %s for query "%s"' % (n, query)) + elif n > self._MAX_RESULTS: + self._downloader.report_warning(u'%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n)) + n = self._MAX_RESULTS + return self._get_n_results(query, n) + + def _get_n_results(self, query, n): + """Get a specified number of results for a query""" + raise NotImplementedError("This method must be implemented by sublclasses") diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py new file mode 100644 index 000000000..34306b073 --- /dev/null +++ b/youtube_dl/extractor/dailymotion.py @@ -0,0 +1,77 @@ +import re + +from .common import InfoExtractor +from ..utils import ( + compat_urllib_request, + compat_urllib_parse, + + ExtractorError, + unescapeHTML, +) + +class DailymotionIE(InfoExtractor): + """Information Extractor for Dailymotion""" + + _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^/]+)' + IE_NAME = u'dailymotion' + + def _real_extract(self, url): + # Extract id and simplified title from URL + mobj = re.match(self._VALID_URL, url) + + video_id = mobj.group(1).split('_')[0].split('?')[0] + + video_extension = 'mp4' + + # Retrieve video webpage to extract further information + request = compat_urllib_request.Request(url) + request.add_header('Cookie', 'family_filter=off') + webpage = self._download_webpage(request, video_id) + + # Extract URL, uploader and title from webpage + self.report_extraction(video_id) + mobj = re.search(r'\s*var flashvars = (.*)', webpage) + if mobj is None: + raise ExtractorError(u'Unable to extract media URL') + flashvars = compat_urllib_parse.unquote(mobj.group(1)) + + for key in ['hd1080URL', 'hd720URL', 'hqURL', 'sdURL', 'ldURL', 'video_url']: + if key in flashvars: + max_quality = key + self.to_screen(u'Using %s' % key) + break + else: + raise ExtractorError(u'Unable to extract video URL') + + mobj = re.search(r'"' + max_quality + r'":"(.+?)"', flashvars) + if mobj is None: + raise ExtractorError(u'Unable to extract video URL') + + video_url = compat_urllib_parse.unquote(mobj.group(1)).replace('\\/', '/') + + # TODO: support choosing qualities + + mobj = re.search(r'<meta property="og:title" content="(?P<title>[^"]*)" />', webpage) + if mobj is None: + raise ExtractorError(u'Unable to extract title') + video_title = unescapeHTML(mobj.group('title')) + + video_uploader = None + video_uploader = self._search_regex([r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>', + # Looking for official user + r'<(?:span|a) .*?rel="author".*?>([^<]+?)</'], + webpage, 'video uploader') + + video_upload_date = None + mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage) + if mobj is not None: + video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1) + + return [{ + 'id': video_id, + 'url': video_url, + 'uploader': video_uploader, + 'upload_date': video_upload_date, + 'title': video_title, + 'ext': video_extension, + }] diff --git a/youtube_dl/extractor/depositfiles.py b/youtube_dl/extractor/depositfiles.py new file mode 100644 index 000000000..d43348955 --- /dev/null +++ b/youtube_dl/extractor/depositfiles.py @@ -0,0 +1,60 @@ +import re +import os +import socket + +from .common import InfoExtractor +from ..utils import ( + compat_http_client, + compat_str, + compat_urllib_error, + compat_urllib_parse, + compat_urllib_request, + + ExtractorError, +) + + +class DepositFilesIE(InfoExtractor): + """Information extractor for depositfiles.com""" + + _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles\.com/(?:../(?#locale))?files/(.+)' + + def _real_extract(self, url): + file_id = url.split('/')[-1] + # Rebuild url in english locale + url = 'http://depositfiles.com/en/files/' + file_id + + # Retrieve file webpage with 'Free download' button pressed + free_download_indication = { 'gateway_result' : '1' } + request = compat_urllib_request.Request(url, compat_urllib_parse.urlencode(free_download_indication)) + try: + self.report_download_webpage(file_id) + webpage = compat_urllib_request.urlopen(request).read() + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + raise ExtractorError(u'Unable to retrieve file webpage: %s' % compat_str(err)) + + # Search for the real file URL + mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage) + if (mobj is None) or (mobj.group(1) is None): + # Try to figure out reason of the error. + mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL) + if (mobj is not None) and (mobj.group(1) is not None): + restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip() + raise ExtractorError(u'%s' % restriction_message) + else: + raise ExtractorError(u'Unable to extract download URL from: %s' % url) + + file_url = mobj.group(1) + file_extension = os.path.splitext(file_url)[1][1:] + + # Search for file title + file_title = self._search_regex(r'<b title="(.*?)">', webpage, u'title') + + return [{ + 'id': file_id.decode('utf-8'), + 'url': file_url.decode('utf-8'), + 'uploader': None, + 'upload_date': None, + 'title': file_title, + 'ext': file_extension.decode('utf-8'), + }] diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py new file mode 100644 index 000000000..c3d4343f4 --- /dev/null +++ b/youtube_dl/extractor/eighttracks.py @@ -0,0 +1,51 @@ +import itertools +import json +import random +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, +) + + +class EightTracksIE(InfoExtractor): + IE_NAME = '8tracks' + _VALID_URL = r'https?://8tracks.com/(?P<user>[^/]+)/(?P<id>[^/#]+)(?:#.*)?$' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + raise ExtractorError(u'Invalid URL: %s' % url) + playlist_id = mobj.group('id') + + webpage = self._download_webpage(url, playlist_id) + + json_like = self._search_regex(r"PAGE.mix = (.*?);\n", webpage, u'trax information', flags=re.DOTALL) + data = json.loads(json_like) + + session = str(random.randint(0, 1000000000)) + mix_id = data['id'] + track_count = data['tracks_count'] + first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id) + next_url = first_url + res = [] + for i in itertools.count(): + api_json = self._download_webpage(next_url, playlist_id, + note=u'Downloading song information %s/%s' % (str(i+1), track_count), + errnote=u'Failed to download song information') + api_data = json.loads(api_json) + track_data = api_data[u'set']['track'] + info = { + 'id': track_data['id'], + 'url': track_data['track_file_stream_url'], + 'title': track_data['performer'] + u' - ' + track_data['name'], + 'raw_title': track_data['name'], + 'uploader_id': data['user']['login'], + 'ext': 'm4a', + } + res.append(info) + if api_data['set']['at_last_track']: + break + next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % (session, mix_id, track_data['id']) + return res diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py new file mode 100644 index 000000000..86b145bca --- /dev/null +++ b/youtube_dl/extractor/escapist.py @@ -0,0 +1,68 @@ +import json +import re + +from .common import InfoExtractor +from ..utils import ( + compat_str, + compat_urllib_parse, + + ExtractorError, +) + + +class EscapistIE(InfoExtractor): + _VALID_URL = r'^(https?://)?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?]?.*$' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + raise ExtractorError(u'Invalid URL: %s' % url) + showName = mobj.group('showname') + videoId = mobj.group('episode') + + self.report_extraction(videoId) + webpage = self._download_webpage(url, videoId) + + videoDesc = self._html_search_regex('<meta name="description" content="([^"]*)"', + webpage, u'description', fatal=False) + + imgUrl = self._html_search_regex('<meta property="og:image" content="([^"]*)"', + webpage, u'thumbnail', fatal=False) + + playerUrl = self._html_search_regex('<meta property="og:video" content="([^"]*)"', + webpage, u'player url') + + title = self._html_search_regex('<meta name="title" content="([^"]*)"', + webpage, u'player url').split(' : ')[-1] + + configUrl = self._search_regex('config=(.*)$', playerUrl, u'config url') + configUrl = compat_urllib_parse.unquote(configUrl) + + configJSON = self._download_webpage(configUrl, videoId, + u'Downloading configuration', + u'unable to download configuration') + + # Technically, it's JavaScript, not JSON + configJSON = configJSON.replace("'", '"') + + try: + config = json.loads(configJSON) + except (ValueError,) as err: + raise ExtractorError(u'Invalid JSON in configuration file: ' + compat_str(err)) + + playlist = config['playlist'] + videoUrl = playlist[1]['url'] + + info = { + 'id': videoId, + 'url': videoUrl, + 'uploader': showName, + 'upload_date': None, + 'title': title, + 'ext': 'mp4', + 'thumbnail': imgUrl, + 'description': videoDesc, + 'player_url': playerUrl, + } + + return [info] diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py new file mode 100644 index 000000000..c694f9adb --- /dev/null +++ b/youtube_dl/extractor/facebook.py @@ -0,0 +1,111 @@ +import json +import netrc +import re +import socket + +from .common import InfoExtractor +from ..utils import ( + compat_http_client, + compat_str, + compat_urllib_error, + compat_urllib_parse, + compat_urllib_request, + + ExtractorError, +) + + +class FacebookIE(InfoExtractor): + """Information Extractor for Facebook""" + + _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)' + _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&' + _NETRC_MACHINE = 'facebook' + IE_NAME = u'facebook' + + def report_login(self): + """Report attempt to log in.""" + self.to_screen(u'Logging in') + + def _real_initialize(self): + if self._downloader is None: + return + + useremail = None + password = None + downloader_params = self._downloader.params + + # Attempt to use provided username and password or .netrc data + if downloader_params.get('username', None) is not None: + useremail = downloader_params['username'] + password = downloader_params['password'] + elif downloader_params.get('usenetrc', False): + try: + info = netrc.netrc().authenticators(self._NETRC_MACHINE) + if info is not None: + useremail = info[0] + password = info[2] + else: + raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE) + except (IOError, netrc.NetrcParseError) as err: + self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err)) + return + + if useremail is None: + return + + # Log in + login_form = { + 'email': useremail, + 'pass': password, + 'login': 'Log+In' + } + request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form)) + try: + self.report_login() + login_results = compat_urllib_request.urlopen(request).read() + if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None: + self._downloader.report_warning(u'unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.') + return + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + self._downloader.report_warning(u'unable to log in: %s' % compat_str(err)) + return + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + raise ExtractorError(u'Invalid URL: %s' % url) + video_id = mobj.group('ID') + + url = 'https://www.facebook.com/video/video.php?v=%s' % video_id + webpage = self._download_webpage(url, video_id) + + BEFORE = '{swf.addParam(param[0], param[1]);});\n' + AFTER = '.forEach(function(variable) {swf.addVariable(variable[0], variable[1]);});' + m = re.search(re.escape(BEFORE) + '(.*?)' + re.escape(AFTER), webpage) + if not m: + raise ExtractorError(u'Cannot parse data') + data = dict(json.loads(m.group(1))) + params_raw = compat_urllib_parse.unquote(data['params']) + params = json.loads(params_raw) + video_data = params['video_data'][0] + video_url = video_data.get('hd_src') + if not video_url: + video_url = video_data['sd_src'] + if not video_url: + raise ExtractorError(u'Cannot find video URL') + video_duration = int(video_data['video_duration']) + thumbnail = video_data['thumbnail_src'] + + video_title = self._html_search_regex('<h2 class="uiHeaderTitle">([^<]+)</h2>', + webpage, u'title') + + info = { + 'id': video_id, + 'title': video_title, + 'url': video_url, + 'ext': 'mp4', + 'duration': video_duration, + 'thumbnail': thumbnail, + } + return [info] diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py new file mode 100644 index 000000000..791d5b61d --- /dev/null +++ b/youtube_dl/extractor/flickr.py @@ -0,0 +1,57 @@ +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + unescapeHTML, +) + + +class FlickrIE(InfoExtractor): + """Information Extractor for Flickr videos""" + _VALID_URL = r'(?:https?://)?(?:www\.)?flickr\.com/photos/(?P<uploader_id>[\w\-_@]+)/(?P<id>\d+).*' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + + video_id = mobj.group('id') + video_uploader_id = mobj.group('uploader_id') + webpage_url = 'http://www.flickr.com/photos/' + video_uploader_id + '/' + video_id + webpage = self._download_webpage(webpage_url, video_id) + + secret = self._search_regex(r"photo_secret: '(\w+)'", webpage, u'secret') + + first_url = 'https://secure.flickr.com/apps/video/video_mtl_xml.gne?v=x&photo_id=' + video_id + '&secret=' + secret + '&bitrate=700&target=_self' + first_xml = self._download_webpage(first_url, video_id, 'Downloading first data webpage') + + node_id = self._html_search_regex(r'<Item id="id">(\d+-\d+)</Item>', + first_xml, u'node_id') + + second_url = 'https://secure.flickr.com/video_playlist.gne?node_id=' + node_id + '&tech=flash&mode=playlist&bitrate=700&secret=' + secret + '&rd=video.yahoo.com&noad=1' + second_xml = self._download_webpage(second_url, video_id, 'Downloading second data webpage') + + self.report_extraction(video_id) + + mobj = re.search(r'<STREAM APP="(.+?)" FULLPATH="(.+?)"', second_xml) + if mobj is None: + raise ExtractorError(u'Unable to extract video url') + video_url = mobj.group(1) + unescapeHTML(mobj.group(2)) + + video_title = self._html_search_regex(r'<meta property="og:title" content=(?:"([^"]+)"|\'([^\']+)\')', + webpage, u'video title') + + video_description = self._html_search_regex(r'<meta property="og:description" content=(?:"([^"]+)"|\'([^\']+)\')', + webpage, u'description', fatal=False) + + thumbnail = self._html_search_regex(r'<meta property="og:image" content=(?:"([^"]+)"|\'([^\']+)\')', + webpage, u'thumbnail', fatal=False) + + return [{ + 'id': video_id, + 'url': video_url, + 'ext': 'mp4', + 'title': video_title, + 'description': video_description, + 'thumbnail': thumbnail, + 'uploader_id': video_uploader_id, + }] diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py new file mode 100644 index 000000000..3045978f1 --- /dev/null +++ b/youtube_dl/extractor/funnyordie.py @@ -0,0 +1,31 @@ +import re + +from .common import InfoExtractor + + +class FunnyOrDieIE(InfoExtractor): + _VALID_URL = r'^(?:https?://)?(?:www\.)?funnyordie\.com/videos/(?P<id>[0-9a-f]+)/.*$' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + + video_id = mobj.group('id') + webpage = self._download_webpage(url, video_id) + + video_url = self._html_search_regex(r'<video[^>]*>\s*<source[^>]*>\s*<source src="(?P<url>[^"]+)"', + webpage, u'video URL', flags=re.DOTALL) + + title = self._html_search_regex((r"<h1 class='player_page_h1'.*?>(?P<title>.*?)</h1>", + r'<title>(?P<title>[^<]+?)'), webpage, 'title', flags=re.DOTALL) + + video_description = self._html_search_regex(r'videos|reviews|full-episodes)/(?P.*?)/(?P.*)' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + raise ExtractorError(u'Invalid URL: %s' % url) + video_id = mobj.group('id') + video_type = mobj.group('type') + webpage = self._download_webpage(url, video_id) + if video_type == 'full-episodes': + mgid_re = r'data-video="(?P<mgid>mgid:.*?)"' + else: + mgid_re = r'data-contentId=\'(?P<mgid>mgid:.*?)\'' + mgid = self._search_regex(mgid_re, webpage, u'mgid') + data = compat_urllib_parse.urlencode({'uri': mgid, 'acceptMethods': 'fms'}) + + info_page = self._download_webpage('http://www.gametrailers.com/feeds/mrss?' + data, + video_id, u'Downloading video info') + links_webpage = self._download_webpage('http://www.gametrailers.com/feeds/mediagen/?' + data, + video_id, u'Downloading video urls info') + + self.report_extraction(video_id) + info_re = r'''<title><!\[CDATA\[(?P<title>.*?)\]\]>.* + .*?)\]\]>.* + .* + (?P.*?).* + ''' + + m_info = re.search(info_re, info_page, re.VERBOSE|re.DOTALL) + if m_info is None: + raise ExtractorError(u'Unable to extract video info') + video_title = m_info.group('title') + video_description = m_info.group('description') + video_thumb = m_info.group('thumb') + + m_urls = list(re.finditer(r'(?P.*)', links_webpage)) + if m_urls is None or len(m_urls) == 0: + raise ExtractorError(u'Unable to extract video url') + # They are sorted from worst to best quality + video_url = m_urls[-1].group('url') + + return {'url': video_url, + 'id': video_id, + 'title': video_title, + # Videos are actually flv not mp4 + 'ext': 'flv', + 'thumbnail': video_thumb, + 'description': video_description, + } diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py new file mode 100644 index 000000000..7a877b3bc --- /dev/null +++ b/youtube_dl/extractor/generic.py @@ -0,0 +1,151 @@ +import os +import re + +from .common import InfoExtractor +from ..utils import ( + compat_urllib_error, + compat_urllib_parse, + compat_urllib_request, + + ExtractorError, +) + +class GenericIE(InfoExtractor): + """Generic last-resort information extractor.""" + + _VALID_URL = r'.*' + IE_NAME = u'generic' + + def report_download_webpage(self, video_id): + """Report webpage download.""" + if not self._downloader.params.get('test', False): + self._downloader.report_warning(u'Falling back on generic information extractor.') + super(GenericIE, self).report_download_webpage(video_id) + + def report_following_redirect(self, new_url): + """Report information extraction.""" + self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url) + + def _test_redirect(self, url): + """Check if it is a redirect, like url shorteners, in case return the new url.""" + class HeadRequest(compat_urllib_request.Request): + def get_method(self): + return "HEAD" + + class HEADRedirectHandler(compat_urllib_request.HTTPRedirectHandler): + """ + Subclass the HTTPRedirectHandler to make it use our + HeadRequest also on the redirected URL + """ + def redirect_request(self, req, fp, code, msg, headers, newurl): + if code in (301, 302, 303, 307): + newurl = newurl.replace(' ', '%20') + newheaders = dict((k,v) for k,v in req.headers.items() + if k.lower() not in ("content-length", "content-type")) + return HeadRequest(newurl, + headers=newheaders, + origin_req_host=req.get_origin_req_host(), + unverifiable=True) + else: + raise compat_urllib_error.HTTPError(req.get_full_url(), code, msg, headers, fp) + + class HTTPMethodFallback(compat_urllib_request.BaseHandler): + """ + Fallback to GET if HEAD is not allowed (405 HTTP error) + """ + def http_error_405(self, req, fp, code, msg, headers): + fp.read() + fp.close() + + newheaders = dict((k,v) for k,v in req.headers.items() + if k.lower() not in ("content-length", "content-type")) + return self.parent.open(compat_urllib_request.Request(req.get_full_url(), + headers=newheaders, + origin_req_host=req.get_origin_req_host(), + unverifiable=True)) + + # Build our opener + opener = compat_urllib_request.OpenerDirector() + for handler in [compat_urllib_request.HTTPHandler, compat_urllib_request.HTTPDefaultErrorHandler, + HTTPMethodFallback, HEADRedirectHandler, + compat_urllib_request.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]: + opener.add_handler(handler()) + + response = opener.open(HeadRequest(url)) + if response is None: + raise ExtractorError(u'Invalid URL protocol') + new_url = response.geturl() + + if url == new_url: + return False + + self.report_following_redirect(new_url) + return new_url + + def _real_extract(self, url): + new_url = self._test_redirect(url) + if new_url: return [self.url_result(new_url)] + + video_id = url.split('/')[-1] + try: + webpage = self._download_webpage(url, video_id) + except ValueError: + # since this is the last-resort InfoExtractor, if + # this error is thrown, it'll be thrown here + raise ExtractorError(u'Invalid URL: %s' % url) + + self.report_extraction(video_id) + # Start with something easy: JW Player in SWFObject + mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage) + if mobj is None: + # Broaden the search a little bit + mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage) + if mobj is None: + # Broaden the search a little bit: JWPlayer JS loader + mobj = re.search(r'[^A-Za-z0-9]?file:\s*["\'](http[^\'"&]*)', webpage) + if mobj is None: + # Try to find twitter cards info + mobj = re.search(r'(.*)', + webpage, u'video title') + + # video uploader is domain name + video_uploader = self._search_regex(r'(?:https?://)?([^/]*)/.*', + url, u'video uploader') + + return [{ + 'id': video_id, + 'url': video_url, + 'uploader': video_uploader, + 'upload_date': None, + 'title': video_title, + 'ext': video_extension, + }] diff --git a/youtube_dl/extractor/googleplus.py b/youtube_dl/extractor/googleplus.py new file mode 100644 index 000000000..ff2cdeebb --- /dev/null +++ b/youtube_dl/extractor/googleplus.py @@ -0,0 +1,86 @@ +import datetime +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, +) + + +class GooglePlusIE(InfoExtractor): + """Information extractor for plus.google.com.""" + + _VALID_URL = r'(?:https://)?plus\.google\.com/(?:[^/]+/)*?posts/(\w+)' + IE_NAME = u'plus.google' + + def _real_extract(self, url): + # Extract id from URL + mobj = re.match(self._VALID_URL, url) + if mobj is None: + raise ExtractorError(u'Invalid URL: %s' % url) + + post_url = mobj.group(0) + video_id = mobj.group(1) + + video_extension = 'flv' + + # Step 1, Retrieve post webpage to extract further information + webpage = self._download_webpage(post_url, video_id, u'Downloading entry webpage') + + self.report_extraction(video_id) + + # Extract update date + upload_date = self._html_search_regex('title="Timestamp">(.*?)', + webpage, u'upload date', fatal=False) + if upload_date: + # Convert timestring to a format suitable for filename + upload_date = datetime.datetime.strptime(upload_date, "%Y-%m-%d") + upload_date = upload_date.strftime('%Y%m%d') + + # Extract uploader + uploader = self._html_search_regex(r'rel\="author".*?>(.*?)', + webpage, u'uploader', fatal=False) + + # Extract title + # Get the first line for title + video_title = self._html_search_regex(r' n) or not re.search(self._MORE_PAGES_INDICATOR, webpage): + return res diff --git a/youtube_dl/extractor/howcast.py b/youtube_dl/extractor/howcast.py new file mode 100644 index 000000000..7b94f85ad --- /dev/null +++ b/youtube_dl/extractor/howcast.py @@ -0,0 +1,37 @@ +import re + +from .common import InfoExtractor + + +class HowcastIE(InfoExtractor): + _VALID_URL = r'(?:https?://)?(?:www\.)?howcast\.com/videos/(?P\d+)' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + + video_id = mobj.group('id') + webpage_url = 'http://www.howcast.com/videos/' + video_id + webpage = self._download_webpage(webpage_url, video_id) + + self.report_extraction(video_id) + + video_url = self._search_regex(r'\'?file\'?: "(http://mobile-media\.howcast\.com/[0-9]+\.mp4)', + webpage, u'video URL') + + video_title = self._html_search_regex(r'(.*?)', + response, u'tracks', flags=re.MULTILINE|re.DOTALL).strip() + try: + track_list = json.loads(html_tracks) + track = track_list[u'tracks'][0] + except ValueError: + raise ExtractorError(u'Hypemachine contained invalid JSON.') + + key = track[u"key"] + track_id = track[u"id"] + artist = track[u"artist"] + title = track[u"song"] + + serve_url = "http://hypem.com/serve/source/%s/%s" % (compat_str(track_id), compat_str(key)) + request = compat_urllib_request.Request(serve_url, "" , {'Content-Type': 'application/json'}) + request.add_header('cookie', cookie) + song_data_json = self._download_webpage(request, track_id, u'Downloading metadata') + try: + song_data = json.loads(song_data_json) + except ValueError: + raise ExtractorError(u'Hypemachine contained invalid JSON.') + final_url = song_data[u"url"] + + return [{ + 'id': track_id, + 'url': final_url, + 'ext': "mp3", + 'title': title, + 'artist': artist, + }] \ No newline at end of file diff --git a/youtube_dl/extractor/ina.py b/youtube_dl/extractor/ina.py new file mode 100644 index 000000000..c19b95659 --- /dev/null +++ b/youtube_dl/extractor/ina.py @@ -0,0 +1,31 @@ +import re + +from .common import InfoExtractor + + +class InaIE(InfoExtractor): + """Information Extractor for Ina.fr""" + _VALID_URL = r'(?:http://)?(?:www\.)?ina\.fr/video/(?PI[0-9]+)/.*' + + def _real_extract(self,url): + mobj = re.match(self._VALID_URL, url) + + video_id = mobj.group('id') + mrss_url='http://player.ina.fr/notices/%s.mrss' % video_id + video_extension = 'mp4' + webpage = self._download_webpage(mrss_url, video_id) + + self.report_extraction(video_id) + + video_url = self._html_search_regex(r'.*?)]]>', + webpage, u'title') + + return [{ + 'id': video_id, + 'url': video_url, + 'ext': video_extension, + 'title': video_title, + }] diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py new file mode 100644 index 000000000..905674282 --- /dev/null +++ b/youtube_dl/extractor/infoq.py @@ -0,0 +1,50 @@ +import base64 +import re + +from .common import InfoExtractor +from ..utils import ( + compat_urllib_parse, + + ExtractorError, +) + + +class InfoQIE(InfoExtractor): + _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + + webpage = self._download_webpage(url, video_id=url) + self.report_extraction(url) + + # Extract video URL + mobj = re.search(r"jsclassref ?= ?'([^']*)'", webpage) + if mobj is None: + raise ExtractorError(u'Unable to extract video url') + real_id = compat_urllib_parse.unquote(base64.b64decode(mobj.group(1).encode('ascii')).decode('utf-8')) + video_url = 'rtmpe://video.infoq.com/cfx/st/' + real_id + + # Extract title + video_title = self._search_regex(r'contentTitle = "(.*?)";', + webpage, u'title') + + # Extract description + video_description = self._html_search_regex(r'', + webpage, u'description', fatal=False) + + video_filename = video_url.split('/')[-1] + video_id, extension = video_filename.split('.') + + info = { + 'id': video_id, + 'url': video_url, + 'uploader': None, + 'upload_date': None, + 'title': video_title, + 'ext': extension, # Extension is always(?) mp4, but seems to be flv + 'thumbnail': None, + 'description': video_description, + } + + return [info] \ No newline at end of file diff --git a/youtube_dl/extractor/jukebox.py b/youtube_dl/extractor/jukebox.py new file mode 100644 index 000000000..c7bb234fe --- /dev/null +++ b/youtube_dl/extractor/jukebox.py @@ -0,0 +1,56 @@ +# coding: utf-8 +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + unescapeHTML, +) + +class JukeboxIE(InfoExtractor): + _VALID_URL = r'^http://www\.jukebox?\..+?\/.+[,](?P[a-z0-9\-]+).html' + _IFRAME = r'