From 174aba3223d0b01ec30cc0a814c6c0813ce13f15 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 1 May 2016 10:19:14 +0200 Subject: [PATCH 01/24] release 2016.05.01 --- .github/ISSUE_TEMPLATE.md | 6 +++--- docs/supportedsites.md | 14 +++++++++----- youtube_dl/version.py | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index c208eb689..a26ff1de4 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -6,8 +6,8 @@ --- -### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.04.24*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected. -- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.04.24** +### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.05.01*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected. +- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.05.01** ### Before submitting an *issue* make sure you have: - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections @@ -35,7 +35,7 @@ $ youtube-dl -v [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 -[debug] youtube-dl version 2016.04.24 +[debug] youtube-dl version 2016.05.01 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/docs/supportedsites.md b/docs/supportedsites.md index 03875b8db..9fb43671f 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -338,7 +338,6 @@ - **mailru**: Видео@Mail.Ru - **MakersChannel** - **MakerTV** - - **Malemotion** - **MatchTV** - **MDR**: MDR.DE and KiKA - **media.ccc.de** @@ -375,8 +374,8 @@ - **mtvservices:embedded** - **MuenchenTV**: münchen.tv - **MusicPlayOn** - - **muzu.tv** - **Mwave** + - **MwaveMeetGreet** - **MySpace** - **MySpace:album** - **MySpass** @@ -554,7 +553,6 @@ - **SenateISVP** - **ServingSys** - **Sexu** - - **SexyKarma**: Sexy Karma and Watch Indian Porn - **Shahid** - **Shared**: shared.sx and vivo.sx - **ShareSix** @@ -567,8 +565,6 @@ - **smotri:broadcast**: Smotri.com broadcasts - **smotri:community**: Smotri.com community videos - **smotri:user**: Smotri.com user videos - - **SnagFilms** - - **SnagFilmsEmbed** - **Snotr** - **Sohu** - **soundcloud** @@ -610,6 +606,7 @@ - **Syfy** - **SztvHu** - **Tagesschau** + - **tagesschau:player** - **Tapely** - **Tass** - **TDSLifeway** @@ -725,6 +722,8 @@ - **Vidzi** - **vier** - **vier:videos** + - **ViewLift** + - **ViewLiftEmbed** - **Viewster** - **Viidea** - **viki** @@ -756,6 +755,7 @@ - **Walla** - **WashingtonPost** - **wat.tv** + - **WatchIndianPorn**: Watch Indian Porn - **WDR** - **wdr:mobile** - **WDRMaus**: Sendung mit der Maus @@ -775,6 +775,10 @@ - **XFileShare**: XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me - **XHamster** - **XHamsterEmbed** + - **xiami:album**: 虾米音乐 - 专辑 + - **xiami:artist**: 虾米音乐 - 歌手 + - **xiami:collection**: 虾米音乐 - 精选集 + - **xiami:song**: 虾米音乐 - **XMinus** - **XNXX** - **Xstream** diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 8befd9607..551160897 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2016.04.24' +__version__ = '2016.05.01' From e0da32df6e924535dbbe1e1c8ad05386dcf93029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 May 2016 00:48:26 +0600 Subject: [PATCH 02/24] [vevo:playlist] Add extractor (Closes #9334, closes #9364) --- youtube_dl/extractor/extractors.py | 5 +- youtube_dl/extractor/vevo.py | 74 +++++++++++++++++++++++++++++- 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 4aee53d6a..ef4431364 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -849,7 +849,10 @@ from .veehd import VeeHDIE from .veoh import VeohIE from .vessel import VesselIE from .vesti import VestiIE -from .vevo import VevoIE +from .vevo import ( + VevoIE, + VevoPlaylistIE, +) from .vgtv import ( BTArticleIE, BTVestlendingenIE, diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index 147480f64..4ad1e87e4 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -3,7 +3,10 @@ from __future__ import unicode_literals import re from .common import InfoExtractor -from ..compat import compat_etree_fromstring +from ..compat import ( + compat_etree_fromstring, + compat_urlparse, +) from ..utils import ( ExtractorError, int_or_none, @@ -18,7 +21,7 @@ class VevoIE(InfoExtractor): (currently used by MTVIE and MySpaceIE) ''' _VALID_URL = r'''(?x) - (?:https?://www\.vevo\.com/watch/(?:[^/]+/(?:[^/]+/)?)?| + (?:https?://www\.vevo\.com/watch/(?!playlist|genre)(?:[^/]+/(?:[^/]+/)?)?| https?://cache\.vevo\.com/m/html/embed\.html\?video=| https?://videoplayer\.vevo\.com/embed/embedded\?videoId=| vevo:) @@ -301,3 +304,70 @@ class VevoIE(InfoExtractor): 'view_count': view_count, 'age_limit': age_limit, } + + +class VevoPlaylistIE(InfoExtractor): + _VALID_URL = r'https?://www\.vevo\.com/watch/(?:playlist|genre)/(?P[^/?#&]+)' + + _TESTS = [{ + 'url': 'http://www.vevo.com/watch/playlist/dadbf4e7-b99f-4184-9670-6f0e547b6a29', + 'info_dict': { + 'id': 'dadbf4e7-b99f-4184-9670-6f0e547b6a29', + 'title': 'Best-Of: Birdman', + }, + 'playlist_count': 10, + 'params': { + 'proxy': '52.53.186.253:8083', + 'no_check_certificate': True, + }, + }, { + 'url': 'http://www.vevo.com/watch/playlist/dadbf4e7-b99f-4184-9670-6f0e547b6a29?index=0', + 'md5': '32dcdfddddf9ec6917fc88ca26d36282', + 'info_dict': { + 'id': 'USCMV1100073', + 'ext': 'mp4', + 'title': 'Y.U. MAD', + 'timestamp': 1323417600, + 'upload_date': '20111209', + 'uploader': 'Birdman', + }, + 'expected_warnings': ['Unable to download SMIL file'], + 'params': { + 'proxy': '52.53.186.253:8083', + 'no_check_certificate': True, + }, + }, { + 'url': 'http://www.vevo.com/watch/genre/rock?index=0', + 'only_matching': True, + }] + + def _real_extract(self, url): + playlist_id = self._match_id(url) + + webpage = self._download_webpage(url, playlist_id) + + qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query) + index = qs.get('index', [None])[0] + + if index: + video_id = self._search_regex( + r']+content=(["\'])vevo://video/(?P.+?)\1[^>]*>', + webpage, 'video id', default=None, group='id') + if video_id: + return self.url_result('vevo:%s' % video_id, VevoIE.ie_key()) + + playlists = self._parse_json( + self._search_regex( + r'window\.__INITIAL_STORE__\s*=\s*({.+?});\s*', + webpage, 'initial store'), + playlist_id)['default']['playlists'] + + playlist = list(playlists.values())[0] + + entries = [ + self.url_result('vevo:%s' % src, VevoIE.ie_key()) + for src in playlist['isrcs']] + + return self.playlist_result( + entries, playlist.get('playlistId'), + playlist.get('name'), playlist.get('description')) From 0c9d288ba0db3c1309bc6090457e11375073f3eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 May 2016 00:50:31 +0600 Subject: [PATCH 03/24] [vevo:playlist] Remove debug params --- youtube_dl/extractor/vevo.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index 4ad1e87e4..8431077ad 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -316,10 +316,6 @@ class VevoPlaylistIE(InfoExtractor): 'title': 'Best-Of: Birdman', }, 'playlist_count': 10, - 'params': { - 'proxy': '52.53.186.253:8083', - 'no_check_certificate': True, - }, }, { 'url': 'http://www.vevo.com/watch/playlist/dadbf4e7-b99f-4184-9670-6f0e547b6a29?index=0', 'md5': '32dcdfddddf9ec6917fc88ca26d36282', @@ -332,10 +328,6 @@ class VevoPlaylistIE(InfoExtractor): 'uploader': 'Birdman', }, 'expected_warnings': ['Unable to download SMIL file'], - 'params': { - 'proxy': '52.53.186.253:8083', - 'no_check_certificate': True, - }, }, { 'url': 'http://www.vevo.com/watch/genre/rock?index=0', 'only_matching': True, From e2bd301ce7795597a7e3ef7f5a5446f9ec987883 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 May 2016 01:00:42 +0600 Subject: [PATCH 04/24] [vevo:playlist] Fix genre playlists --- youtube_dl/extractor/vevo.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index 8431077ad..2d1ff05e1 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -307,7 +307,7 @@ class VevoIE(InfoExtractor): class VevoPlaylistIE(InfoExtractor): - _VALID_URL = r'https?://www\.vevo\.com/watch/(?:playlist|genre)/(?P[^/?#&]+)' + _VALID_URL = r'https?://www\.vevo\.com/watch/(?Pplaylist|genre)/(?P[^/?#&]+)' _TESTS = [{ 'url': 'http://www.vevo.com/watch/playlist/dadbf4e7-b99f-4184-9670-6f0e547b6a29', @@ -316,6 +316,13 @@ class VevoPlaylistIE(InfoExtractor): 'title': 'Best-Of: Birdman', }, 'playlist_count': 10, + }, { + 'url': 'http://www.vevo.com/watch/genre/rock', + 'info_dict': { + 'id': 'rock', + 'title': 'Rock', + }, + 'playlist_count': 20, }, { 'url': 'http://www.vevo.com/watch/playlist/dadbf4e7-b99f-4184-9670-6f0e547b6a29?index=0', 'md5': '32dcdfddddf9ec6917fc88ca26d36282', @@ -334,7 +341,9 @@ class VevoPlaylistIE(InfoExtractor): }] def _real_extract(self, url): - playlist_id = self._match_id(url) + mobj = re.match(self._VALID_URL, url) + playlist_id = mobj.group('id') + playlist_kind = mobj.group('kind') webpage = self._download_webpage(url, playlist_id) @@ -352,9 +361,10 @@ class VevoPlaylistIE(InfoExtractor): self._search_regex( r'window\.__INITIAL_STORE__\s*=\s*({.+?});\s*', webpage, 'initial store'), - playlist_id)['default']['playlists'] + playlist_id)['default']['%ss' % playlist_kind] - playlist = list(playlists.values())[0] + playlist = (list(playlists.values())[0] + if playlist_kind == 'playlist' else playlists[playlist_id]) entries = [ self.url_result('vevo:%s' % src, VevoIE.ie_key()) From 516ea41a7dd5a350e93ea7cc5ca2c1fcbd0cb43b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 May 2016 02:54:50 +0600 Subject: [PATCH 05/24] [vevo] Fix _call_api --- youtube_dl/extractor/vevo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index 2d1ff05e1..35f974c4e 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -149,8 +149,8 @@ class VevoIE(InfoExtractor): auth_info = self._parse_json(webpage, video_id) self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['access_token'] - def _call_api(self, path, video_id, note, errnote, fatal=True): - return self._download_json(self._api_url_template % path, video_id, note, errnote) + def _call_api(self, path, *args, **kwargs): + return self._download_json(self._api_url_template % path, *args, **kwargs) def _real_extract(self, url): video_id = self._match_id(url) From 9618c448247a6aa528b4bf2f289d3dd164c11417 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 May 2016 02:58:20 +0600 Subject: [PATCH 06/24] [vevo] Extract video versions from webpage as a last resort (Closes #8426, closes #9366) --- youtube_dl/extractor/vevo.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index 35f974c4e..b6c6ba89f 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -15,7 +15,16 @@ from ..utils import ( ) -class VevoIE(InfoExtractor): +class VevoBaseIE(InfoExtractor): + def _extract_json(self, webpage, video_id, item): + return self._parse_json( + self._search_regex( + r'window\.__INITIAL_STORE__\s*=\s*({.+?});\s*', + webpage, 'initial store'), + video_id)['default'][item] + + +class VevoIE(VevoBaseIE): ''' Accepts urls from vevo.com or in the format 'vevo:{id}' (currently used by MTVIE and MySpaceIE) @@ -186,7 +195,14 @@ class VevoIE(InfoExtractor): video_versions = self._call_api( 'video/%s/streams' % video_id, video_id, 'Downloading video versions info', - 'Failed to download video versions info') + 'Failed to download video versions info', + fatal=False) + + # Some videos are only available via webpage (e.g. + # https://github.com/rg3/youtube-dl/issues/9366) + if not video_versions: + webpage = self._download_webpage(url, video_id) + video_versions = self._extract_json(webpage, video_id, 'streams')[video_id][0] timestamp = parse_iso8601(video_info.get('releaseDate')) artists = video_info.get('artists') @@ -306,7 +322,7 @@ class VevoIE(InfoExtractor): } -class VevoPlaylistIE(InfoExtractor): +class VevoPlaylistIE(VevoBaseIE): _VALID_URL = r'https?://www\.vevo\.com/watch/(?Pplaylist|genre)/(?P[^/?#&]+)' _TESTS = [{ @@ -357,11 +373,7 @@ class VevoPlaylistIE(InfoExtractor): if video_id: return self.url_result('vevo:%s' % video_id, VevoIE.ie_key()) - playlists = self._parse_json( - self._search_regex( - r'window\.__INITIAL_STORE__\s*=\s*({.+?});\s*', - webpage, 'initial store'), - playlist_id)['default']['%ss' % playlist_kind] + playlists = self._extract_json(webpage, playlist_id, '%ss' % playlist_kind) playlist = (list(playlists.values())[0] if playlist_kind == 'playlist' else playlists[playlist_id]) From 8e7d0048886f374a58f0fe8ba021644d7074d02c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 May 2016 03:06:48 +0600 Subject: [PATCH 07/24] [vevo] Add test for video only available via webpage --- youtube_dl/extractor/vevo.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index b6c6ba89f..4eb98e50d 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -84,6 +84,20 @@ class VevoIE(VevoBaseIE): 'uploader': 'K Camp', 'timestamp': 1449468000, }, + }, { + 'note': 'Only available via webpage', + 'url': 'http://www.vevo.com/watch/GBUV71600656', + 'md5': '67e79210613865b66a47c33baa5e37fe', + 'info_dict': { + 'id': 'GBUV71600656', + 'ext': 'mp4', + 'title': 'Viva Love', + 'upload_date': '20160428', + 'age_limit': 0, + 'uploader': 'ABC', + 'timestamp': 1461830400, + }, + 'expected_warnings': ['Failed to download video versions info'], }] _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com' _SOURCE_TYPES = { From 881dbc86c4e70252a5b8e5c726a6f2a32ff878c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 May 2016 03:28:58 +0600 Subject: [PATCH 08/24] [vevo] Extract track related metafields and add artists to title (Closes #1684) --- youtube_dl/extractor/vevo.py | 58 ++++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index 4eb98e50d..49cb3f479 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -42,11 +42,15 @@ class VevoIE(VevoBaseIE): 'info_dict': { 'id': 'GB1101300280', 'ext': 'mp4', - 'title': 'Somebody to Die For', + 'title': 'Hurts - Somebody to Die For', + 'timestamp': 1372057200, 'upload_date': '20130624', 'uploader': 'Hurts', - 'timestamp': 1372057200, + 'track': 'Somebody to Die For', + 'artist': 'Hurts', + 'genre': 'Pop', }, + 'expected_warnings': ['Unable to download SMIL file'], }, { 'note': 'v3 SMIL format', 'url': 'http://www.vevo.com/watch/cassadee-pope/i-wish-i-could-break-your-heart/USUV71302923', @@ -54,23 +58,31 @@ class VevoIE(VevoBaseIE): 'info_dict': { 'id': 'USUV71302923', 'ext': 'mp4', - 'title': 'I Wish I Could Break Your Heart', + 'title': 'Cassadee Pope - I Wish I Could Break Your Heart', + 'timestamp': 1392796919, 'upload_date': '20140219', 'uploader': 'Cassadee Pope', - 'timestamp': 1392796919, + 'track': 'I Wish I Could Break Your Heart', + 'artist': 'Cassadee Pope', + 'genre': 'Country', }, + 'expected_warnings': ['Unable to download SMIL file'], }, { 'note': 'Age-limited video', 'url': 'https://www.vevo.com/watch/justin-timberlake/tunnel-vision-explicit/USRV81300282', 'info_dict': { 'id': 'USRV81300282', 'ext': 'mp4', - 'title': 'Tunnel Vision (Explicit)', - 'upload_date': '20130703', + 'title': 'Justin Timberlake - Tunnel Vision (Explicit)', 'age_limit': 18, - 'uploader': 'Justin Timberlake', 'timestamp': 1372888800, + 'upload_date': '20130703', + 'uploader': 'Justin Timberlake', + 'track': 'Tunnel Vision (Explicit)', + 'artist': 'Justin Timberlake', + 'genre': 'Pop', }, + 'expected_warnings': ['Unable to download SMIL file'], }, { 'note': 'No video_info', 'url': 'http://www.vevo.com/watch/k-camp-1/Till-I-Die/USUV71503000', @@ -78,11 +90,14 @@ class VevoIE(VevoBaseIE): 'info_dict': { 'id': 'USUV71503000', 'ext': 'mp4', - 'title': 'Till I Die', - 'upload_date': '20151207', + 'title': 'K Camp - Till I Die', 'age_limit': 18, - 'uploader': 'K Camp', 'timestamp': 1449468000, + 'upload_date': '20151207', + 'uploader': 'K Camp', + 'track': 'Till I Die', + 'artist': 'K Camp', + 'genre': 'Rap/Hip-Hop', }, }, { 'note': 'Only available via webpage', @@ -91,11 +106,14 @@ class VevoIE(VevoBaseIE): 'info_dict': { 'id': 'GBUV71600656', 'ext': 'mp4', - 'title': 'Viva Love', - 'upload_date': '20160428', + 'title': 'ABC - Viva Love', 'age_limit': 0, - 'uploader': 'ABC', 'timestamp': 1461830400, + 'upload_date': '20160428', + 'uploader': 'ABC', + 'track': 'Viva Love', + 'artist': 'ABC', + 'genre': 'Pop', }, 'expected_warnings': ['Failed to download video versions info'], }] @@ -184,8 +202,8 @@ class VevoIE(VevoBaseIE): video_info = response.get('video') or {} video_versions = video_info.get('videoVersions') uploader = None - timestamp = None view_count = None + timestamp = None formats = [] if not video_info: @@ -311,7 +329,9 @@ class VevoIE(VevoBaseIE): smil_parsed = True self._sort_formats(formats) - title = video_info['title'] + track = video_info['title'] + title = '%s - %s' % (uploader, track) if uploader else track + genre = video_info.get('genres', [None])[0] is_explicit = video_info.get('isExplicit') if is_explicit is True: @@ -333,6 +353,9 @@ class VevoIE(VevoBaseIE): 'duration': duration, 'view_count': view_count, 'age_limit': age_limit, + 'track': track, + 'artist': uploader, + 'genre': genre, } @@ -359,10 +382,13 @@ class VevoPlaylistIE(VevoBaseIE): 'info_dict': { 'id': 'USCMV1100073', 'ext': 'mp4', - 'title': 'Y.U. MAD', + 'title': 'Birdman - Y.U. MAD', 'timestamp': 1323417600, 'upload_date': '20111209', 'uploader': 'Birdman', + 'track': 'Y.U. MAD', + 'artist': 'Birdman', + 'genre': 'Rap/Hip-Hop', }, 'expected_warnings': ['Unable to download SMIL file'], }, { From 78a3ff33ab686bc6fc75735e2b2a5935b80311be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 May 2016 03:29:48 +0600 Subject: [PATCH 09/24] [vevo:playlist] Add fallback for playlist id --- youtube_dl/extractor/vevo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index 49cb3f479..bbe29fc51 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -423,5 +423,5 @@ class VevoPlaylistIE(VevoBaseIE): for src in playlist['isrcs']] return self.playlist_result( - entries, playlist.get('playlistId'), + entries, playlist.get('playlistId') or playlist_id, playlist.get('name'), playlist.get('description')) From 9508738f9a9d6fd3de2e60cd7ccb4c8631bea6f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 May 2016 03:36:40 +0600 Subject: [PATCH 10/24] [vevo] Extract featured artist --- youtube_dl/extractor/vevo.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index bbe29fc51..63eab4148 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -201,6 +201,8 @@ class VevoIE(VevoBaseIE): json_url, video_id, 'Downloading video info', 'Unable to download info') video_info = response.get('video') or {} video_versions = video_info.get('videoVersions') + artist = None + featured_artist = None uploader = None view_count = None timestamp = None @@ -239,7 +241,7 @@ class VevoIE(VevoBaseIE): timestamp = parse_iso8601(video_info.get('releaseDate')) artists = video_info.get('artists') if artists: - uploader = artists[0]['name'] + artist = uploader = artists[0]['name'] view_count = int_or_none(video_info.get('views', {}).get('total')) for video_version in video_versions: @@ -292,7 +294,11 @@ class VevoIE(VevoBaseIE): scale=1000) artists = video_info.get('mainArtists') if artists: - uploader = artists[0]['artistName'] + artist = uploader = artists[0]['artistName'] + + featured_artists = video_info.get('featuredArtists') + if featured_artists: + featured_artist = featured_artists[0]['artistName'] smil_parsed = False for video_version in video_info['videoVersions']: @@ -330,7 +336,9 @@ class VevoIE(VevoBaseIE): self._sort_formats(formats) track = video_info['title'] - title = '%s - %s' % (uploader, track) if uploader else track + if featured_artist: + artist = '%s ft. %s' % (artist, featured_artist) + title = '%s - %s' % (artist, track) if artist else track genre = video_info.get('genres', [None])[0] is_explicit = video_info.get('isExplicit') From 686cc8963441c37105c0447f31c5ea21405be05a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 May 2016 07:07:35 +0600 Subject: [PATCH 11/24] [discovery] Fix typo --- youtube_dl/extractor/discovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py index 7c554ec14..55853f76f 100644 --- a/youtube_dl/extractor/discovery.py +++ b/youtube_dl/extractor/discovery.py @@ -71,7 +71,7 @@ class DiscoveryIE(InfoExtractor): entries = [] for idx, video_info in enumerate(info['playlist']): - subtitles = [] + subtitles = {} caption_url = video_info.get('captionsUrl') if caption_url: subtitles = { From 5c9ced9504bd2ceb8e55a929124aad2091b23403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 May 2016 18:19:00 +0600 Subject: [PATCH 12/24] [vevo] Improve genre extraction --- youtube_dl/extractor/vevo.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index 63eab4148..3cb0343e6 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -5,6 +5,7 @@ import re from .common import InfoExtractor from ..compat import ( compat_etree_fromstring, + compat_str, compat_urlparse, ) from ..utils import ( @@ -116,6 +117,10 @@ class VevoIE(VevoBaseIE): 'genre': 'Pop', }, 'expected_warnings': ['Failed to download video versions info'], + }, { + # no genres available + 'url': 'http://www.vevo.com/watch/INS171400764', + 'only_matching': True, }] _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com' _SOURCE_TYPES = { @@ -339,7 +344,11 @@ class VevoIE(VevoBaseIE): if featured_artist: artist = '%s ft. %s' % (artist, featured_artist) title = '%s - %s' % (artist, track) if artist else track - genre = video_info.get('genres', [None])[0] + + genres = video_info.get('genres') + genre = ( + genres[0] if genres and isinstance(genres, list) and + isinstance(genres[0], compat_str) else None) is_explicit = video_info.get('isExplicit') if is_explicit is True: From 7960b0563b957d418ddd36555275d98ba4668c03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 May 2016 18:35:50 +0600 Subject: [PATCH 13/24] [YoutubeDL] Properly process unable-to-download-error on python2 --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 055433362..4e57c9687 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1639,7 +1639,7 @@ class YoutubeDL(object): # Just a single file success = dl(filename, info_dict) except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self.report_error('unable to download video data: %s' % str(err)) + self.report_error('unable to download video data: %s' % error_to_compat_str(err)) return except (OSError, IOError) as err: raise UnavailableVideoError(err) From df5f4e8888bc02f6064b9b92fbf4cfc4eedd4c1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 May 2016 18:47:35 +0600 Subject: [PATCH 14/24] [vevo] Remove superfluous code --- youtube_dl/extractor/vevo.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index 3cb0343e6..a6177f2cb 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -205,12 +205,10 @@ class VevoIE(VevoBaseIE): response = self._download_json( json_url, video_id, 'Downloading video info', 'Unable to download info') video_info = response.get('video') or {} - video_versions = video_info.get('videoVersions') artist = None featured_artist = None uploader = None view_count = None - timestamp = None formats = [] if not video_info: From f0e14fdd43bf8e86c5385220430eef842a10ccab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 May 2016 20:05:06 +0600 Subject: [PATCH 15/24] [YoutubeDL] Skip non-relevant field types when building output template --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 4e57c9687..2187dcc8f 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -580,7 +580,7 @@ class YoutubeDL(object): is_id=(k == 'id')) template_dict = dict((k, sanitize(k, v)) for k, v in template_dict.items() - if v is not None) + if v is not None and not isinstance(v, (list, tuple, dict))) template_dict = collections.defaultdict(lambda: 'NA', template_dict) outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL) From 8a92e51c60fd122e676e4619e7e132b692292801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 May 2016 21:31:35 +0600 Subject: [PATCH 16/24] [extractor/common] Relax wording for creator metafield --- youtube_dl/extractor/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 61a5d124c..0843d89af 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -163,7 +163,7 @@ class InfoExtractor(object): description: Full video description. uploader: Full name of the video uploader. license: License name the video is licensed under. - creator: The main artist who created the video. + creator: The creator of the video. release_date: The date (YYYYMMDD) when the video was released. timestamp: UNIX timestamp of the moment the video became available. upload_date: Video upload date (YYYYMMDD). From 6c52a86f54b230a3f08dd10a89f55b8af4d98ee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Mon, 2 May 2016 21:32:57 +0600 Subject: [PATCH 17/24] [README.md] Update creator description --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ecf737047..50acb26a0 100644 --- a/README.md +++ b/README.md @@ -465,7 +465,7 @@ The basic usage is not to set any template arguments when downloading a single f - `display_id`: An alternative identifier for the video - `uploader`: Full name of the video uploader - `license`: License name the video is licensed under - - `creator`: The main artist who created the video + - `creator`: The creator of the video - `release_date`: The date (YYYYMMDD) when the video was released - `timestamp`: UNIX timestamp of the moment the video became available - `upload_date`: Video upload date (YYYYMMDD) From a0a309b9734ea3d5f66d4a52e42f5cc24df7f808 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Tue, 3 May 2016 16:06:28 +0800 Subject: [PATCH 18/24] [kuwo:category] Fix description and update test --- youtube_dl/extractor/kuwo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py index 3740869c7..616ed19e1 100644 --- a/youtube_dl/extractor/kuwo.py +++ b/youtube_dl/extractor/kuwo.py @@ -266,7 +266,6 @@ class KuwoCategoryIE(InfoExtractor): 'info_dict': { 'id': '86375', 'title': '八十年代精选', - 'description': '这些都是属于八十年代的回忆！', }, 'playlist_mincount': 24, } @@ -283,6 +282,8 @@ class KuwoCategoryIE(InfoExtractor): category_desc = remove_start( get_element_by_id('intro', webpage).strip(), '%s简介：' % category_name) + if category_desc == '暂无': + category_desc = None jsonm = self._parse_json(self._html_search_regex( r'var\s+jsonm\s*=\s*([^;]+);', webpage, 'category songs'), category_id) From 7759be38da1b5ad99a8ef04d9a1df21b989e3b8c Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Tue, 3 May 2016 16:19:20 +0800 Subject: [PATCH 19/24] [xiami] Detect georestriction and skip tests --- youtube_dl/extractor/xiami.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/xiami.py b/youtube_dl/extractor/xiami.py index e4ed306b4..a6dfc4af9 100644 --- a/youtube_dl/extractor/xiami.py +++ b/youtube_dl/extractor/xiami.py @@ -9,6 +9,11 @@ from ..utils import int_or_none class XiamiBaseIE(InfoExtractor): _API_BASE_URL = 'http://www.xiami.com/song/playlist/cat/json/id' + def _download_webpage(self, *args, **kwargs): + webpage = super(XiamiBaseIE, self)._download_webpage(*args, **kwargs) + if '>Xiami is currently not available in your country.<' in webpage: + self.raise_geo_restricted('Xiami is currently not available in your country') + def _extract_track(self, track, track_id=None): title = track['title'] track_url = self._decrypt(track['location']) @@ -81,7 +86,8 @@ class XiamiSongIE(XiamiBaseIE): 'ext': 'lrc', }], }, - } + }, + 'skip': 'Georestricted', }, { 'url': 'http://www.xiami.com/song/1775256504', 'md5': '932a3abd45c6aa2b1fdbe028fcb4c4fc', @@ -100,7 +106,8 @@ class XiamiSongIE(XiamiBaseIE): 'ext': 'lrc', }], }, - } + }, + 'skip': 'Georestricted', }] def _real_extract(self, url): @@ -124,6 +131,7 @@ class XiamiAlbumIE(XiamiPlaylistBaseIE): 'id': '2100300444', }, 'playlist_count': 10, + 'skip': 'Georestricted', }, { 'url': 'http://www.xiami.com/album/512288?spm=a1z1s.6843761.1110925389.6.hhE9p9', 'only_matching': True, @@ -141,6 +149,7 @@ class XiamiArtistIE(XiamiPlaylistBaseIE): 'id': '2132', }, 'playlist_count': 20, + 'skip': 'Georestricted', } @@ -155,4 +164,5 @@ class XiamiCollectionIE(XiamiPlaylistBaseIE): 'id': '156527391', }, 'playlist_mincount': 29, + 'skip': 'Georestricted', } From 80bc4106aff21c46cf65c7310f1ed988518b3df5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 3 May 2016 15:09:23 +0600 Subject: [PATCH 20/24] [xfileshare] Add support for thevideobee.to (Closes #9374) --- youtube_dl/extractor/xfileshare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py index 2d1504eaa..472ed1b3f 100644 --- a/youtube_dl/extractor/xfileshare.py +++ b/youtube_dl/extractor/xfileshare.py @@ -16,7 +16,7 @@ class XFileShareIE(InfoExtractor): IE_DESC = 'XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me' _VALID_URL = r'''(?x) https?://(?P(?:www\.)? - (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto\.me|powerwatch\.pw))/ + (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto\.me|powerwatch\.pw|thevideobee\.to))/ (?:embed-)?(?P[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)? ''' From 41745523918dee174a3e642b629bcbc585931c4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 3 May 2016 15:35:32 +0600 Subject: [PATCH 21/24] [xfileshare] Refactor _VALID_URL and remove ded sites --- youtube_dl/extractor/xfileshare.py | 39 +++++++++++------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py index 472ed1b3f..4ab42d24e 100644 --- a/youtube_dl/extractor/xfileshare.py +++ b/youtube_dl/extractor/xfileshare.py @@ -13,12 +13,20 @@ from ..utils import ( class XFileShareIE(InfoExtractor): - IE_DESC = 'XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me' - _VALID_URL = r'''(?x) - https?://(?P(?:www\.)? - (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto\.me|powerwatch\.pw|thevideobee\.to))/ - (?:embed-)?(?P[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)? - ''' + _SITES = ( + ('daclips.in', 'DaClips'), + ('filehoot.com', 'FileHoot'), + ('gorillavid.in', 'GorillaVid'), + ('movpod.in', 'MovPod'), + ('powerwatch.pw', 'PowerWatch'), + ('rapidvideo.ws', 'Rapidvideo.ws'), + ('thevideobee.to', 'TheVideoBee'), + ('vidto.me', 'Vidto'), + ) + + IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1]) + _VALID_URL = (r'https?://(?P(?:www\.)?(?:%s))/(?:embed-)?(?P[0-9a-zA-Z]+)' + % '|'.join(re.escape(site) for site in list(zip(*_SITES))[0])) _FILE_NOT_FOUND_REGEX = r'>(?:404 - )?File Not Found<' @@ -43,25 +51,6 @@ class XFileShareIE(InfoExtractor): 'title': 'Micro Pig piglets ready on 16th July 2009-bG0PdrCdxUc', 'thumbnail': 're:http://.*\.jpg', } - }, { - # video with countdown timeout - 'url': 'http://fastvideo.in/1qmdn1lmsmbw', - 'md5': '8b87ec3f6564a3108a0e8e66594842ba', - 'info_dict': { - 'id': '1qmdn1lmsmbw', - 'ext': 'mp4', - 'title': 'Man of Steel - Trailer', - 'thumbnail': 're:http://.*\.jpg', - }, - }, { - 'url': 'http://realvid.net/ctn2y6p2eviw', - 'md5': 'b2166d2cf192efd6b6d764c18fd3710e', - 'info_dict': { - 'id': 'ctn2y6p2eviw', - 'ext': 'flv', - 'title': 'rdx 1955', - 'thumbnail': 're:http://.*\.jpg', - }, }, { 'url': 'http://movpod.in/0wguyyxi1yca', 'only_matching': True, From 57d8e32a3ec7fe70522edad6fd0c2847b4e00944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 3 May 2016 16:58:11 +0600 Subject: [PATCH 22/24] [xfileshare] Add support for streamin.to --- youtube_dl/extractor/xfileshare.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py index 4ab42d24e..769003735 100644 --- a/youtube_dl/extractor/xfileshare.py +++ b/youtube_dl/extractor/xfileshare.py @@ -22,6 +22,7 @@ class XFileShareIE(InfoExtractor): ('rapidvideo.ws', 'Rapidvideo.ws'), ('thevideobee.to', 'TheVideoBee'), ('vidto.me', 'Vidto'), + ('streamin.to', 'Streamin.To'), ) IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1]) From 964f49336fcf94b3a5399a026db3914b27a2a445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 3 May 2016 21:24:51 +0600 Subject: [PATCH 23/24] [aol] Improve _VALID_URL (Closes #9381) --- youtube_dl/extractor/aol.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py index 24df8fe93..42c21bf41 100644 --- a/youtube_dl/extractor/aol.py +++ b/youtube_dl/extractor/aol.py @@ -12,7 +12,7 @@ from ..utils import ( class AolIE(InfoExtractor): IE_NAME = 'on.aol.com' - _VALID_URL = r'(?:aol-video:|https?://on\.aol\.com/.*-)(?P[^/?-]+)' + _VALID_URL = r'(?:aol-video:|https?://on\.aol\.com/(?:[^/]+/)*(?:[^/?#&]+-)?)(?P[^/?#&]+)' _TESTS = [{ # video with 5min ID @@ -53,6 +53,12 @@ class AolIE(InfoExtractor): }, { 'url': 'http://on.aol.com/shows/park-bench-shw518173474-559a1b9be4b0c3bfad3357a7?context=SH:SHW518173474:PL4327:1460619712763', 'only_matching': True, + }, { + 'url': 'http://on.aol.com/video/519442220', + 'only_matching': True, + }, { + 'url': 'aol-video:5707d6b8e4b090497b04f706', + 'only_matching': True, }] def _real_extract(self, url): From bc7e77a04be6094e64263f9c622cff3cd1fc13cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 3 May 2016 23:18:36 +0600 Subject: [PATCH 24/24] [vevo] Use raise_geo_restricted --- youtube_dl/extractor/vevo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index a6177f2cb..c0ef08c02 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -189,8 +189,8 @@ class VevoIE(VevoBaseIE): errnote='Unable to retrieve oauth token') if 'THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION' in webpage: - raise ExtractorError( - '%s said: This page is currently unavailable in your region.' % self.IE_NAME, expected=True) + self.raise_geo_restricted( + '%s said: This page is currently unavailable in your region' % self.IE_NAME) auth_info = self._parse_json(webpage, video_id) self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['access_token']