From 580693d3483fbd1455a33b16b5d9e633398de0cc Mon Sep 17 00:00:00 2001 From: Parmjit Virk Date: Wed, 12 Jul 2017 17:02:58 -0500 Subject: [PATCH 1/7] [GoogleDrive] Added support for captions --- youtube_dl/extractor/googledrive.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py index c40da85c5..b24be85f0 100644 --- a/youtube_dl/extractor/googledrive.py +++ b/youtube_dl/extractor/googledrive.py @@ -44,6 +44,10 @@ class GoogleDriveIE(InfoExtractor): '46': 'webm', '59': 'mp4', } + _SUBTITLE_FORMATS_EXT = ( + 'vtt', + 'ttml' + ) @staticmethod def _extract_url(webpage): @@ -97,10 +101,31 @@ class GoogleDriveIE(InfoExtractor): formats.append(f) self._sort_formats(formats) + hl = self._search_regex( + r'"hl"\s*,\s*"([^"]+)', webpage, 'hl') + ttsurl = self._search_regex( + r'"ttsurl"\s*,\s*"([^"]+)', webpage, 'ttsurl').encode('utf-8').decode('unicode_escape') + # the video Id for subtitles will be the last value in the ttsurl query string + video_subtitles_id = ttsurl.split('=')[-1] + + subtitles = {} + subtitles_by_country = self._download_xml('https://drive.google.com/timedtext?id=%s&vid=%s&hl=%s&type=list&tlangs=1&v=%s&fmts=1&vssids=1' % (video_id, video_subtitles_id, hl, video_id), video_id) + subtitle_available_tracks = subtitles_by_country.findall('track') + for subtitle_track in subtitle_available_tracks: + subtitle_lang_code = subtitle_track.attrib['lang_code'] + subtitle_format_data = [] + for subtitle_format in self._SUBTITLE_FORMATS_EXT: + subtitle_format_data.append({ + 'url': 'https://drive.google.com/timedtext?vid=%s&v=%s&type=track&lang=%s&name&kind&fmt=%s' % (video_subtitles_id, video_id, subtitle_lang_code, subtitle_format), + 'ext': subtitle_format, + }) + subtitles[subtitle_lang_code] = subtitle_format_data + return { 'id': video_id, 'title': title, 'thumbnail': self._og_search_thumbnail(webpage, default=None), 'duration': duration, 'formats': formats, + 'subtitles': subtitles, } From 9870e1b51be16e079105d77f605a7d344be434a6 Mon Sep 17 00:00:00 2001 From: Parmjit Virk Date: Wed, 12 Jul 2017 17:18:29 -0500 Subject: [PATCH 2/7] [GoogleDrive] Updated test suite --- youtube_dl/extractor/googledrive.py | 59 ++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py index b24be85f0..21ad71953 100644 --- a/youtube_dl/extractor/googledrive.py +++ b/youtube_dl/extractor/googledrive.py @@ -24,7 +24,64 @@ class GoogleDriveIE(InfoExtractor): }, { # video id is longer than 28 characters 'url': 'https://drive.google.com/file/d/1ENcQ_jeCuj7y19s66_Ou9dRP4GKGsodiDQ/edit', - 'only_matching': True, + 'md5': 'c230c67252874fddd8170e3fd1a45886', + 'info_dict': { + 'id': '1ENcQ_jeCuj7y19s66_Ou9dRP4GKGsodiDQ', + 'ext': 'mp4', + 'title': 'Andreea Banica feat Smiley - Hooky Song (Official Video).mp4', + 'duration': 189, + }, + 'params': { + 'only_matching': True, + } + }, { + 'url': 'https://drive.google.com/file/d/0B0Pcx5-LUToEblNxcXRPWmtmS00/view', + 'md5': '3085012a025cf683c67f12010626d897', + 'info_dict': { + 'id': '0B0Pcx5-LUToEblNxcXRPWmtmS00', + 'ext': 'mp4', + 'title': 'bbo20wmv.mp4', + 'duration': 5888, + 'subtitles' : 'mincount:1', + } + }, { + 'url': 'https://drive.google.com/file/d/0B7uqFWpTkjfcVi1RaXp1OW1ycFE/edit', + 'md5': 'eab2f1adacf1b283362a9ce28d947db2', + 'info_dict': { + 'id': '0B7uqFWpTkjfcVi1RaXp1OW1ycFE', + 'ext': 'mp4', + 'title': '[18] 101 Dalmatians (1961)', + 'duration': 4756, + 'subtitles' : 'mincount:1', + }, + 'params': { + 'only_matching': True, + } + }, { + 'url': 'https://drive.google.com/file/d/0B7klLRUbm38_cXdENHY1cFlHenM/preview', + 'md5': '0eabf6a71f61469b6b4df1c93c57b669', + 'info_dict': { + 'id': '0B7klLRUbm38_cXdENHY1cFlHenM', + 'ext': 'mp4', + 'title': '101 Dalmatians 1961 1080p BluRay x264 AC3 - Ozlem.mp4', + 'duration': 4755, + 'subtitles' : 'mincount:1', + }, + 'params': { + 'only_matching': True, + } + }, { + 'url': 'https://drive.google.com/file/d/0B_Cc_rAzzVX9blZZZzZJX2tNYVk/edit', + 'md5': 'a5dd2666a9ee9da523b7a1a6f6c916ae', + 'info_dict': { + 'id': '0B_Cc_rAzzVX9blZZZzZJX2tNYVk', + 'ext': 'mp4', + 'title': 'Buffy 5x04 - Out of My Mind.mkv', + 'duration': 2548, + }, + 'params': { + 'only_matching': True, + } }] _FORMATS_EXT = { '5': 'flv', From 61397c1973ce20da92463cf29c6b3b2e6c358a80 Mon Sep 17 00:00:00 2001 From: Parmjit Virk Date: Wed, 12 Jul 2017 17:21:44 -0500 Subject: [PATCH 3/7] [GoogleDrive] flake8 corrections --- youtube_dl/extractor/googledrive.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py index 21ad71953..d46f27627 100644 --- a/youtube_dl/extractor/googledrive.py +++ b/youtube_dl/extractor/googledrive.py @@ -32,7 +32,7 @@ class GoogleDriveIE(InfoExtractor): 'duration': 189, }, 'params': { - 'only_matching': True, + 'only_matching': True, } }, { 'url': 'https://drive.google.com/file/d/0B0Pcx5-LUToEblNxcXRPWmtmS00/view', @@ -42,7 +42,7 @@ class GoogleDriveIE(InfoExtractor): 'ext': 'mp4', 'title': 'bbo20wmv.mp4', 'duration': 5888, - 'subtitles' : 'mincount:1', + 'subtitles': 'mincount:1', } }, { 'url': 'https://drive.google.com/file/d/0B7uqFWpTkjfcVi1RaXp1OW1ycFE/edit', @@ -52,10 +52,10 @@ class GoogleDriveIE(InfoExtractor): 'ext': 'mp4', 'title': '[18] 101 Dalmatians (1961)', 'duration': 4756, - 'subtitles' : 'mincount:1', + 'subtitles': 'mincount:1', }, 'params': { - 'only_matching': True, + 'only_matching': True, } }, { 'url': 'https://drive.google.com/file/d/0B7klLRUbm38_cXdENHY1cFlHenM/preview', @@ -65,10 +65,10 @@ class GoogleDriveIE(InfoExtractor): 'ext': 'mp4', 'title': '101 Dalmatians 1961 1080p BluRay x264 AC3 - Ozlem.mp4', 'duration': 4755, - 'subtitles' : 'mincount:1', + 'subtitles': 'mincount:1', }, 'params': { - 'only_matching': True, + 'only_matching': True, } }, { 'url': 'https://drive.google.com/file/d/0B_Cc_rAzzVX9blZZZzZJX2tNYVk/edit', @@ -80,7 +80,7 @@ class GoogleDriveIE(InfoExtractor): 'duration': 2548, }, 'params': { - 'only_matching': True, + 'only_matching': True, } }] _FORMATS_EXT = { @@ -102,8 +102,8 @@ class GoogleDriveIE(InfoExtractor): '59': 'mp4', } _SUBTITLE_FORMATS_EXT = ( - 'vtt', - 'ttml' + 'vtt', + 'ttml' ) @staticmethod From 9a2158f7e884d51e3403d429d21f48f5b3809031 Mon Sep 17 00:00:00 2001 From: Parmjit Virk Date: Thu, 13 Jul 2017 02:40:42 -0500 Subject: [PATCH 4/7] [GoogleDrive] Applied requested fixes for PR #13638 --- youtube_dl/extractor/googledrive.py | 94 ++++++++--------------------- 1 file changed, 25 insertions(+), 69 deletions(-) diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py index d46f27627..f9b9c47b3 100644 --- a/youtube_dl/extractor/googledrive.py +++ b/youtube_dl/extractor/googledrive.py @@ -31,57 +31,7 @@ class GoogleDriveIE(InfoExtractor): 'title': 'Andreea Banica feat Smiley - Hooky Song (Official Video).mp4', 'duration': 189, }, - 'params': { - 'only_matching': True, - } - }, { - 'url': 'https://drive.google.com/file/d/0B0Pcx5-LUToEblNxcXRPWmtmS00/view', - 'md5': '3085012a025cf683c67f12010626d897', - 'info_dict': { - 'id': '0B0Pcx5-LUToEblNxcXRPWmtmS00', - 'ext': 'mp4', - 'title': 'bbo20wmv.mp4', - 'duration': 5888, - 'subtitles': 'mincount:1', - } - }, { - 'url': 'https://drive.google.com/file/d/0B7uqFWpTkjfcVi1RaXp1OW1ycFE/edit', - 'md5': 'eab2f1adacf1b283362a9ce28d947db2', - 'info_dict': { - 'id': '0B7uqFWpTkjfcVi1RaXp1OW1ycFE', - 'ext': 'mp4', - 'title': '[18] 101 Dalmatians (1961)', - 'duration': 4756, - 'subtitles': 'mincount:1', - }, - 'params': { - 'only_matching': True, - } - }, { - 'url': 'https://drive.google.com/file/d/0B7klLRUbm38_cXdENHY1cFlHenM/preview', - 'md5': '0eabf6a71f61469b6b4df1c93c57b669', - 'info_dict': { - 'id': '0B7klLRUbm38_cXdENHY1cFlHenM', - 'ext': 'mp4', - 'title': '101 Dalmatians 1961 1080p BluRay x264 AC3 - Ozlem.mp4', - 'duration': 4755, - 'subtitles': 'mincount:1', - }, - 'params': { - 'only_matching': True, - } - }, { - 'url': 'https://drive.google.com/file/d/0B_Cc_rAzzVX9blZZZzZJX2tNYVk/edit', - 'md5': 'a5dd2666a9ee9da523b7a1a6f6c916ae', - 'info_dict': { - 'id': '0B_Cc_rAzzVX9blZZZzZJX2tNYVk', - 'ext': 'mp4', - 'title': 'Buffy 5x04 - Out of My Mind.mkv', - 'duration': 2548, - }, - 'params': { - 'only_matching': True, - } + 'only_matching': True }] _FORMATS_EXT = { '5': 'flv', @@ -114,6 +64,23 @@ class GoogleDriveIE(InfoExtractor): if mobj: return 'https://drive.google.com/file/d/%s' % mobj.group('id') + def _get_subtitles(self, video_id, video_subtitles_id, hl): + if not video_subtitles_id or not hl: + return None + subtitles = {} + subtitles_by_country = self._download_xml('https://drive.google.com/timedtext?id=%s&vid=%s&hl=%s&type=list&tlangs=1&v=%s&fmts=1&vssids=1' % (video_id, video_subtitles_id, hl, video_id), video_id) + subtitle_available_tracks = subtitles_by_country.findall('track') + for subtitle_track in subtitle_available_tracks: + subtitle_lang_code = subtitle_track.attrib['lang_code'] + subtitle_format_data = [] + for subtitle_format in self._SUBTITLE_FORMATS_EXT: + subtitle_format_data.append({ + 'url': 'https://drive.google.com/timedtext?vid=%s&v=%s&type=track&lang=%s&name&kind&fmt=%s' % (video_subtitles_id, video_id, subtitle_lang_code, subtitle_format), + 'ext': subtitle_format, + }) + subtitles[subtitle_lang_code] = subtitle_format_data + return subtitles + def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage( @@ -159,24 +126,13 @@ class GoogleDriveIE(InfoExtractor): self._sort_formats(formats) hl = self._search_regex( - r'"hl"\s*,\s*"([^"]+)', webpage, 'hl') + r'"hl"\s*,\s*"([^"]+)', webpage, 'hl', default=None) + video_subtitles_id = None ttsurl = self._search_regex( - r'"ttsurl"\s*,\s*"([^"]+)', webpage, 'ttsurl').encode('utf-8').decode('unicode_escape') - # the video Id for subtitles will be the last value in the ttsurl query string - video_subtitles_id = ttsurl.split('=')[-1] - - subtitles = {} - subtitles_by_country = self._download_xml('https://drive.google.com/timedtext?id=%s&vid=%s&hl=%s&type=list&tlangs=1&v=%s&fmts=1&vssids=1' % (video_id, video_subtitles_id, hl, video_id), video_id) - subtitle_available_tracks = subtitles_by_country.findall('track') - for subtitle_track in subtitle_available_tracks: - subtitle_lang_code = subtitle_track.attrib['lang_code'] - subtitle_format_data = [] - for subtitle_format in self._SUBTITLE_FORMATS_EXT: - subtitle_format_data.append({ - 'url': 'https://drive.google.com/timedtext?vid=%s&v=%s&type=track&lang=%s&name&kind&fmt=%s' % (video_subtitles_id, video_id, subtitle_lang_code, subtitle_format), - 'ext': subtitle_format, - }) - subtitles[subtitle_lang_code] = subtitle_format_data + r'"ttsurl"\s*,\s*"([^"]+)', webpage, 'ttsurl', default=None) + if ttsurl: + # the video Id for subtitles will be the last value in the ttsurl query string + video_subtitles_id = ttsurl.encode('utf-8').decode('unicode_escape').split('=')[-1] return { 'id': video_id, @@ -184,5 +140,5 @@ class GoogleDriveIE(InfoExtractor): 'thumbnail': self._og_search_thumbnail(webpage, default=None), 'duration': duration, 'formats': formats, - 'subtitles': subtitles, + 'subtitles': self.extract_subtitles(video_id, video_subtitles_id, hl), } From a6808fd37d56604b052b07d1944685d245c785e4 Mon Sep 17 00:00:00 2001 From: Parmjit Virk Date: Tue, 1 Aug 2017 16:54:58 -0500 Subject: [PATCH 5/7] [GoogleDrive] Applied second requested fixes for PR #13638 --- youtube_dl/extractor/googledrive.py | 92 ++++++++++++++++++++++++++--- 1 file changed, 84 insertions(+), 8 deletions(-) diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py index f9b9c47b3..0e5208fe4 100644 --- a/youtube_dl/extractor/googledrive.py +++ b/youtube_dl/extractor/googledrive.py @@ -7,6 +7,8 @@ from ..utils import ( ExtractorError, int_or_none, lowercase_escape, + error_to_compat_str, + update_url_query, ) @@ -51,10 +53,8 @@ class GoogleDriveIE(InfoExtractor): '46': 'webm', '59': 'mp4', } - _SUBTITLE_FORMATS_EXT = ( - 'vtt', - 'ttml' - ) + _CAPTION_FORMATS_EXT = [] + _CAPTIONS_BY_COUNTRY_XML = None @staticmethod def _extract_url(webpage): @@ -64,23 +64,98 @@ class GoogleDriveIE(InfoExtractor): if mobj: return 'https://drive.google.com/file/d/%s' % mobj.group('id') + def _set_captions_data(self, video_id, video_subtitles_id, hl): + try: + self._CAPTIONS_BY_COUNTRY_XML = self._download_xml( + 'https://drive.google.com/timedtext?id=%s&vid=%s&hl=%s&type=list&tlangs=1&v=%s&fmts=1&vssids=1', video_id, query={ + 'id': video_id, + 'vid': video_subtitles_id, + 'hl': hl, + 'v': video_id, + }) + except ExtractorError as ee: + self.report_warning('unable to download video subtitles: %s' % error_to_compat_str(ee)) + if self._CAPTIONS_BY_COUNTRY_XML is not None: + caption_available_extensions = self._CAPTIONS_BY_COUNTRY_XML.findall('format') + for caption_extension in caption_available_extensions: + if caption_extension.attrib.get('fmt_code') and not caption_extension.attrib.get('default'): + self._CAPTION_FORMATS_EXT.append(caption_extension.attrib['fmt_code']) + def _get_subtitles(self, video_id, video_subtitles_id, hl): if not video_subtitles_id or not hl: return None + if self._CAPTIONS_BY_COUNTRY_XML is None: + self._set_captions_data(video_id, video_subtitles_id, hl) + if self._CAPTIONS_BY_COUNTRY_XML is None: + return None + subtitles = {} - subtitles_by_country = self._download_xml('https://drive.google.com/timedtext?id=%s&vid=%s&hl=%s&type=list&tlangs=1&v=%s&fmts=1&vssids=1' % (video_id, video_subtitles_id, hl, video_id), video_id) - subtitle_available_tracks = subtitles_by_country.findall('track') + subtitle_available_tracks = self._CAPTIONS_BY_COUNTRY_XML.findall('track') for subtitle_track in subtitle_available_tracks: + if not subtitle_track.attrib.get('lang_code'): + continue subtitle_lang_code = subtitle_track.attrib['lang_code'] subtitle_format_data = [] - for subtitle_format in self._SUBTITLE_FORMATS_EXT: + for subtitle_format in self._CAPTION_FORMATS_EXT: + query = { + 'vid': video_subtitles_id, + 'v': video_id, + 'lang': subtitle_lang_code, + 'fmt': subtitle_format, + 'name': '', + 'kind': '', + } subtitle_format_data.append({ - 'url': 'https://drive.google.com/timedtext?vid=%s&v=%s&type=track&lang=%s&name&kind&fmt=%s' % (video_subtitles_id, video_id, subtitle_lang_code, subtitle_format), + 'url': update_url_query('https://drive.google.com/timedtext?vid=%s&v=%s&type=track&lang=%s&name&kind&fmt=%s', query), 'ext': subtitle_format, }) subtitles[subtitle_lang_code] = subtitle_format_data + if not subtitles: + self.report_warning('video doesn\'t have subtitles') return subtitles + def _get_automatic_captions(self, video_id, video_subtitles_id, hl): + if not video_subtitles_id or not hl: + return None + if self._CAPTIONS_BY_COUNTRY_XML is None: + self._set_captions_data(video_id, video_subtitles_id, hl) + if self._CAPTIONS_BY_COUNTRY_XML is None: + return None + self.to_screen('%s: Looking for automatic captions' % video_id) + + subtitle_original_track = self._CAPTIONS_BY_COUNTRY_XML.find('track') + if subtitle_original_track is None: + return None + if not subtitle_original_track.attrib.get('lang_code'): + return None + subtitle_original_lang_code = subtitle_original_track.attrib['lang_code'] + + automatic_captions = {} + automatic_caption_available_targets = self._CAPTIONS_BY_COUNTRY_XML.findall('target') + for automatic_caption_target in automatic_caption_available_targets: + if not automatic_caption_target.attrib.get('lang_code'): + continue + automatic_caption_lang_code = automatic_caption_target.attrib['lang_code'] + automatic_caption_format_data = [] + for automatic_caption_format in self._CAPTION_FORMATS_EXT: + query = { + 'vid': video_subtitles_id, + 'v': video_id, + 'lang': subtitle_original_lang_code, + 'fmt': automatic_caption_format, + 'tlang': automatic_caption_lang_code, + 'name': '', + 'kind': '', + } + automatic_caption_format_data.append({ + 'url': update_url_query('https://drive.google.com/timedtext?vid=%s&v=%s&type=track&lang=%s&name&kind&fmt=%s&tlang=%s', query), + 'ext': automatic_caption_format, + }) + automatic_captions[automatic_caption_lang_code] = automatic_caption_format_data + if not automatic_captions: + self.report_warning('video doesn\'t have automatic captions') + return automatic_captions + def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage( @@ -141,4 +216,5 @@ class GoogleDriveIE(InfoExtractor): 'duration': duration, 'formats': formats, 'subtitles': self.extract_subtitles(video_id, video_subtitles_id, hl), + 'automatic_captions': self.extract_automatic_captions(video_id, video_subtitles_id, hl), } From f27b2abec19725baa8435275a81a64fb6eaeff41 Mon Sep 17 00:00:00 2001 From: Parmjit Virk Date: Sun, 6 Aug 2017 01:34:13 -0500 Subject: [PATCH 6/7] [GoogleDrive] Applied third requested fixes for PR #13638 --- youtube_dl/extractor/googledrive.py | 50 ++++++++++++++--------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py index 0e5208fe4..2b382efa3 100644 --- a/youtube_dl/extractor/googledrive.py +++ b/youtube_dl/extractor/googledrive.py @@ -53,8 +53,8 @@ class GoogleDriveIE(InfoExtractor): '46': 'webm', '59': 'mp4', } - _CAPTION_FORMATS_EXT = [] - _CAPTIONS_BY_COUNTRY_XML = None + _caption_formats_ext = [] + _captions_by_country_xml = None @staticmethod def _extract_url(webpage): @@ -66,7 +66,7 @@ class GoogleDriveIE(InfoExtractor): def _set_captions_data(self, video_id, video_subtitles_id, hl): try: - self._CAPTIONS_BY_COUNTRY_XML = self._download_xml( + self._captions_by_country_xml = self._download_xml( 'https://drive.google.com/timedtext?id=%s&vid=%s&hl=%s&type=list&tlangs=1&v=%s&fmts=1&vssids=1', video_id, query={ 'id': video_id, 'vid': video_subtitles_id, @@ -75,38 +75,38 @@ class GoogleDriveIE(InfoExtractor): }) except ExtractorError as ee: self.report_warning('unable to download video subtitles: %s' % error_to_compat_str(ee)) - if self._CAPTIONS_BY_COUNTRY_XML is not None: - caption_available_extensions = self._CAPTIONS_BY_COUNTRY_XML.findall('format') + if self._captions_by_country_xml is not None: + caption_available_extensions = self._captions_by_country_xml.findall('format') for caption_extension in caption_available_extensions: if caption_extension.attrib.get('fmt_code') and not caption_extension.attrib.get('default'): - self._CAPTION_FORMATS_EXT.append(caption_extension.attrib['fmt_code']) + self._caption_formats_ext.append(caption_extension.attrib['fmt_code']) def _get_subtitles(self, video_id, video_subtitles_id, hl): if not video_subtitles_id or not hl: return None - if self._CAPTIONS_BY_COUNTRY_XML is None: + if self._captions_by_country_xml is None: self._set_captions_data(video_id, video_subtitles_id, hl) - if self._CAPTIONS_BY_COUNTRY_XML is None: + if self._captions_by_country_xml is None: return None subtitles = {} - subtitle_available_tracks = self._CAPTIONS_BY_COUNTRY_XML.findall('track') - for subtitle_track in subtitle_available_tracks: - if not subtitle_track.attrib.get('lang_code'): + for subtitle_track in self._captions_by_country_xml.findall('track'): + subtitle_lang_code = subtitle_track.attrib.get('lang_code') + if not subtitle_lang_code: continue - subtitle_lang_code = subtitle_track.attrib['lang_code'] subtitle_format_data = [] - for subtitle_format in self._CAPTION_FORMATS_EXT: + for subtitle_format in self._caption_formats_ext: query = { 'vid': video_subtitles_id, 'v': video_id, 'lang': subtitle_lang_code, 'fmt': subtitle_format, + 'type': 'track', 'name': '', 'kind': '', } subtitle_format_data.append({ - 'url': update_url_query('https://drive.google.com/timedtext?vid=%s&v=%s&type=track&lang=%s&name&kind&fmt=%s', query), + 'url': update_url_query('https://drive.google.com/timedtext', query), 'ext': subtitle_format, }) subtitles[subtitle_lang_code] = subtitle_format_data @@ -117,38 +117,38 @@ class GoogleDriveIE(InfoExtractor): def _get_automatic_captions(self, video_id, video_subtitles_id, hl): if not video_subtitles_id or not hl: return None - if self._CAPTIONS_BY_COUNTRY_XML is None: + if self._captions_by_country_xml is None: self._set_captions_data(video_id, video_subtitles_id, hl) - if self._CAPTIONS_BY_COUNTRY_XML is None: + if self._captions_by_country_xml is None: return None self.to_screen('%s: Looking for automatic captions' % video_id) - subtitle_original_track = self._CAPTIONS_BY_COUNTRY_XML.find('track') + subtitle_original_track = self._captions_by_country_xml.find('track') if subtitle_original_track is None: return None - if not subtitle_original_track.attrib.get('lang_code'): + subtitle_original_lang_code = subtitle_original_track.attrib.get('lang_code') + if not subtitle_original_lang_code: return None - subtitle_original_lang_code = subtitle_original_track.attrib['lang_code'] automatic_captions = {} - automatic_caption_available_targets = self._CAPTIONS_BY_COUNTRY_XML.findall('target') - for automatic_caption_target in automatic_caption_available_targets: - if not automatic_caption_target.attrib.get('lang_code'): + for automatic_caption_target in self._captions_by_country_xml.findall('target'): + automatic_caption_lang_code = automatic_caption_target.attrib.get('lang_code') + if not automatic_caption_lang_code: continue - automatic_caption_lang_code = automatic_caption_target.attrib['lang_code'] automatic_caption_format_data = [] - for automatic_caption_format in self._CAPTION_FORMATS_EXT: + for automatic_caption_format in self._caption_formats_ext: query = { 'vid': video_subtitles_id, 'v': video_id, 'lang': subtitle_original_lang_code, 'fmt': automatic_caption_format, 'tlang': automatic_caption_lang_code, + 'type': 'track', 'name': '', 'kind': '', } automatic_caption_format_data.append({ - 'url': update_url_query('https://drive.google.com/timedtext?vid=%s&v=%s&type=track&lang=%s&name&kind&fmt=%s&tlang=%s', query), + 'url': update_url_query('https://drive.google.com/timedtext', query), 'ext': automatic_caption_format, }) automatic_captions[automatic_caption_lang_code] = automatic_caption_format_data From da8804fcdcd4a27c571b6aaa33af6a462e109abb Mon Sep 17 00:00:00 2001 From: Parmjit Virk Date: Sat, 12 Aug 2017 23:40:25 -0500 Subject: [PATCH 7/7] [GoogleDrive] Applied fourth requested fixes for PR #13638 --- youtube_dl/extractor/googledrive.py | 106 ++++++++++++---------------- 1 file changed, 47 insertions(+), 59 deletions(-) diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py index 2b382efa3..35edc7440 100644 --- a/youtube_dl/extractor/googledrive.py +++ b/youtube_dl/extractor/googledrive.py @@ -53,6 +53,11 @@ class GoogleDriveIE(InfoExtractor): '46': 'webm', '59': 'mp4', } + _BASE_URL_CAPTIONS = 'https://drive.google.com/timedtext' + _CAPTIONS_ENTRY_TAG = { + 'subtitles': 'track', + 'automatic_captions': 'target', + } _caption_formats_ext = [] _captions_by_country_xml = None @@ -66,13 +71,16 @@ class GoogleDriveIE(InfoExtractor): def _set_captions_data(self, video_id, video_subtitles_id, hl): try: - self._captions_by_country_xml = self._download_xml( - 'https://drive.google.com/timedtext?id=%s&vid=%s&hl=%s&type=list&tlangs=1&v=%s&fmts=1&vssids=1', video_id, query={ - 'id': video_id, - 'vid': video_subtitles_id, - 'hl': hl, - 'v': video_id, - }) + self._captions_by_country_xml = self._download_xml(self._BASE_URL_CAPTIONS, video_id, query={ + 'id': video_id, + 'vid': video_subtitles_id, + 'hl': hl, + 'v': video_id, + 'type': 'list', + 'tlangs': '1', + 'fmts': '1', + 'vssids': '1', + }) except ExtractorError as ee: self.report_warning('unable to download video subtitles: %s' % error_to_compat_str(ee)) if self._captions_by_country_xml is not None: @@ -81,6 +89,36 @@ class GoogleDriveIE(InfoExtractor): if caption_extension.attrib.get('fmt_code') and not caption_extension.attrib.get('default'): self._caption_formats_ext.append(caption_extension.attrib['fmt_code']) + def _get_captions_by_type(self, video_id, video_subtitles_id, caption_type, caption_original_lang_code=None): + if not video_subtitles_id or not caption_type: + return None + captions = {} + for caption_entry in self._captions_by_country_xml.findall(self._CAPTIONS_ENTRY_TAG[caption_type]): + caption_lang_code = caption_entry.attrib.get('lang_code') + if not caption_lang_code: + continue + caption_format_data = [] + for caption_format in self._caption_formats_ext: + query = { + 'vid': video_subtitles_id, + 'v': video_id, + 'fmt': caption_format, + 'lang': caption_lang_code if caption_original_lang_code is None else caption_original_lang_code, + 'type': 'track', + 'name': '', + 'kind': '', + } + if caption_original_lang_code is not None: + query.update({'tlang': caption_lang_code}) + caption_format_data.append({ + 'url': update_url_query(self._BASE_URL_CAPTIONS, query), + 'ext': caption_format, + }) + captions[caption_lang_code] = caption_format_data + if not captions: + self.report_warning('video doesn\'t have %s' % caption_type.replace('_', ' ')) + return captions + def _get_subtitles(self, video_id, video_subtitles_id, hl): if not video_subtitles_id or not hl: return None @@ -88,31 +126,7 @@ class GoogleDriveIE(InfoExtractor): self._set_captions_data(video_id, video_subtitles_id, hl) if self._captions_by_country_xml is None: return None - - subtitles = {} - for subtitle_track in self._captions_by_country_xml.findall('track'): - subtitle_lang_code = subtitle_track.attrib.get('lang_code') - if not subtitle_lang_code: - continue - subtitle_format_data = [] - for subtitle_format in self._caption_formats_ext: - query = { - 'vid': video_subtitles_id, - 'v': video_id, - 'lang': subtitle_lang_code, - 'fmt': subtitle_format, - 'type': 'track', - 'name': '', - 'kind': '', - } - subtitle_format_data.append({ - 'url': update_url_query('https://drive.google.com/timedtext', query), - 'ext': subtitle_format, - }) - subtitles[subtitle_lang_code] = subtitle_format_data - if not subtitles: - self.report_warning('video doesn\'t have subtitles') - return subtitles + return self._get_captions_by_type(video_id, video_subtitles_id, 'subtitles') def _get_automatic_captions(self, video_id, video_subtitles_id, hl): if not video_subtitles_id or not hl: @@ -122,39 +136,13 @@ class GoogleDriveIE(InfoExtractor): if self._captions_by_country_xml is None: return None self.to_screen('%s: Looking for automatic captions' % video_id) - subtitle_original_track = self._captions_by_country_xml.find('track') if subtitle_original_track is None: return None subtitle_original_lang_code = subtitle_original_track.attrib.get('lang_code') if not subtitle_original_lang_code: return None - - automatic_captions = {} - for automatic_caption_target in self._captions_by_country_xml.findall('target'): - automatic_caption_lang_code = automatic_caption_target.attrib.get('lang_code') - if not automatic_caption_lang_code: - continue - automatic_caption_format_data = [] - for automatic_caption_format in self._caption_formats_ext: - query = { - 'vid': video_subtitles_id, - 'v': video_id, - 'lang': subtitle_original_lang_code, - 'fmt': automatic_caption_format, - 'tlang': automatic_caption_lang_code, - 'type': 'track', - 'name': '', - 'kind': '', - } - automatic_caption_format_data.append({ - 'url': update_url_query('https://drive.google.com/timedtext', query), - 'ext': automatic_caption_format, - }) - automatic_captions[automatic_caption_lang_code] = automatic_caption_format_data - if not automatic_captions: - self.report_warning('video doesn\'t have automatic captions') - return automatic_captions + return self._get_captions_by_type(video_id, video_subtitles_id, 'automatic_captions', subtitle_original_lang_code) def _real_extract(self, url): video_id = self._match_id(url)