From 9ff9e2643187e526af1663591f29ac6b5bd35917 Mon Sep 17 00:00:00 2001 From: Protuhj Date: Fri, 19 May 2017 01:53:03 -0400 Subject: [PATCH 1/7] Fixes issue with not being able to decode signatures on DASH MPD URLs --- youtube_dl/extractor/youtube.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 44a39282f..d448f4a6b 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1458,11 +1458,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # manifest pointed by get_video_info's dashmpd). # The general idea is to take a union of itags of both DASH manifests (for example # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093) + + # up-to-date sts value is required to properly decode the signature + sts = self._search_regex(r'"sts"\s*:\s*(\d+)', video_webpage, 'sts', default='') self.report_video_info_webpage_download(video_id) for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']: video_info_url = ( - '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en' - % (proto, video_id, el_type)) + '%s://www.youtube.com/get_video_info?&video_id=%s%s&eurl=&gl=US&hl=en&sts=%s' + % (proto, video_id, el_type, sts)) video_info_webpage = self._download_webpage( video_info_url, video_id, note=False, From 28b5b85cbc8c858afbeb3b7b5d4383223878a828 Mon Sep 17 00:00:00 2001 From: Protuhj Date: Fri, 19 May 2017 02:07:19 -0400 Subject: [PATCH 2/7] Re-adding ps=default, since it doesn't hurt having it. --- youtube_dl/extractor/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index d448f4a6b..994b8a5e7 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1464,7 +1464,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): self.report_video_info_webpage_download(video_id) for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']: video_info_url = ( - '%s://www.youtube.com/get_video_info?&video_id=%s%s&eurl=&gl=US&hl=en&sts=%s' + '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en&sts=%s' % (proto, video_id, el_type, sts)) video_info_webpage = self._download_webpage( video_info_url, From 9f6db1b2f4c0bf15951e32052ccf90d43dc1d273 Mon Sep 17 00:00:00 2001 From: Protuhj Date: Fri, 19 May 2017 18:28:18 -0400 Subject: [PATCH 3/7] Use ytplayer_config rather than the sts regular expression. --- youtube_dl/extractor/youtube.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 994b8a5e7..740fea40c 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1435,9 +1435,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor): else: age_gate = False video_info = None + sts = '' # Try looking directly into the video webpage ytplayer_config = self._get_ytplayer_config(video_id, video_webpage) if ytplayer_config: + sts = ytplayer_config['sts'] args = ytplayer_config['args'] if args.get('url_encoded_fmt_stream_map'): # Convert to the same format returned by compat_parse_qs @@ -1460,7 +1462,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093) # up-to-date sts value is required to properly decode the signature - sts = self._search_regex(r'"sts"\s*:\s*(\d+)', video_webpage, 'sts', default='') self.report_video_info_webpage_download(video_id) for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']: video_info_url = ( From 8997ae0469fa3f56055ccae428fe3145f3f99d88 Mon Sep 17 00:00:00 2001 From: Protuhj Date: Sat, 20 May 2017 13:28:42 -0400 Subject: [PATCH 4/7] Don't crash if the 'sts' value doesn't exist in the ytplayer_config Tested with values that definitely don't exist in the configuration. --- youtube_dl/extractor/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 740fea40c..9e2a62dd8 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1439,7 +1439,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # Try looking directly into the video webpage ytplayer_config = self._get_ytplayer_config(video_id, video_webpage) if ytplayer_config: - sts = ytplayer_config['sts'] + sts = ytplayer_config.get('sts', '') args = ytplayer_config['args'] if args.get('url_encoded_fmt_stream_map'): # Convert to the same format returned by compat_parse_qs From ae02d4b1cd051640e1772689aee756a38dfc2eea Mon Sep 17 00:00:00 2001 From: Protuhj Date: Sat, 20 May 2017 13:38:20 -0400 Subject: [PATCH 5/7] Moved comment to where sts is retrieved. --- youtube_dl/extractor/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 9e2a62dd8..b24933867 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1439,6 +1439,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # Try looking directly into the video webpage ytplayer_config = self._get_ytplayer_config(video_id, video_webpage) if ytplayer_config: + # up-to-date sts value is required to properly decode the signature, if the video has one sts = ytplayer_config.get('sts', '') args = ytplayer_config['args'] if args.get('url_encoded_fmt_stream_map'): @@ -1461,7 +1462,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # The general idea is to take a union of itags of both DASH manifests (for example # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093) - # up-to-date sts value is required to properly decode the signature self.report_video_info_webpage_download(video_id) for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']: video_info_url = ( From 53bb578a4ba1e03a297ac49d3a13936929c2aeb4 Mon Sep 17 00:00:00 2001 From: Sergey M Date: Sun, 21 May 2017 01:06:55 +0700 Subject: [PATCH 6/7] Update youtube.py --- youtube_dl/extractor/youtube.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index b24933867..3152d836a 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1461,7 +1461,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # manifest pointed by get_video_info's dashmpd). # The general idea is to take a union of itags of both DASH manifests (for example # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093) - self.report_video_info_webpage_download(video_id) for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']: video_info_url = ( From 91a970cbe17f1aba16680dcaef9bcb7a7da6c951 Mon Sep 17 00:00:00 2001 From: Sergey M Date: Sun, 21 May 2017 01:10:25 +0700 Subject: [PATCH 7/7] Update youtube.py --- youtube_dl/extractor/youtube.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 3152d836a..ae832cac5 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1439,8 +1439,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # Try looking directly into the video webpage ytplayer_config = self._get_ytplayer_config(video_id, video_webpage) if ytplayer_config: - # up-to-date sts value is required to properly decode the signature, if the video has one - sts = ytplayer_config.get('sts', '') args = ytplayer_config['args'] if args.get('url_encoded_fmt_stream_map'): # Convert to the same format returned by compat_parse_qs @@ -1454,6 +1452,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): args['ypc_vid'], YoutubeIE.ie_key(), video_id=args['ypc_vid']) if args.get('livestream') == '1' or args.get('live_playback') == 1: is_live = True + sts = ytplayer_config.get('sts', '') if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True): # We also try looking in get_video_info since it may contain different dashmpd # URL that points to a DASH manifest with possibly different itag set (some itags