From b55cd2d07e339e9e2b0ebbce768f7a94ca06465c Mon Sep 17 00:00:00 2001 From: Elliot Algase Date: Sun, 5 May 2019 16:16:27 -0400 Subject: [PATCH 1/2] [openload] add support for verystream.com (website is similar to openload) [openload] minor verystream bug fixes [verystream] seperate extractor (now, a wrapper around Openload) [verystream] become flake8 compliant [generic] fix _extract_urls for Openload and Verystream [genetic] fix _extract_urls for Openload and Verystream [verystream] change test to only_matching [verystream] don't require phantomjs [verystream] flake8 compliance [verystream] don't use Openload's built in ID's [verystream] make requested changes on GitHub [openload] add support for verystream.com (website is similar to openload) [openload] minor verystream bug fixes [verystream] seperate extractor (now, a wrapper around Openload) [verystream] become flake8 compliant [generic] fix _extract_urls for Openload and Verystream [genetic] fix _extract_urls for Openload and Verystream [verystream] change test to only_matching [verystream] don't require phantomjs [verystream] flake8 compliance [verystream] don't use Openload's built in ID's [verystream] make requested changes on GitHub --- youtube_dl/extractor/extractors.py | 5 +- youtube_dl/extractor/generic.py | 11 +++- youtube_dl/extractor/openload.py | 84 +++++++++++++++++++++++------- 3 files changed, 80 insertions(+), 20 deletions(-) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 392b1f92b..762000271 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -833,7 +833,10 @@ from .ooyala import ( OoyalaIE, OoyalaExternalIE, ) -from .openload import OpenloadIE +from .openload import ( + OpenloadIE, + VerystreamIE, +) from .ora import OraTVIE from .orf import ( ORFTVthekIE, diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 6f48b04da..495fa4975 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -89,7 +89,10 @@ from .piksel import PikselIE from .videa import VideaIE from .twentymin import TwentyMinutenIE from .ustream import UstreamIE -from .openload import OpenloadIE +from .openload import ( + OpenloadIE, + VerystreamIE, +) from .videopress import VideoPressIE from .rutube import RutubeIE from .limelight import LimelightBaseIE @@ -3017,6 +3020,12 @@ class GenericIE(InfoExtractor): return self.playlist_from_matches( openload_urls, video_id, video_title, ie=OpenloadIE.ie_key()) + # Look for Verystream embeds + verystream_urls = VerystreamIE._extract_urls(webpage) + if verystream_urls: + return self.playlist_from_matches( + verystream_urls, video_id, video_title, ie=VerystreamIE.ie_key()) + # Look for VideoPress embeds videopress_urls = VideoPressIE._extract_urls(webpage) if videopress_urls: diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py index 6a8ef67bd..c324d9082 100644 --- a/youtube_dl/extractor/openload.py +++ b/youtube_dl/extractor/openload.py @@ -254,7 +254,10 @@ class OpenloadIE(InfoExtractor): (?:f|embed)/ (?P[a-zA-Z0-9-_]+) ''' % _DOMAINS - + _EMBED_WORD = 'embed' + _STREAM_WORD = 'f' + _REDIR_WORD = 'stream' + _URL_IDS = ('streamurl', 'streamuri', 'streamurj') _TESTS = [{ 'url': 'https://openload.co/f/kUEfGclsU9o', 'md5': 'bf1c059b004ebc7a256f89408e65c36e', @@ -1954,6 +1957,11 @@ class OpenloadIE(InfoExtractor): r']+src=["\']((?:https?://)?%s/embed/[a-zA-Z0-9-_]+)' % OpenloadIE._DOMAINS, webpage) + def _extract_decrypted_page(self, page_url, webpage, video_id, headers): + phantom = PhantomJSwrapper(self, required_version='2.0') + webpage, _ = phantom.get(page_url, html=webpage, video_id=video_id, headers=headers) + return webpage + def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) host = mobj.group('host') @@ -1964,9 +1972,9 @@ class OpenloadIE(InfoExtractor): 'User-Agent': self._USER_AGENT_TPL % random.choice(self._CHROME_VERSIONS), } - for path in ('embed', 'f'): + for path in (self._EMBED_WORD, self._STREAM_WORD): page_url = url_pattern % path - last = path == 'f' + last = path == self._STREAM_WORD webpage = self._download_webpage( page_url, video_id, 'Downloading %s webpage' % path, headers=headers, fatal=last) @@ -1978,21 +1986,20 @@ class OpenloadIE(InfoExtractor): raise ExtractorError('File not found', expected=True, video_id=video_id) break - phantom = PhantomJSwrapper(self, required_version='2.0') - webpage, _ = phantom.get(page_url, html=webpage, video_id=video_id, headers=headers) - - decoded_id = (get_element_by_id('streamurl', webpage) or - get_element_by_id('streamuri', webpage) or - get_element_by_id('streamurj', webpage) or - self._search_regex( - (r'>\s*([\w-]+~\d{10,}~\d+\.\d+\.0\.0~[\w-]+)\s*<', - r'>\s*([\w~-]+~\d+\.\d+\.\d+\.\d+~[\w~-]+)', - r'>\s*([\w-]+~\d{10,}~(?:[a-f\d]+:){2}:~[\w-]+)\s*<', - r'>\s*([\w~-]+~[a-f0-9:]+~[\w~-]+)\s*<', - r'>\s*([\w~-]+~[a-f0-9:]+~[\w~-]+)'), webpage, - 'stream URL')) - - video_url = 'https://%s/stream/%s?mime=true' % (host, decoded_id) + webpage = self._extract_decrypted_page(page_url, webpage, video_id, headers) + for element_id in self._URL_IDS: + decoded_id = get_element_by_id(element_id, webpage) + if decoded_id: + break + if not decoded_id: + decoded_id = self._search_regex( + (r'>\s*([\w-]+~\d{10,}~\d+\.\d+\.0\.0~[\w-]+)\s*<', + r'>\s*([\w~-]+~\d+\.\d+\.\d+\.\d+~[\w~-]+)', + r'>\s*([\w-]+~\d{10,}~(?:[a-f\d]+:){2}:~[\w-]+)\s*<', + r'>\s*([\w~-]+~[a-f0-9:]+~[\w~-]+)\s*<', + r'>\s*([\w~-]+~[a-f0-9:]+~[\w~-]+)'), webpage, + 'stream URL') + video_url = 'https://%s/%s/%s?mime=true' % (host, self._REDIR_WORD, decoded_id) title = self._og_search_title(webpage, default=None) or self._search_regex( r']+class=["\']title["\'][^>]*>([^<]+)', webpage, @@ -2012,3 +2019,44 @@ class OpenloadIE(InfoExtractor): 'subtitles': subtitles, 'http_headers': headers, } + + +class VerystreamIE(OpenloadIE): + IE_NAME = 'verystream' + + _DOMAINS = r'(verystream\.com)' + _VALID_URL = r'''(?x) + https?:// + (?P + (?:www\.)? + %s + )/ + (?:stream|e)/ + (?P[a-zA-Z0-9-_]+) + ''' % _DOMAINS + _EMBED_WORD = 'e' + _STREAM_WORD = 'stream' + _REDIR_WORD = 'gettoken' + _URL_IDS = ('videolink', ) + _TESTS = [{ + 'url': 'https://verystream.com/stream/c1GWQ9ngBBx/', + 'md5': 'd3e8c5628ccb9970b65fd65269886795', + 'info_dict': { + 'id': 'c1GWQ9ngBBx', + 'ext': 'mp4', + 'title': 'Big Buck Bunny.mp4', + 'thumbnail': r're:^https?://.*\.jpg$', + }, + }, { + 'url': 'https://verystream.com/e/c1GWQ9ngBBx/', + 'only_matching': True, + }] + + def _extract_decrypted_page(self, page_url, webpage, video_id, headers): + return webpage # for Verystream, the webpage is already decrypted + + @staticmethod + def _extract_urls(webpage): + return re.findall( + r']+src=["\']((?:https?://)?%s/e/[a-zA-Z0-9-_]+)' + % VerystreamIE._DOMAINS, webpage) From 2752cc932b6dcb7cec2dca9243df6d0817eaa8c9 Mon Sep 17 00:00:00 2001 From: Elliot Algase Date: Fri, 10 May 2019 14:29:03 -0400 Subject: [PATCH 2/2] [verystream] use classmethod instead of staticmethod, don't capture domain --- youtube_dl/extractor/openload.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py index c324d9082..f77296f42 100644 --- a/youtube_dl/extractor/openload.py +++ b/youtube_dl/extractor/openload.py @@ -1951,11 +1951,11 @@ class OpenloadIE(InfoExtractor): '69.0.3497.28', ) - @staticmethod - def _extract_urls(webpage): + @classmethod + def _extract_urls(cls, webpage): return re.findall( - r']+src=["\']((?:https?://)?%s/embed/[a-zA-Z0-9-_]+)' - % OpenloadIE._DOMAINS, webpage) + r']+src=["\']((?:https?://)?%s/%s/[a-zA-Z0-9-_]+)' + % (cls._DOMAINS, cls._EMBED_WORD), webpage) def _extract_decrypted_page(self, page_url, webpage, video_id, headers): phantom = PhantomJSwrapper(self, required_version='2.0') @@ -2024,7 +2024,7 @@ class OpenloadIE(InfoExtractor): class VerystreamIE(OpenloadIE): IE_NAME = 'verystream' - _DOMAINS = r'(verystream\.com)' + _DOMAINS = r'(?:verystream\.com)' _VALID_URL = r'''(?x) https?:// (?P @@ -2054,9 +2054,3 @@ class VerystreamIE(OpenloadIE): def _extract_decrypted_page(self, page_url, webpage, video_id, headers): return webpage # for Verystream, the webpage is already decrypted - - @staticmethod - def _extract_urls(webpage): - return re.findall( - r']+src=["\']((?:https?://)?%s/e/[a-zA-Z0-9-_]+)' - % VerystreamIE._DOMAINS, webpage)