diff --git a/youtube_dl/extractor/blendercloud.py b/youtube_dl/extractor/blendercloud.py index 527d41a27..9da745121 100644 --- a/youtube_dl/extractor/blendercloud.py +++ b/youtube_dl/extractor/blendercloud.py @@ -4,8 +4,60 @@ from .common import InfoExtractor import re -class BlenderCloudIE(InfoExtractor): - _VALID_URL = r'https?://cloud\.blender\.org/[^/]+/(?P[0-9a-z-]+)/(?P[0-9a-z]+)?' +class BlenderCloudBaseIE(InfoExtractor): + # A video on the Blender Cloud site is referenced by a single alphanumeric node, + # i.e. '56041550044a2a00d0d7e068' + # + # The data we want for any given node ID can be fetched at: + url_node = "https://cloud.blender.org/nodes/%s/view" + + # TODO: Add authentication scheme for subscriber-only videos. + # + # This will require the use of a (paid) Blender ID token available from: + # https://store.blender.org/product/membership/ + # + # For now - ignore any subscriber-only videos and just grab the public ones. + warning_subscribers_only = 'Only available to Blender Cloud subscribers.' + warning_no_video_sources = 'No video sources available.' + + def get_node_title(self, source): + node_title = None + node_title = self._html_search_regex( + r'(.*?)', source, 'title').strip() + #print "BlenderCloudBaseIE : get_node_title : node_title : %s" % node_title + return node_title + + def get_webpage_title(self, source): + webpage_title = None + webpage_title = self._html_search_regex( + r'(.*?)', source, 'title').strip() + #print "BlenderCloudBaseIE : get_webpage_title : webpage_title : %s" % webpage_title + return webpage_title + + @staticmethod + def is_video_subscriber_only(source): + errmsg_subscribers_only = 'Only available to Blender Cloud subscribers.' + return True if errmsg_subscribers_only in source else False + + @staticmethod + def get_video_formats(source): + video_formats = [] + for video in re.findall(r'[0-9a-z-]+)/(?P[0-9a-z]+)/?' _TESTS = [ { @@ -61,8 +113,106 @@ class BlenderCloudIE(InfoExtractor): ], }, { - # Playlist - 'url': 'https://cloud.blender.org/p/blenderella/', + # Playlist (subsection) + 'url': 'https://cloud.blender.org/p/creature-factory-2/5604151f044a2a00caa7b04b', + 'info_dict': { + 'id': '5604151f044a2a00caa7b04b', + 'title': '01 - First steps', + }, + 'playlist': [ + { + 'info_dict': { + 'id': '5604151f044a2a00caa7b04c', + 'display_id': 'creature-factory-2', + 'ext': 'mp4', + 'title': 'Introduction', + }, + }, + ], + 'expected_warnings': [ + 'Only available to Blender Cloud subscribers.' + ], + }, + ] + + def _real_extract(self, url): + #print "BlenderCloudIE : _real_extract : %s" % url + + mobj = re.match(self._VALID_URL, url) + base_node_id = mobj.group('base_node_id') + display_id = mobj.group('display_id') + #print "BlenderCloudIE : _real_extract : base_node_id : %s" % base_node_id + #print "BlenderCloudIE : _real_extract : display_id : %s" % display_id + + # extract a single video -or- a playlist of subsection videos + + webpage = self._download_webpage(self.url_node % base_node_id, base_node_id) + + if '
' in webpage: + # this base node references a single video (i.e. a single node) + + title = None + formats = [] + + if self.is_video_subscriber_only(webpage): + self.report_warning('%s - %s' % (base_node_id, self.warning_subscribers_only)) + else: + title = self.get_node_title(webpage) + formats = self.get_video_formats(webpage) + #self._check_formats(formats, base_node_id) + self._sort_formats(formats) + + return { + 'id': base_node_id, + 'display_id': display_id, + 'title': title, + 'formats': formats, + } + elif '
' in webpage: + # this base node references a playlist of subsection videos (i.e. multiple nodes) + + entries = [] + for node_id in re.findall(r'data-node_id=\"([0-9a-z]+)\"\s*title=\"', webpage): + #print "BlenderCloudIE : _real_extract : node_id : %s" % node_id + + webpage_node = self._download_webpage(self.url_node % node_id, node_id) + + if '
' in webpage_node: + if self.is_video_subscriber_only(webpage_node): + self.report_warning('%s - %s' % (node_id, self.warning_subscribers_only)) + else: + title = self.get_node_title(webpage_node) + formats = self.get_video_formats(webpage_node) + #self._check_formats(formats, node_id) + self._sort_formats(formats) + entries.append({ + 'id': node_id, + 'display_id': display_id, + 'title': title, + 'formats': formats, + }) + else: + self.report_warning('%s - %s' % (node_id, warning_no_video_sources)) + + #print "BlenderCloudIE : _real_extract : entries : %s" % entries + return self.playlist_result(entries, playlist_id=base_node_id, playlist_title=self.get_node_title(webpage)) + else: + self.report_warning('%s - %s' % (base_node_id, self.warning_no_video_sources)) + return { + 'id': base_node_id, + 'display_id': display_id, + 'title': None, + 'formats': [], + } + + +class BlenderCloudPlaylistIE(BlenderCloudBaseIE): + _VALID_URL = r'https?://cloud\.blender\.org/[^/]+/(?P[0-9a-z-]+)/?$' + + _TESTS = [ + { + # Playlist (complete) + 'url': 'https://cloud.blender.org/p/blenderella', 'info_dict': { 'id': 'blenderella', 'title': 'Learn Character Modeling — Blender Cloud', @@ -84,174 +234,64 @@ class BlenderCloudIE(InfoExtractor): }, ] - def get_node_title(self, source): - node_title = None - node_title = self._html_search_regex( - r'(.*?)', source, 'title').strip() - #print "BlenderCloudIE : get_node_title : node_title : %s" % node_title - return node_title - - def get_webpage_title(self, source): - webpage_title = None - webpage_title = self._html_search_regex( - r'(.*?)', source, 'title').strip() - #print "BlenderCloudIE : get_webpage_title : webpage_title : %s" % webpage_title - return webpage_title - - @staticmethod - def is_video_subscriber_only(source): - errmsg_subscribers_only = 'Only available to Blender Cloud subscribers.' - return True if errmsg_subscribers_only in source else False - - @staticmethod - def get_video_formats(source): - video_formats = [] - for video in re.findall(r'" in webpage: + if '
' in webpage_node: # this node references a single video (i.e. a single node) - title = None - formats = [] - - if self.is_video_subscriber_only(webpage): - self.report_warning('%s - %s' % (base_node_id, warning_subscribers_only)) + if self.is_video_subscriber_only(webpage_node): + self.report_warning('%s - %s' % (node_id, self.warning_subscribers_only)) else: - title = self.get_node_title(webpage) - formats = self.get_video_formats(webpage) - self._check_formats(formats, base_node_id) + title = self.get_node_title(webpage_node) + formats = self.get_video_formats(webpage_node) + #self._check_formats(formats, node_id) self._sort_formats(formats) - - return { - 'id': base_node_id, - 'display_id': display_id, - 'title': title, - 'formats': formats, - } - elif "
" in webpage: + entries.append({ + 'id': node_id, + 'display_id': display_id, + 'title': title, + 'formats': formats, + }) + elif '
' in webpage_node: # this node references a playlist of subsection videos (i.e. multiple nodes) - entries = [] - for node_id in re.findall(r'data-node_id=\"([0-9a-z]+)\"\s*title=\"', webpage): - #print "BlenderCloudIE : _real_extract : node_id : %s" % node_id + for sub_node_id in re.findall(r'data-node_id=\"([0-9a-z]+)\"\s*title=\"', webpage_node): + #print "BlenderCloudPlaylistIE : _real_extract : sub_node_id : %s" % sub_node_id - webpage_node = self._download_webpage(url_node % node_id, node_id) + webpage_sub_node = self._download_webpage(self.url_node % sub_node_id, sub_node_id) - if "
" in webpage_node: - if self.is_video_subscriber_only(webpage_node): - self.report_warning('%s - %s' % (node_id, warning_subscribers_only)) + if '
' in webpage_sub_node: + if self.is_video_subscriber_only(webpage_sub_node): + self.report_warning('%s - %s' % (sub_node_id, self.warning_subscribers_only)) else: - title = self.get_node_title(webpage_node) - formats = self.get_video_formats(webpage_node) - self._check_formats(formats, node_id) + title = self.get_node_title(webpage_sub_node) + formats = self.get_video_formats(webpage_sub_node) + #self._check_formats(formats, sub_node_id) self._sort_formats(formats) - entries.append({ - 'id': node_id, + 'id': sub_node_id, 'display_id': display_id, 'title': title, 'formats': formats, }) else: - self.report_warning('%s - %s' % (node_id, warning_no_video_sources)) - - #print "BlenderCloudIE : _real_extract : entries : %s" % entries - return self.playlist_result(entries, playlist_id=base_node_id, playlist_title=self.get_node_title(webpage)) + self.report_warning('%s - %s' % (sub_node_id, self.warning_no_video_sources)) else: - self.report_warning('%s - %s' % (base_node_id, warning_no_video_sources)) - return { - 'id': base_node_id, - 'display_id': display_id, - 'title': None, - 'formats': [], - } - else: - # extract the entire playlist for an entire video section + self.report_warning('%s - %s' % (node_id, self.warning_no_video_sources)) - webpage = self._download_webpage(url, display_id) - - entries = [] - for node_id in re.findall(r'data-node_id=\"([0-9a-z]+)\"\s*class=\"', webpage): - #print "BlenderCloudIE : _real_extract : node_id : %s" % node_id - - webpage_node = self._download_webpage(url_node % node_id, node_id) - - if "
" in webpage_node: - # this node references a single video (i.e. a single node) - - title = None - formats = [] - - if self.is_video_subscriber_only(webpage_node): - self.report_warning('%s - %s' % (node_id, warning_subscribers_only)) - else: - title = self.get_node_title(webpage_node) - formats = self.get_video_formats(webpage_node) - self._check_formats(formats, node_id) - self._sort_formats(formats) - - entries.append({ - 'id': node_id, - 'display_id': display_id, - 'title': title, - 'formats': formats, - }) - elif "
" in webpage_node: - # this node references a playlist of subsection videos (i.e. multiple nodes) - - for sub_node_id in re.findall(r'data-node_id=\"([0-9a-z]+)\"\s*title=\"', webpage_node): - #print "BlenderCloudIE : _real_extract : sub_node_id : %s" % sub_node_id - - webpage_sub_node = self._download_webpage(url_node % sub_node_id, sub_node_id) - - if "
" in webpage_sub_node: - if self.is_video_subscriber_only(webpage_sub_node): - self.report_warning('%s - %s' % (sub_node_id, warning_subscribers_only)) - else: - title = self.get_node_title(webpage_sub_node) - formats = self.get_video_formats(webpage_sub_node) - self._check_formats(formats, sub_node_id) - self._sort_formats(formats) - - entries.append({ - 'id': sub_node_id, - 'display_id': display_id, - 'title': title, - 'formats': formats, - }) - else: - self.report_warning('%s - %s' % (sub_node_id, warning_no_video_sources)) - else: - self.report_warning('%s - %s' % (node_id, warning_no_video_sources)) - - return self.playlist_result(entries, playlist_id=display_id, playlist_title=self.get_webpage_title(webpage)) + return self.playlist_result(entries, playlist_id=display_id, playlist_title=self.get_webpage_title(webpage)) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 14597e242..9dd9e6ae2 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -123,7 +123,10 @@ from .bleacherreport import ( BleacherReportIE, BleacherReportCMSIE, ) -from .blendercloud import (BlenderCloudIE) +from .blendercloud import ( + BlenderCloudIE, + BlenderCloudPlaylistIE +) from .blinkx import BlinkxIE from .bloomberg import BloombergIE from .bokecc import BokeCCIE