diff --git a/README.md b/README.md index 1ad87d075..1d53fbc7d 100644 --- a/README.md +++ b/README.md @@ -466,7 +466,7 @@ The basic usage is not to set any template arguments when downloading a single f - `display_id`: An alternative identifier for the video - `uploader`: Full name of the video uploader - `license`: License name the video is licensed under - - `creator`: The main artist who created the video + - `creator`: The creator of the video - `release_date`: The date (YYYYMMDD) when the video was released - `timestamp`: UNIX timestamp of the moment the video became available - `upload_date`: Video upload date (YYYYMMDD) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 055433362..2187dcc8f 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -580,7 +580,7 @@ class YoutubeDL(object): is_id=(k == 'id')) template_dict = dict((k, sanitize(k, v)) for k, v in template_dict.items() - if v is not None) + if v is not None and not isinstance(v, (list, tuple, dict))) template_dict = collections.defaultdict(lambda: 'NA', template_dict) outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL) @@ -1639,7 +1639,7 @@ class YoutubeDL(object): # Just a single file success = dl(filename, info_dict) except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self.report_error('unable to download video data: %s' % str(err)) + self.report_error('unable to download video data: %s' % error_to_compat_str(err)) return except (OSError, IOError) as err: raise UnavailableVideoError(err) diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py index 24df8fe93..42c21bf41 100644 --- a/youtube_dl/extractor/aol.py +++ b/youtube_dl/extractor/aol.py @@ -12,7 +12,7 @@ from ..utils import ( class AolIE(InfoExtractor): IE_NAME = 'on.aol.com' - _VALID_URL = r'(?:aol-video:|https?://on\.aol\.com/.*-)(?P[^/?-]+)' + _VALID_URL = r'(?:aol-video:|https?://on\.aol\.com/(?:[^/]+/)*(?:[^/?#&]+-)?)(?P[^/?#&]+)' _TESTS = [{ # video with 5min ID @@ -53,6 +53,12 @@ class AolIE(InfoExtractor): }, { 'url': 'http://on.aol.com/shows/park-bench-shw518173474-559a1b9be4b0c3bfad3357a7?context=SH:SHW518173474:PL4327:1460619712763', 'only_matching': True, + }, { + 'url': 'http://on.aol.com/video/519442220', + 'only_matching': True, + }, { + 'url': 'aol-video:5707d6b8e4b090497b04f706', + 'only_matching': True, }] def _real_extract(self, url): diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 61a5d124c..0843d89af 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -163,7 +163,7 @@ class InfoExtractor(object): description: Full video description. uploader: Full name of the video uploader. license: License name the video is licensed under. - creator: The main artist who created the video. + creator: The creator of the video. release_date: The date (YYYYMMDD) when the video was released. timestamp: UNIX timestamp of the moment the video became available. upload_date: Video upload date (YYYYMMDD). diff --git a/youtube_dl/extractor/kuwo.py b/youtube_dl/extractor/kuwo.py index 3740869c7..616ed19e1 100644 --- a/youtube_dl/extractor/kuwo.py +++ b/youtube_dl/extractor/kuwo.py @@ -266,7 +266,6 @@ class KuwoCategoryIE(InfoExtractor): 'info_dict': { 'id': '86375', 'title': '八十年代精选', - 'description': '这些都是属于八十年代的回忆！', }, 'playlist_mincount': 24, } @@ -283,6 +282,8 @@ class KuwoCategoryIE(InfoExtractor): category_desc = remove_start( get_element_by_id('intro', webpage).strip(), '%s简介：' % category_name) + if category_desc == '暂无': + category_desc = None jsonm = self._parse_json(self._html_search_regex( r'var\s+jsonm\s*=\s*([^;]+);', webpage, 'category songs'), category_id) diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index 63eab4148..c0ef08c02 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -5,6 +5,7 @@ import re from .common import InfoExtractor from ..compat import ( compat_etree_fromstring, + compat_str, compat_urlparse, ) from ..utils import ( @@ -116,6 +117,10 @@ class VevoIE(VevoBaseIE): 'genre': 'Pop', }, 'expected_warnings': ['Failed to download video versions info'], + }, { + # no genres available + 'url': 'http://www.vevo.com/watch/INS171400764', + 'only_matching': True, }] _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com' _SOURCE_TYPES = { @@ -184,8 +189,8 @@ class VevoIE(VevoBaseIE): errnote='Unable to retrieve oauth token') if 'THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION' in webpage: - raise ExtractorError( - '%s said: This page is currently unavailable in your region.' % self.IE_NAME, expected=True) + self.raise_geo_restricted( + '%s said: This page is currently unavailable in your region' % self.IE_NAME) auth_info = self._parse_json(webpage, video_id) self._api_url_template = self.http_scheme() + '//apiv2.vevo.com/%s?token=' + auth_info['access_token'] @@ -200,12 +205,10 @@ class VevoIE(VevoBaseIE): response = self._download_json( json_url, video_id, 'Downloading video info', 'Unable to download info') video_info = response.get('video') or {} - video_versions = video_info.get('videoVersions') artist = None featured_artist = None uploader = None view_count = None - timestamp = None formats = [] if not video_info: @@ -339,7 +342,11 @@ class VevoIE(VevoBaseIE): if featured_artist: artist = '%s ft. %s' % (artist, featured_artist) title = '%s - %s' % (artist, track) if artist else track - genre = video_info.get('genres', [None])[0] + + genres = video_info.get('genres') + genre = ( + genres[0] if genres and isinstance(genres, list) and + isinstance(genres[0], compat_str) else None) is_explicit = video_info.get('isExplicit') if is_explicit is True: diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py index 2d1504eaa..769003735 100644 --- a/youtube_dl/extractor/xfileshare.py +++ b/youtube_dl/extractor/xfileshare.py @@ -13,12 +13,21 @@ from ..utils import ( class XFileShareIE(InfoExtractor): - IE_DESC = 'XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me' - _VALID_URL = r'''(?x) - https?://(?P(?:www\.)? - (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto\.me|powerwatch\.pw))/ - (?:embed-)?(?P[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)? - ''' + _SITES = ( + ('daclips.in', 'DaClips'), + ('filehoot.com', 'FileHoot'), + ('gorillavid.in', 'GorillaVid'), + ('movpod.in', 'MovPod'), + ('powerwatch.pw', 'PowerWatch'), + ('rapidvideo.ws', 'Rapidvideo.ws'), + ('thevideobee.to', 'TheVideoBee'), + ('vidto.me', 'Vidto'), + ('streamin.to', 'Streamin.To'), + ) + + IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1]) + _VALID_URL = (r'https?://(?P(?:www\.)?(?:%s))/(?:embed-)?(?P[0-9a-zA-Z]+)' + % '|'.join(re.escape(site) for site in list(zip(*_SITES))[0])) _FILE_NOT_FOUND_REGEX = r'>(?:404 - )?File Not Found<' @@ -43,25 +52,6 @@ class XFileShareIE(InfoExtractor): 'title': 'Micro Pig piglets ready on 16th July 2009-bG0PdrCdxUc', 'thumbnail': 're:http://.*\.jpg', } - }, { - # video with countdown timeout - 'url': 'http://fastvideo.in/1qmdn1lmsmbw', - 'md5': '8b87ec3f6564a3108a0e8e66594842ba', - 'info_dict': { - 'id': '1qmdn1lmsmbw', - 'ext': 'mp4', - 'title': 'Man of Steel - Trailer', - 'thumbnail': 're:http://.*\.jpg', - }, - }, { - 'url': 'http://realvid.net/ctn2y6p2eviw', - 'md5': 'b2166d2cf192efd6b6d764c18fd3710e', - 'info_dict': { - 'id': 'ctn2y6p2eviw', - 'ext': 'flv', - 'title': 'rdx 1955', - 'thumbnail': 're:http://.*\.jpg', - }, }, { 'url': 'http://movpod.in/0wguyyxi1yca', 'only_matching': True, diff --git a/youtube_dl/extractor/xiami.py b/youtube_dl/extractor/xiami.py index e4ed306b4..a6dfc4af9 100644 --- a/youtube_dl/extractor/xiami.py +++ b/youtube_dl/extractor/xiami.py @@ -9,6 +9,11 @@ from ..utils import int_or_none class XiamiBaseIE(InfoExtractor): _API_BASE_URL = 'http://www.xiami.com/song/playlist/cat/json/id' + def _download_webpage(self, *args, **kwargs): + webpage = super(XiamiBaseIE, self)._download_webpage(*args, **kwargs) + if '>Xiami is currently not available in your country.<' in webpage: + self.raise_geo_restricted('Xiami is currently not available in your country') + def _extract_track(self, track, track_id=None): title = track['title'] track_url = self._decrypt(track['location']) @@ -81,7 +86,8 @@ class XiamiSongIE(XiamiBaseIE): 'ext': 'lrc', }], }, - } + }, + 'skip': 'Georestricted', }, { 'url': 'http://www.xiami.com/song/1775256504', 'md5': '932a3abd45c6aa2b1fdbe028fcb4c4fc', @@ -100,7 +106,8 @@ class XiamiSongIE(XiamiBaseIE): 'ext': 'lrc', }], }, - } + }, + 'skip': 'Georestricted', }] def _real_extract(self, url): @@ -124,6 +131,7 @@ class XiamiAlbumIE(XiamiPlaylistBaseIE): 'id': '2100300444', }, 'playlist_count': 10, + 'skip': 'Georestricted', }, { 'url': 'http://www.xiami.com/album/512288?spm=a1z1s.6843761.1110925389.6.hhE9p9', 'only_matching': True, @@ -141,6 +149,7 @@ class XiamiArtistIE(XiamiPlaylistBaseIE): 'id': '2132', }, 'playlist_count': 20, + 'skip': 'Georestricted', } @@ -155,4 +164,5 @@ class XiamiCollectionIE(XiamiPlaylistBaseIE): 'id': '156527391', }, 'playlist_mincount': 29, + 'skip': 'Georestricted', }