R-yan/l1ving_youtube-dl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[vidlii] Fixed suggestions for new extractor (see pull request for issue #14779)

Browse Source 
- improved regular expressions
- improved tests
- improved code style
This commit is contained in:
Patrick Jattke 2017-12-28 14:03:14 +01:00
parent 00cc5ca340
commit d362c2b98e
1 changed files with 24 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
youtube_dl/extractor/vidlii.py
View File

@ -11,20 +11,16 @@ from ..utils import (
class VidliiIE(InfoExtractor): class VidliiIE(InfoExtractor):
_VALID_URL = r'(?:https*?:\/\/)*(?:www\.)*vidlii.com\/watch\?v=(?P<id>[^?\s]{11})' _VALID_URL = r'https?://(?:www\.)?vidlii.com/watch\?v=(?P<id>.{11})'
_TESTS = [{ _TESTS = [{
'url': 'https://www.vidlii.com/watch?v=tJluaH4BJ3v', 'url': 'https://www.vidlii.com/watch?v=tJluaH4BJ3v',
'md5': '9bf7d1e005dfa909b6efb0a1ff5175e2', 'md5': '9bf7d1e005dfa909b6efb0a1ff5175e2',
'info_dict': { 'info_dict': {
'id': 'tJluaH4BJ3v', 'id': 'tJluaH4BJ3v',
'title': 'Vidlii is against me', 'title': 'Vidlii is against me',
'description': 'I have HAD it. Vidlii does not like me. I have tried to uplaod videos and submit them to the ' 'description': 'md5:de24ab8a9a310976d66bebb824aa2420',
'contest and no ne of my videos show up so maybe it is broken for everyone else but this one was ' 'thumbnail': 're:https://.*.jpg',
'trying to submit it because I wanted to submit to the contest :) Tanks I hope the website is '
'fixed PS: Jan you are cool please add my video',
'thumbnail': 'https://www.vidlii.com/usfi/thmp/tJluaH4BJ3v.jpg',
'uploader': 'APPle5auc31995', 'uploader': 'APPle5auc31995',
'url': 'https://cdn.vidlii.com/videos/tJluaH4BJ3v.mp4',
'uploader_url': 'https://www.vidlii.com/user/APPle5auc31995', 'uploader_url': 'https://www.vidlii.com/user/APPle5auc31995',
'upload_date': '20171107', 'upload_date': '20171107',
'categories': 'News & Politics', 'categories': 'News & Politics',
@ -33,7 +29,6 @@ class VidliiIE(InfoExtractor):
'view_count': int, 'view_count': int,
'comment_count': int, 'comment_count': int,
'average_rating': float, 'average_rating': float,
'type': 'video',
'ext': 'mp4' 'ext': 'mp4'
} }
}, { }, {
@ -43,9 +38,8 @@ class VidliiIE(InfoExtractor):
'id': 'vBo2IcrwOkO', 'id': 'vBo2IcrwOkO',
'title': '(OLD VIDEO) i like youtube!!', 'title': '(OLD VIDEO) i like youtube!!',
'description': 'Original upload date:<br />\nMarch 10th 2011<br />\nCredit goes to people who own content in the video', 'description': 'Original upload date:<br />\nMarch 10th 2011<br />\nCredit goes to people who own content in the video',
'thumbnail': 'https://www.vidlii.com/usfi/thmp/vBo2IcrwOkO.jpg', 'thumbnail': 're:https://.*.jpg',
'uploader': 'MyEditedVideoSpartan', 'uploader': 'MyEditedVideoSpartan',
'url': 'https://cdn.vidlii.com/videos/vBo2IcrwOkO.mp4',
'uploader_url': 'https://www.vidlii.com/user/MyEditedVideoSpartan', 'uploader_url': 'https://www.vidlii.com/user/MyEditedVideoSpartan',
'upload_date': '20171011', 'upload_date': '20171011',
'categories': 'Film & Animation', 'categories': 'Film & Animation',
@ -54,7 +48,6 @@ class VidliiIE(InfoExtractor):
'view_count': int, 'view_count': int,
'comment_count': int, 'comment_count': int,
'average_rating': float, 'average_rating': float,
'type': 'video',
'ext': 'mp4' 'ext': 'mp4'
} }
}] }]
@ -64,23 +57,23 @@ class VidliiIE(InfoExtractor):
webpage = self._download_webpage(url, video_id) webpage = self._download_webpage(url, video_id)
# extract videoInfo variable for further use # extract videoInfo variable for further use
videoInfo_dict = self._parse_json(js_to_json(self._html_search_regex(r'var videoInfo\s*=\s*({[^}]*})', webpage, videoInfo_dict = self._parse_json(js_to_json(self._html_search_regex(r'var\s*videoInfo\s*=\s*({[^}]*})', webpage,
'videoInfo', fatal=False)), video_id) 'videoInfo', fatal=True)), video_id)
# extract basic properties of video # extract basic properties of video
title = (self._html_search_regex(r'<title>([^<]+?)</title>', webpage, 'title', default='') or title = (self._html_search_regex(r'<title>([^<]+?)</title>', webpage, 'title', default='', fatal=True) or
self._html_search_meta('twitter:title', webpage, 'title', default='')).replace(" - VidLii", self._html_search_meta('twitter:title', webpage, 'title', default='', fatal=True)).replace(' - VidLii', '') \
"") or self._html_search_regex( or self._html_search_regex(r'<h1>(.+?)</h1>', webpage, 'title', default=None, fatal=True)
r'<h1>(.+?)</h1>', webpage, 'title', default=None)
description = strip_or_none(get_element_by_id('des_text', webpage)) description = strip_or_none(get_element_by_id('des_text', webpage))
uploader = self._html_search_regex( uploader_patterns = [r'<div[^>]+class="wt_person"[^>]*>(?:[^<]+)<a href="/user/[^>]*?>([^<]*?)<',
r'<div[^>]+class="wt_person"[^>]*>(?:[^<]+)<a href="/user/[^>]*?>([^<]*?)<|<img src="[^>]+?class=(' r'<img src="[^>]+?class=(?:"avt2\s*"',
r'?:"avt2\s*"|\'avt2\s*\')[^>]+?alt=(?:"([^"]+?)"|\'([^\']+?)\')>', webpage, 'uploader', default=None, r'\'avt2\s*\')[^>]+?alt=(?:"([^"]+?)"',
fatal=False) r'\'([^\']+?)\')>']
uploader = self._html_search_regex(uploader_patterns, webpage, 'uploader', fatal=False)
video_url = videoInfo_dict.get("src") video_url = videoInfo_dict.get('src')
# get additional properties # get additional properties
uploader_url = urljoin('https://www.vidlii.com/user/', uploader) uploader_url = urljoin('https://www.vidlii.com/user/', uploader)
@ -93,29 +86,28 @@ class VidliiIE(InfoExtractor):
default=None, fatal=False)) default=None, fatal=False))
categories = self._html_search_regex( categories = self._html_search_regex(
r'<div>Category:\s*<\/div>[\s\r]*<div>[\s\r]*<a href="\/videos\?c=[^>]*>([^<]*?)<\/a>', webpage, 'categories', r'<div>Category:\s*</div>[\s\r]*<div>[\s\r]*<a href="/videos\?c=[^>]*>([^<]*?)</a>', webpage, 'categories',
default=None, fatal=False) default=None, fatal=False)
tags = re.findall(r'<a href="/results\?q=[^>]*>\s*([^<]*)</a>', webpage) or None tags = re.findall(r'<a href="/results\?q=[^>]*>\s*([^<]*)</a>', webpage) or None
duration = int_or_none( duration = int_or_none(
self._html_search_meta('video:duration', webpage, 'duration', default=False, fatal=False) or self._html_search_meta('video:duration', webpage, 'duration', default=False, fatal=False) or
videoInfo_dict.get("dur")) videoInfo_dict.get('dur'))
view_count_fb = re.findall(r'<strong>([^<]*?)</strong>', get_element_by_class("w_views", webpage)) view_count_fb = re.findall(r'<strong>([^<]*?)</strong>', get_element_by_class('w_views', webpage) or '')
view_count_fb = view_count_fb[0] if view_count_fb else None view_count_fb = view_count_fb[0] if view_count_fb else None
view_count = int_or_none(self._html_search_regex(r'Views:[^<]*<strong>([^<]*?)<\/strong>', webpage, 'view_count', view_count = int_or_none(self._html_search_regex(r'Views:[^<]*<strong>([^<]*?)</strong>', webpage, 'view_count',
default=None, fatal=False)) or int_or_none(view_count_fb) default=None, fatal=False)) or int_or_none(view_count_fb)
comment_count_patterns = [r'Comments:[^<]*<strong>([^<]*?)</strong>',
r'<span[^>]+id="cmt_num"[^>]*>([^<]+?)</span>']
comment_count = int_or_none( comment_count = int_or_none(
self._html_search_regex(r'Comments:[^<]*<strong>([^<]*?)<\/strong>|<span[^>]+id="cmt_num"[^>]*>([' self._html_search_regex(comment_count_patterns, webpage, 'comment_count', default=None, fatal=False))
r'^<]+?)<\/span>', webpage, 'comment_count',
default=None, fatal=False))
average_rating = float_or_none( average_rating = float_or_none(
self._html_search_regex(r'{[\s\r]*\$\("#rateYo"\).rateYo\({[^}]*rating:\s*([^,]*?),[^}.]*}', self._html_search_regex(r'rating:\s*([^,]*),',
webpage, 'average_rating', default=None, fatal=False)) webpage, 'average_rating', default=None, fatal=False))
thumbnail_link = videoInfo_dict.get("img") thumbnail_link = videoInfo_dict.get('img')
thumbnail = urljoin('https://www.vidlii.com/', thumbnail_link) thumbnail = urljoin('https://www.vidlii.com/', thumbnail_link)
video_type = self._og_search_property('type', webpage, 'type')
return { return {
'id': video_id, 'id': video_id,
@ -132,5 +124,4 @@ class VidliiIE(InfoExtractor):
'comment_count': comment_count, 'comment_count': comment_count,
'average_rating': average_rating, 'average_rating': average_rating,
'thumbnail': thumbnail, 'thumbnail': thumbnail,
'type': video_type
} }