R-yan/l1ving_youtube-dl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[vidlii] Fixed suggestions for new extractor (see pull request for issue #14779)

Browse Source 
- improved regular expressions
- implemented use of helper functions js_to_json, unified_strdate
This commit is contained in:
Patrick Jattke 2017-11-22 20:35:02 +01:00
parent 0d3ea70cf3
commit 5463f0abd4
1 changed files with 33 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
youtube_dl/extractor/vidlii.py
View File

@ -7,7 +7,7 @@ from .common import InfoExtractor
from ..utils import ( from ..utils import (
int_or_none, int_or_none,
get_element_by_id, get_element_by_class, strip_or_none, get_element_by_id, get_element_by_class, strip_or_none,
float_or_none, urljoin) float_or_none, urljoin, js_to_json, unified_strdate)
class VidliiIE(InfoExtractor): class VidliiIE(InfoExtractor):
@ -63,51 +63,58 @@ class VidliiIE(InfoExtractor):
video_id = self._match_id(url) video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id) webpage = self._download_webpage(url, video_id)
# extract videoInfo variable for further use
videoInfo_dict = self._parse_json(js_to_json(self._html_search_regex(r'var videoInfo\s*=\s*({[^}]*})', webpage,
'videoInfo', fatal=False)), video_id)
# extract basic properties of video # extract basic properties of video
title = self._html_search_regex(r'<h1>(.+?)</h1>', webpage, 'title', default=None) or self._html_search_regex( title = (self._html_search_regex(r'<title>([^<]+?)</title>', webpage, 'title', default='') or
r'<title>([^<]+?)</title>', webpage, 'title', default="").replace( self._html_search_meta('twitter:title', webpage, 'title', default='')).replace(" - VidLii",
" - VidLii", "") or self._html_search_meta('twitter:title', webpage, 'title', "") or self._html_search_regex(
default="").replace(" - VidLii", "") r'<h1>(.+?)</h1>', webpage, 'title', default=None)
description = strip_or_none(get_element_by_id('des_text', webpage)) description = strip_or_none(get_element_by_id('des_text', webpage))
uploader = self._html_search_regex(r'<div[^>]+class="wt_person"[^>]*>(?:[^<]+)<a href="\/user\/[^>]*?>([^<]*?)<', uploader = self._html_search_regex(
webpage, 'uploader', default=None) or self._html_search_regex( r'<div[^>]+class="wt_person"[^>]*>(?:[^<]+)<a href="/user/[^>]*?>([^<]*?)<|<img src="[^>]+?class=('
r'<img src="[^>]+?class=["\']avt2\s*["\'][^>]+?alt=["\']([^"\']+?)["\']', r'?:"avt2\s*"|\'avt2\s*\')[^>]+?alt=(?:"([^"]+?)"|\'([^\']+?)\')>', webpage, 'uploader', default=None,
webpage, 'uploader', default=None) fatal=False)
url = self._html_search_regex(r'videoInfo[\s]*=[\s]*{[^}]*src:[\s]*(?:"|\')([^"]*?)(?:"|\')', webpage, 'url', video_url = videoInfo_dict.get("src")
default=None)
# get additional properties # get additional properties
uploader_url = urljoin("https://www.vidlii.com/user/", uploader) uploader_url = urljoin('https://www.vidlii.com/user/', uploader)
# returns date as YYYYMMDD # returns date as YYYYMMDD
upload_date = self._html_search_meta('datePublished', webpage, 'upload_date', default="").replace("-", "") upload_date = unified_strdate(
self._html_search_meta('datePublished', webpage, 'upload_date', default=None,
fatal=False) or self._html_search_regex(r'<date>(['r'^<]*?)</date>', webpage,
'upload_date',
default=None, fatal=False))
categories = self._html_search_regex( categories = self._html_search_regex(
r'<div>Category:\s*<\/div>[\s\r]*<div>[\s\r]*<a href="\/videos\?c=[^>]*>([^<]*?)<\/a>', webpage, 'categories', r'<div>Category:\s*<\/div>[\s\r]*<div>[\s\r]*<a href="\/videos\?c=[^>]*>([^<]*?)<\/a>', webpage, 'categories',
default=None) default=None, fatal=False)
tags = re.findall(r'<a href="/results\?q=[^>]*>[\s]*([^<]*)</a>', webpage) or None tags = re.findall(r'<a href="/results\?q=[^>]*>\s*([^<]*)</a>', webpage) or None
duration = int_or_none(self._html_search_meta('video:duration', webpage, 'duration', default=False)) or int_or_none( duration = int_or_none(
self._html_search_regex(r'videoInfo[^=]*=[^{]*{[^}]*dur:([^,}]*?),', webpage, 'duration', default=None)) self._html_search_meta('video:duration', webpage, 'duration', default=False, fatal=False) or
videoInfo_dict.get("dur"))
view_count_fb = re.findall(r'<strong>([^<]*?)</strong>', get_element_by_class("w_views", webpage)) view_count_fb = re.findall(r'<strong>([^<]*?)</strong>', get_element_by_class("w_views", webpage))
view_count_fb = view_count_fb[0] if view_count_fb else None view_count_fb = view_count_fb[0] if view_count_fb else None
view_count = int_or_none(self._html_search_regex(r'Views:[^<]*<strong>([^<]*?)<\/strong>', webpage, 'view_count', view_count = int_or_none(self._html_search_regex(r'Views:[^<]*<strong>([^<]*?)<\/strong>', webpage, 'view_count',
default=None)) or int_or_none(view_count_fb) default=None, fatal=False)) or int_or_none(view_count_fb)
comment_count = int_or_none( comment_count = int_or_none(
self._html_search_regex(r'Comments:[^<]*<strong>([^<]*?)<\/strong>', webpage, 'comment_count', self._html_search_regex(r'Comments:[^<]*<strong>([^<]*?)<\/strong>|<span[^>]+id="cmt_num"[^>]*>(['
default=None) or r'^<]+?)<\/span>', webpage, 'comment_count',
self._html_search_regex(r'<span[^>]+id="cmt_num"[^>]*>([^<]+?)<\/span>', webpage, 'comment_count', default=None)) default=None, fatal=False))
average_rating = float_or_none( average_rating = float_or_none(
self._html_search_regex(r'{[\s\r]*\$\("#rateYo"\).rateYo\({[^}]*rating:\s*([^,]*?),[^}.]*}', self._html_search_regex(r'{[\s\r]*\$\("#rateYo"\).rateYo\({[^}]*rating:\s*([^,]*?),[^}.]*}',
webpage, 'average_rating', default=None)) webpage, 'average_rating', default=None, fatal=False))
thumbnail_link = self._html_search_regex(r'videoInfo[\s]*=[\s]*{[^}]*img:[\s]*(?:"|\')([^"]*?)(?:"|\')', webpage, thumbnail_link = videoInfo_dict.get("img")
'thumbnail', default=None) thumbnail = urljoin('https://www.vidlii.com/', thumbnail_link)
thumbnail = urljoin("https://www.vidlii.com/", thumbnail_link)
video_type = self._og_search_property('type', webpage, 'type') video_type = self._og_search_property('type', webpage, 'type')
return { return {
@ -115,7 +122,7 @@ class VidliiIE(InfoExtractor):
'title': title, 'title': title,
'description': description, 'description': description,
'uploader': uploader, 'uploader': uploader,
'url': url, 'url': video_url,
'uploader_url': uploader_url, 'uploader_url': uploader_url,
'upload_date': upload_date, 'upload_date': upload_date,
'categories': categories, 'categories': categories,