R-yan/l1ving_youtube-dl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'viu' of https://github.com/ping/youtube-dl into viu

Browse Source 
# Conflicts:
#	youtube_dl/extractor/__init__.py
#	youtube_dl/extractor/viu.py
This commit is contained in:
lkho 2016-08-06 00:49:56 +08:00
commit d5ca9a12ca
1 changed files with 113 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

241
youtube_dl/extractor/viu.py
View File

@ -1,151 +1,136 @@
# coding: utf-8
from __future__ import unicode_literals
import json
import re
from .common import InfoExtractor
from ..utils import ExtractorError
from ..utils import (
ExtractorError,
int_or_none,
clean_html,
)
class ViuIE(InfoExtractor):
IE_DESC = 'Viu.com'
_VALID_URL = r'''(?x)^
(?:https?://|//) # http(s):// or protocol-independent URL
(?:(?:www\.)?viu\.com) # main hostname
.* # path components
/vod/ # path anchor, make sure it is a video page
(?P<id>[0-9]+) # product_id (=video_id)
/ # trailing slash is required
.* # anything can follow
$'''
_config_url = 'http://www.viu.com/ott/hk/v1/js/config.js'
_js_var_pattern = r'var\s+%s\s*=\s*(.*)\s*;' # get a value from javascript variable declaration
_subtitle_lang = {
'1': 'zh_hk',
'2': 'zh_cn',
'3': 'en',
}
_TESTS = [
{
'url': 'http://www.viu.com/ott/hk/zh-hk/vod/17732/Doctors',
'md5': '563f4efac43f62873bab47ba0e84d2f9',
'info_dict': {
'id': '17732',
'title': 'Doctors 13 [我想念嫲嫲的湯飯]',
'thumbnail': 're:(https?:)?//[0-9a-zA-Z]+\.cloudfront\.net/2849538801/bb28adaf740c168ecb9340e73ddc9c5b4e62e313',
}
IE_NAME = 'viu'
_VALID_URL = r'https?://www\.viu\.com/ott/[a-z]+/[a-z\-]+/vod/(?P<id>[0-9]+)/'
_TESTS = [{
'url': 'http://www.viu.com/ott/sg/en-us/vod/3421/The%20Prime%20Minister%20and%20I',
'info_dict': {
'id': '3421',
'ext': 'mp4',
'title': 'The Prime Minister and I - Episode 17',
'description': 'md5:1e7486a619b6399b25ba6a41c0fe5b2c',
},
{
'url': 'https://www.viu.com/ott/hk/zh-hk/vod/16061/',
'info_dict': {
'id': '16061',
'title': 'Doctors 1 [我們的相遇是孽緣嗎？]',
'thumbnail': 're:(https?:)?//[0-9a-zA-Z]+\.cloudfront\.net/3543435935/6696b3b32ec1213adbe4f251ba824b019f4c83c1'
}
'params': {
'skip_download': 'm3u8 download',
},
{
'url': '//www.viu.com/ott/hk/zh-hk/vod/16915/%E3%80%8AW%EF%BC%8D%E5%85%A9%E5%80%8B%E4%B8%96%E7%95%8C%E3%80%8B%E9%A0%90%E5%91%8A',
'info_dict': {
'id': '16915',
'title': '《W－兩個世界》預告 1 [漫畫人物姜哲來到現實！]',
'thumbnail': 're:(https?:)?//[0-9a-zA-Z]+\.cloudfront\.net/1928834/02b2382ff1799c200f8f03500f9da1d87ea68d22'
}
'skip': 'Geo-restricted to Singapore',
}, {
'url': 'http://www.viu.com/ott/hk/zh-hk/vod/7123/%E5%A4%A7%E4%BA%BA%E5%A5%B3%E5%AD%90',
'info_dict': {
'id': '7123',
'ext': 'mp4',
'title': '大人女子 - Episode 10',
'description': 'md5:4eb0d8b08cf04fcdc6bbbeb16043434f',
},
{
'url': 'http://www.viu.com/ott/hk/zh-hk/vod/7379/%E6%88%91%E5%80%91%E7%B5%90%E5%A9%9A%E4%BA%86%20(2015)',
'info_dict': {
'id': '7379',
'title': '我們結婚了 (2015) 301 [養眼夫婦百日合約到期]',
'thumbnail': 're:(https?:)?//[0-9a-zA-Z]+\.cloudfront\.net/2521531018/04f4b8c5d94865cb46118e206b7ba5d5329d8064'
}
}
]
def search_js_var(self, string, var_name):
result = self._search_regex(self._js_var_pattern % var_name, string, var_name)
result = re.sub(r'(^\')|(\'$)', '"', result)
return json.loads(result) if result is not None else None
'params': {
'skip_download': 'm3u8 download',
},
'skip': 'Geo-restricted to Hong Kong',
}]
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
# confirm the product_id from the url with the one found in content script
product_id = self.search_js_var(webpage, 'product_id')
if video_id != product_id:
raise ExtractorError('Video ID in webpage "%s" does not match URL "%s"' % (product_id, video_id))
# fetch variables
webpage = self._download_webpage(
url, video_id, note='Downloading video page')
mobj = re.search(
r'<div class=["\']error-title[^<>]+?>(?P<err>.+?)</div>', webpage, flags=re.DOTALL)
if mobj:
raise ExtractorError(clean_html(mobj.group('err')), expected=True)
config_js_url = self._search_regex(
r'src=(["\'])(?P<api_url>.+?/js/config\.js)(?:\?.+?)?\1', webpage, 'config_js',
group='api_url')
config_js = self._download_webpage(
'http://www.viu.com/ott/hk/v1/js/config.js',
product_id,
'Downloading runtime variables from config.js',
'Unable to download config.js')
web_api_url = self.search_js_var(config_js, 'web_api_url')
web_api_tail = self.search_js_var(config_js, 'web_api_tail')
video_url = self.search_js_var(config_js, 'video_url')
user_param = "";
ut_param = '0';
'http://www.viu.com' + config_js_url, video_id, note='Downloading config js')
# video info
url = web_api_url + 'vod/ajax-detail' + web_api_tail + user_param + '&product_id=' + product_id + "&ut=" + ut_param
if url.startswith('//'):
url = 'https:' + url
info = self._download_json(
url,
product_id,
'Downloading video info',
'Unable to download video info')
info = info['data']
current_product = info['current_product']
series = info.get('series')
# try to strip away commented code which contains test urls
config_js = re.sub(r'^//.*?$', '', config_js, flags=re.MULTILINE)
config_js = re.sub(r'/\*.*?\*/', '', config_js, flags=re.DOTALL)
title = '%s %s [%s]' % (series.get('name'), current_product.get('number'), current_product.get('synopsis'))
#stream info
ccs_product_id = current_product['ccs_product_id']
streams = self._download_json(
video_url + ccs_product_id,
product_id,
'Downloading streams info',
'Unable to download streams info')
streams = streams['data']['stream']
# populate formats
# Slightly different api_url between HK and SG config.js
# http://www.viu.com/ott/hk/v1/js/config.js => '//www.viu.com/ott/hk/index.php?r='
# http://www.viu.com/ott/sg/v1/js/config.js => 'http://www.viu.com/ott/sg/index.php?r='
api_url = self._proto_relative_url(
self._search_regex(
r'var\s+api_url\s*=\s*(["\'])(?P<api_url>(?:https?:)?//.+?\?r=)\1',
config_js, 'api_url', group='api_url'), scheme='http:')
stream_info_url = self._proto_relative_url(
self._search_regex(
r'var\s+video_url\s*=\s*(["\'])(?P<video_url>(?:https?:)?//.+?\?ccs_product_id=)\1',
config_js, 'video_url', group='video_url'), scheme='http:')
video_info = self._download_json(
api_url + 'vod/ajax-detail&platform_flag_label=web&product_id=' + video_id,
video_id, note='Downloading video info').get('data', {})
ccs_product_id = video_info.get('current_product', {}).get('ccs_product_id')
if not ccs_product_id:
raise ExtractorError('This video is not available in your region.', expected=True)
stream_info = self._download_json(
stream_info_url + ccs_product_id, video_id,
note='Downloading stream info').get('data', {}).get('stream', {})
formats = []
sizes = streams.get('size')
for key in streams['url']:
height = self._search_regex(r'(\d+)', key, 'video_size', None, False)
for vid_format, stream_url in stream_info.get('url', {}).items():
br = int_or_none(self._search_regex(
r's(?P<br>[0-9]+)p', vid_format, 'bitrate', group='br'))
formats.append({
'url': streams['url'][key],
'protocol': 'm3u8',
'ext': 'ts',
'format': 'hls with mpeg2-ts segments',
'format_id': key,
'height': int(height) if height is not None else None,
'filesize_approx': int(sizes.get(key)) if sizes and sizes.get(key) is not None else None
'format_id': vid_format,
'url': stream_url,
'vbr': br,
'ext': 'mp4',
'filesize': stream_info.get('size', {}).get(vid_format)
})
#populate subtitles
subtitles = {}
list = current_product.get('subtitle')
if list is not None:
for sub in list:
if sub['product_subtitle_language_id'] in self._subtitle_lang:
subtitles[self._subtitle_lang[sub['product_subtitle_language_id']]] = [{
'url': sub['url'],
'ext': 'srt',
}]
self._sort_formats(formats)
subtitles = {}
for sub in video_info.get('current_product', {}).get('subtitle', []):
subtitles[sub.get('name')] = [{
'url': sub.get('url'),
'ext': 'srt',
}]
episode_info = next(
p for p in video_info.get('series', {}).get('product', [])
if p.get('product_id') == video_id)
title = '%s - Episode %s' % (video_info.get('series', {}).get('name'),
episode_info.get('number'))
description = episode_info.get('description')
thumbnail = episode_info.get('cover_image_url')
duration = int_or_none(stream_info.get('duration'))
series = video_info.get('series', {}).get('name')
episode_title = episode_info.get('synopsis')
episode_num = int_or_none(episode_info.get('number'))
return {
'id': product_id,
'id': video_id,
'title': title,
'description': current_product.get('description'),
'thumbnail': current_product.get('cover_image_url'),
'duration': streams.get('duration'),
'description': description,
'series': series,
'episode': episode_title,
'episode_number': episode_num,
'duration': duration,
'thumbnail': thumbnail,
'formats': formats,
'subtitles': subtitles
# TODO more properties (see youtube_dl/extractor/common.py)
}
'subtitles': subtitles,
}