Merge branch 'viu' of https://github.com/ping/youtube-dl into viu
# Conflicts: # youtube_dl/extractor/__init__.py # youtube_dl/extractor/viu.py
This commit is contained in:
lkho
commit
d5ca9a12ca
@ -1,151 +1,136 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import ExtractorError
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
int_or_none,
|
||||
clean_html,
|
||||
)
|
||||
|
||||
|
||||
class ViuIE(InfoExtractor):
|
||||
IE_DESC = 'Viu.com'
|
||||
_VALID_URL = r'''(?x)^
|
||||
(?:https?://|//) # http(s):// or protocol-independent URL
|
||||
(?:(?:www\.)?viu\.com) # main hostname
|
||||
.* # path components
|
||||
/vod/ # path anchor, make sure it is a video page
|
||||
(?P<id>[0-9]+) # product_id (=video_id)
|
||||
/ # trailing slash is required
|
||||
.* # anything can follow
|
||||
$'''
|
||||
_config_url = 'http://www.viu.com/ott/hk/v1/js/config.js'
|
||||
_js_var_pattern = r'var\s+%s\s*=\s*(.*)\s*;' # get a value from javascript variable declaration
|
||||
_subtitle_lang = {
|
||||
'1': 'zh_hk',
|
||||
'2': 'zh_cn',
|
||||
'3': 'en',
|
||||
}
|
||||
_TESTS = [
|
||||
{
|
||||
'url': 'http://www.viu.com/ott/hk/zh-hk/vod/17732/Doctors',
|
||||
'md5': '563f4efac43f62873bab47ba0e84d2f9',
|
||||
IE_NAME = 'viu'
|
||||
_VALID_URL = r'https?://www\.viu\.com/ott/[a-z]+/[a-z\-]+/vod/(?P<id>[0-9]+)/'
|
||||
_TESTS = [{
|
||||
'url': 'http://www.viu.com/ott/sg/en-us/vod/3421/The%20Prime%20Minister%20and%20I',
|
||||
'info_dict': {
|
||||
'id': '17732',
|
||||
'title': 'Doctors 13 [我想念嫲嫲的湯飯]',
|
||||
'thumbnail': 're:(https?:)?//[0-9a-zA-Z]+\.cloudfront\.net/2849538801/bb28adaf740c168ecb9340e73ddc9c5b4e62e313',
|
||||
}
|
||||
'id': '3421',
|
||||
'ext': 'mp4',
|
||||
'title': 'The Prime Minister and I - Episode 17',
|
||||
'description': 'md5:1e7486a619b6399b25ba6a41c0fe5b2c',
|
||||
},
|
||||
{
|
||||
'url': 'https://www.viu.com/ott/hk/zh-hk/vod/16061/',
|
||||
'info_dict': {
|
||||
'id': '16061',
|
||||
'title': 'Doctors 1 [我們的相遇是孽緣嗎？]',
|
||||
'thumbnail': 're:(https?:)?//[0-9a-zA-Z]+\.cloudfront\.net/3543435935/6696b3b32ec1213adbe4f251ba824b019f4c83c1'
|
||||
}
|
||||
'params': {
|
||||
'skip_download': 'm3u8 download',
|
||||
},
|
||||
{
|
||||
'url': '//www.viu.com/ott/hk/zh-hk/vod/16915/%E3%80%8AW%EF%BC%8D%E5%85%A9%E5%80%8B%E4%B8%96%E7%95%8C%E3%80%8B%E9%A0%90%E5%91%8A',
|
||||
'skip': 'Geo-restricted to Singapore',
|
||||
}, {
|
||||
'url': 'http://www.viu.com/ott/hk/zh-hk/vod/7123/%E5%A4%A7%E4%BA%BA%E5%A5%B3%E5%AD%90',
|
||||
'info_dict': {
|
||||
'id': '16915',
|
||||
'title': '《W－兩個世界》預告 1 [漫畫人物姜哲來到現實！]',
|
||||
'thumbnail': 're:(https?:)?//[0-9a-zA-Z]+\.cloudfront\.net/1928834/02b2382ff1799c200f8f03500f9da1d87ea68d22'
|
||||
}
|
||||
'id': '7123',
|
||||
'ext': 'mp4',
|
||||
'title': '大人女子 - Episode 10',
|
||||
'description': 'md5:4eb0d8b08cf04fcdc6bbbeb16043434f',
|
||||
},
|
||||
{
|
||||
'url': 'http://www.viu.com/ott/hk/zh-hk/vod/7379/%E6%88%91%E5%80%91%E7%B5%90%E5%A9%9A%E4%BA%86%20(2015)',
|
||||
'info_dict': {
|
||||
'id': '7379',
|
||||
'title': '我們結婚了 (2015) 301 [養眼夫婦百日合約到期]',
|
||||
'thumbnail': 're:(https?:)?//[0-9a-zA-Z]+\.cloudfront\.net/2521531018/04f4b8c5d94865cb46118e206b7ba5d5329d8064'
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
def search_js_var(self, string, var_name):
|
||||
result = self._search_regex(self._js_var_pattern % var_name, string, var_name)
|
||||
result = re.sub(r'(^\')|(\'$)', '"', result)
|
||||
return json.loads(result) if result is not None else None
|
||||
'params': {
|
||||
'skip_download': 'm3u8 download',
|
||||
},
|
||||
'skip': 'Geo-restricted to Hong Kong',
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
# confirm the product_id from the url with the one found in content script
|
||||
product_id = self.search_js_var(webpage, 'product_id')
|
||||
if video_id != product_id:
|
||||
raise ExtractorError('Video ID in webpage "%s" does not match URL "%s"' % (product_id, video_id))
|
||||
webpage = self._download_webpage(
|
||||
url, video_id, note='Downloading video page')
|
||||
|
||||
mobj = re.search(
|
||||
r'<div class=["\']error-title[^<>]+?>(?P<err>.+?)</div>', webpage, flags=re.DOTALL)
|
||||
|
||||
if mobj:
|
||||
raise ExtractorError(clean_html(mobj.group('err')), expected=True)
|
||||
|
||||
config_js_url = self._search_regex(
|
||||
r'src=(["\'])(?P<api_url>.+?/js/config\.js)(?:\?.+?)?\1', webpage, 'config_js',
|
||||
group='api_url')
|
||||
|
||||
# fetch variables
|
||||
config_js = self._download_webpage(
|
||||
'http://www.viu.com/ott/hk/v1/js/config.js',
|
||||
product_id,
|
||||
'Downloading runtime variables from config.js',
|
||||
'Unable to download config.js')
|
||||
web_api_url = self.search_js_var(config_js, 'web_api_url')
|
||||
web_api_tail = self.search_js_var(config_js, 'web_api_tail')
|
||||
video_url = self.search_js_var(config_js, 'video_url')
|
||||
user_param = "";
|
||||
ut_param = '0';
|
||||
'http://www.viu.com' + config_js_url, video_id, note='Downloading config js')
|
||||
|
||||
# video info
|
||||
url = web_api_url + 'vod/ajax-detail' + web_api_tail + user_param + '&product_id=' + product_id + "&ut=" + ut_param
|
||||
if url.startswith('//'):
|
||||
url = 'https:' + url
|
||||
info = self._download_json(
|
||||
url,
|
||||
product_id,
|
||||
'Downloading video info',
|
||||
'Unable to download video info')
|
||||
info = info['data']
|
||||
current_product = info['current_product']
|
||||
series = info.get('series')
|
||||
# try to strip away commented code which contains test urls
|
||||
config_js = re.sub(r'^//.*?$', '', config_js, flags=re.MULTILINE)
|
||||
config_js = re.sub(r'/\*.*?\*/', '', config_js, flags=re.DOTALL)
|
||||
|
||||
title = '%s %s [%s]' % (series.get('name'), current_product.get('number'), current_product.get('synopsis'))
|
||||
# Slightly different api_url between HK and SG config.js
|
||||
# http://www.viu.com/ott/hk/v1/js/config.js => '//www.viu.com/ott/hk/index.php?r='
|
||||
# http://www.viu.com/ott/sg/v1/js/config.js => 'http://www.viu.com/ott/sg/index.php?r='
|
||||
api_url = self._proto_relative_url(
|
||||
self._search_regex(
|
||||
r'var\s+api_url\s*=\s*(["\'])(?P<api_url>(?:https?:)?//.+?\?r=)\1',
|
||||
config_js, 'api_url', group='api_url'), scheme='http:')
|
||||
|
||||
#stream info
|
||||
ccs_product_id = current_product['ccs_product_id']
|
||||
streams = self._download_json(
|
||||
video_url + ccs_product_id,
|
||||
product_id,
|
||||
'Downloading streams info',
|
||||
'Unable to download streams info')
|
||||
streams = streams['data']['stream']
|
||||
stream_info_url = self._proto_relative_url(
|
||||
self._search_regex(
|
||||
r'var\s+video_url\s*=\s*(["\'])(?P<video_url>(?:https?:)?//.+?\?ccs_product_id=)\1',
|
||||
config_js, 'video_url', group='video_url'), scheme='http:')
|
||||
|
||||
video_info = self._download_json(
|
||||
api_url + 'vod/ajax-detail&platform_flag_label=web&product_id=' + video_id,
|
||||
video_id, note='Downloading video info').get('data', {})
|
||||
|
||||
ccs_product_id = video_info.get('current_product', {}).get('ccs_product_id')
|
||||
|
||||
if not ccs_product_id:
|
||||
raise ExtractorError('This video is not available in your region.', expected=True)
|
||||
|
||||
stream_info = self._download_json(
|
||||
stream_info_url + ccs_product_id, video_id,
|
||||
note='Downloading stream info').get('data', {}).get('stream', {})
|
||||
|
||||
# populate formats
|
||||
formats = []
|
||||
sizes = streams.get('size')
|
||||
for key in streams['url']:
|
||||
height = self._search_regex(r'(\d+)', key, 'video_size', None, False)
|
||||
for vid_format, stream_url in stream_info.get('url', {}).items():
|
||||
br = int_or_none(self._search_regex(
|
||||
r's(?P<br>[0-9]+)p', vid_format, 'bitrate', group='br'))
|
||||
formats.append({
|
||||
'url': streams['url'][key],
|
||||
'protocol': 'm3u8',
|
||||
'ext': 'ts',
|
||||
'format': 'hls with mpeg2-ts segments',
|
||||
'format_id': key,
|
||||
'height': int(height) if height is not None else None,
|
||||
'filesize_approx': int(sizes.get(key)) if sizes and sizes.get(key) is not None else None
|
||||
'format_id': vid_format,
|
||||
'url': stream_url,
|
||||
'vbr': br,
|
||||
'ext': 'mp4',
|
||||
'filesize': stream_info.get('size', {}).get(vid_format)
|
||||
})
|
||||
self._sort_formats(formats)
|
||||
|
||||
#populate subtitles
|
||||
subtitles = {}
|
||||
list = current_product.get('subtitle')
|
||||
if list is not None:
|
||||
for sub in list:
|
||||
if sub['product_subtitle_language_id'] in self._subtitle_lang:
|
||||
subtitles[self._subtitle_lang[sub['product_subtitle_language_id']]] = [{
|
||||
'url': sub['url'],
|
||||
for sub in video_info.get('current_product', {}).get('subtitle', []):
|
||||
subtitles[sub.get('name')] = [{
|
||||
'url': sub.get('url'),
|
||||
'ext': 'srt',
|
||||
}]
|
||||
|
||||
self._sort_formats(formats)
|
||||
episode_info = next(
|
||||
p for p in video_info.get('series', {}).get('product', [])
|
||||
if p.get('product_id') == video_id)
|
||||
|
||||
title = '%s - Episode %s' % (video_info.get('series', {}).get('name'),
|
||||
episode_info.get('number'))
|
||||
description = episode_info.get('description')
|
||||
thumbnail = episode_info.get('cover_image_url')
|
||||
duration = int_or_none(stream_info.get('duration'))
|
||||
series = video_info.get('series', {}).get('name')
|
||||
episode_title = episode_info.get('synopsis')
|
||||
episode_num = int_or_none(episode_info.get('number'))
|
||||
|
||||
return {
|
||||
'id': product_id,
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'description': current_product.get('description'),
|
||||
'thumbnail': current_product.get('cover_image_url'),
|
||||
'duration': streams.get('duration'),
|
||||
'description': description,
|
||||
'series': series,
|
||||
'episode': episode_title,
|
||||
'episode_number': episode_num,
|
||||
'duration': duration,
|
||||
'thumbnail': thumbnail,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles
|
||||
# TODO more properties (see youtube_dl/extractor/common.py)
|
||||
'subtitles': subtitles,
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user