R-yan/l1ving_youtube-dl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[vier] fix VierVideos extraction

Browse Source
This commit is contained in:
remitamine 2015-09-09 03:35:55 +01:00
parent 47004d9579
commit e29d6f35b5
1 changed files with 44 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
youtube_dl/extractor/vier.py
View File

@ -4,6 +4,10 @@ from __future__ import unicode_literals
import re import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import (
compat_urllib_request,
compat_urllib_parse,
)
class VierIE(InfoExtractor): class VierIE(InfoExtractor):
@ -87,35 +91,52 @@ class VierVideosIE(InfoExtractor):
'playlist_mincount': 13, 'playlist_mincount': 13,
}] }]
def extract_videos(self, webpage, page_id):
entries = [
self.url_result('http://www.vier.be%s' % video_url, 'Vier')
for video_url in re.findall(
r'<h3><a href="(/[^/]+/videos/[^/]+(?:/\d+)?)">', webpage)]
return entries
def extract_next_page_id(self, webpage):
return self._html_search_regex(r'<a\s+href="(?:/[^/]+)+\?page=(\d+)">Meer</a>', webpage, 'page_id', None, False)
def _real_extract(self, url): def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
program = mobj.group('program') program = mobj.group('program')
page_id = mobj.group('page')
webpage = self._download_webpage(url, program) webpage = self._download_webpage(url, program)
page_id = mobj.group('page') views = self._parse_json(
if page_id: self._search_regex(
page_id = int(page_id) 'jQuery.extend\(\s*Drupal.settings\s*,\s*({.*})\);',
start_page = page_id webpage, 'Drupal.settings'),
last_page = start_page + 1 page_id
playlist_id = '%s-page%d' % (program, page_id) ).get('views')
else: ajax_path = views.get('ajax_path')
start_page = 0 ajaxViews = views.get('ajaxViews')
last_page = int(self._search_regex( ajaxView = ajaxViews.get(list(ajaxViews)[0])
r'videos\?page=(\d+)">laatste</a>',
webpage, 'last page', default=0)) + 1
playlist_id = program
entries = [] entries = self.extract_videos(webpage, program)
for current_page_id in range(start_page, last_page): if page_id:
current_page = self._download_webpage( playlist_id = '%s-page%s' % (program, page_id)
'http://www.vier.be/%s/videos?page=%d' % (program, current_page_id), else:
program, playlist_id = program
'Downloading page %d' % (current_page_id + 1)) if current_page_id != page_id else webpage page_id = self.extract_next_page_id(webpage)
page_entries = [ while page_id:
self.url_result('http://www.vier.be' + video_url, 'Vier') ajaxView['page'] = page_id
for video_url in re.findall( request = compat_urllib_request.Request(
r'<h3><a href="(/[^/]+/videos/[^/]+(?:/\d+)?)">', current_page)] 'http://www.vier.be%s' % ajax_path,
entries.extend(page_entries) compat_urllib_parse.urlencode(ajaxView),
{'Content-Type': 'application/x-www-form-urlencoded'}
)
json_data = self._download_json(
request,
program,
'Downloading page %s' % (page_id))
data = json_data[1].get('data')
entries.extend(self.extract_videos(data, page_id))
page_id = self.extract_next_page_id(data)
return self.playlist_result(entries, playlist_id) return self.playlist_result(entries, playlist_id)