From d75767a0e68c4eb5d335e0d64ab4a28cd6bdd0e3 Mon Sep 17 00:00:00 2001 From: Gabor Miklos Date: Fri, 17 Nov 2017 16:44:04 +0200 Subject: [PATCH 1/4] [xhamster] Added support for mobile site; fixed thumbnails and tests --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/xhamster.py | 57 +++++++++++++++++++++++------- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index d084707ee..9feadebb5 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -1302,6 +1302,7 @@ from .xfileshare import XFileShareIE from .xhamster import ( XHamsterIE, XHamsterEmbedIE, + XHamsterMobileIE, ) from .xiami import ( XiamiSongIE, diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index be3624ef2..7ebb4f8c7 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -14,15 +14,19 @@ from ..utils import ( ) -class XHamsterIE(InfoExtractor): - _VALID_URL = r'''(?x) - https?:// - (?:.+?\.)?xhamster\.com/ - (?: - movies/(?P\d+)/(?P[^/]*)\.html| - videos/(?P[^/]*)-(?P\d+) - ) - ''' +class XHamsterBase(InfoExtractor): + _VALID_URL_TEMPLATE = r'''(?x) + https?:// + (?:(?:www|[a-z]{2})\.)?%sxhamster\.com/ + (?: + movies/(?P\d+)/(?P[^/]*)\.html| + videos/(?P[^/]*)-(?P\d+) + ) + ''' + + +class XHamsterIE(XHamsterBase): + _VALID_URL = XHamsterBase._VALID_URL_TEMPLATE % '' _TESTS = [{ 'url': 'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html', @@ -148,8 +152,8 @@ class XHamsterIE(InfoExtractor): webpage, 'uploader', default='anonymous') thumbnail = self._search_regex( - [r'''thumb\s*:\s*(?P["'])(?P.+?)(?P=q)''', - r''']+poster=(?P["'])(?P.+?)(?P=q)[^>]*>'''], + [r'''["']thumbUrl["']\s*:\s*(?P["'])(?P.+?)(?P=q)''', + r''']+"poster"=(?P["'])(?P.+?)(?P=q)[^>]*>'''], webpage, 'thumbnail', fatal=False, group='thumbnail') duration = parse_duration(self._search_regex( @@ -195,7 +199,7 @@ class XHamsterIE(InfoExtractor): class XHamsterEmbedIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?xhamster\.com/xembed\.php\?video=(?P\d+)' + _VALID_URL = r'https?://(?:(?:www|[a-z]{2})\.)?xhamster\.com/xembed\.php\?video=(?P\d+)' _TEST = { 'url': 'http://xhamster.com/xembed.php?video=3328539', 'info_dict': { @@ -203,7 +207,7 @@ class XHamsterEmbedIE(InfoExtractor): 'ext': 'mp4', 'title': 'Pen Masturbation', 'upload_date': '20140728', - 'uploader_id': 'anonymous', + 'uploader': 'ManyakisArt', 'duration': 5, 'age_limit': 18, } @@ -231,3 +235,30 @@ class XHamsterEmbedIE(InfoExtractor): video_url = dict_get(vars, ('downloadLink', 'homepageLink', 'commentsLink', 'shareUrl')) return self.url_result(video_url, 'XHamster') + + +class XHamsterMobileIE(XHamsterBase): + _VALID_URL = XHamsterBase._VALID_URL_TEMPLATE % 'm\.' + + _TEST = { + 'url': 'https://m.xhamster.com/videos/cute-teen-jacqueline-solo-masturbation-8559111', + 'md5': 'e863ca8b0cc2e3d03ba3ef3c6288207c', + 'info_dict': { + 'id': '8559111', + 'display_id': 'cute-teen-jacqueline-solo-masturbation', + 'ext': 'mp4', + 'title': 'Cute Teen Jacqueline Solo Masturbation', + 'description': str, + 'upload_date': '20171117', + 'uploader': '10tz4d0114r5', + 'duration': 395, + 'age_limit': 18, + 'categories': ['Teen Dreams Channel', 'Amateur', 'Fingering', 'HD Videos', 'Masturbation', 'Small Tits', + 'Teens', 'Cute Teen', 'Solo', 'Solo Masturbation'] + } + } + + def _real_extract(self, url): + desktop_url = re.sub(r'^(https?://(?:(?:www|[a-z]{2})\.)?)m\.', r'\1', url) + + return self.url_result(desktop_url, 'XHamster') From 04fbacfe9ad406bdf545b9d6c12716c728161227 Mon Sep 17 00:00:00 2001 From: Gabor Miklos Date: Mon, 20 Nov 2017 13:09:15 +0200 Subject: [PATCH 2/4] Removed XHamsterMobileIE, modified XHamsterIE instead --- youtube_dl/extractor/extractors.py | 1 - youtube_dl/extractor/xhamster.py | 73 +++++++++++++----------------- 2 files changed, 31 insertions(+), 43 deletions(-) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 9feadebb5..d084707ee 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -1302,7 +1302,6 @@ from .xfileshare import XFileShareIE from .xhamster import ( XHamsterIE, XHamsterEmbedIE, - XHamsterMobileIE, ) from .xiami import ( XiamiSongIE, diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index 7ebb4f8c7..a4488679b 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -14,19 +14,15 @@ from ..utils import ( ) -class XHamsterBase(InfoExtractor): - _VALID_URL_TEMPLATE = r'''(?x) - https?:// - (?:(?:www|[a-z]{2})\.)?%sxhamster\.com/ - (?: - movies/(?P\d+)/(?P[^/]*)\.html| - videos/(?P[^/]*)-(?P\d+) - ) - ''' - - -class XHamsterIE(XHamsterBase): - _VALID_URL = XHamsterBase._VALID_URL_TEMPLATE % '' +class XHamsterIE(InfoExtractor): + _VALID_URL = r'''(?x) + https?:// + (?:.+?\.)?xhamster\.com/ + (?: + movies/(?P\d+)/(?P[^/]*)\.html| + videos/(?P[^/]*)-(?P\d+) + ) + ''' _TESTS = [{ 'url': 'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html', @@ -74,6 +70,25 @@ class XHamsterIE(XHamsterBase): 'params': { 'skip_download': True, }, + }, { + # mobile site + 'url': 'https://m.xhamster.com/videos/cute-teen-jacqueline-solo-masturbation-8559111', + 'info_dict': { + 'id': '8559111', + 'display_id': 'cute-teen-jacqueline-solo-masturbation', + 'ext': 'mp4', + 'title': 'Cute Teen Jacqueline Solo Masturbation', + 'description': str, + 'upload_date': '20171117', + 'uploader': '10tz4d0114r5', + 'duration': 395, + 'age_limit': 18, + 'categories': ['Teen Dreams Channel', 'Amateur', 'Fingering', 'HD Videos', 'Masturbation', 'Small Tits', + 'Teens', 'Cute Teen', 'Solo', 'Solo Masturbation'] + }, + 'params': { + 'skip_download': True, + }, }, { 'url': 'https://xhamster.com/movies/2272726/amber_slayed_by_the_knight.html', 'only_matching': True, @@ -92,7 +107,8 @@ class XHamsterIE(XHamsterBase): video_id = mobj.group('id') or mobj.group('id_2') display_id = mobj.group('display_id') or mobj.group('display_id_2') - webpage = self._download_webpage(url, video_id) + desktop_url = re.sub(r'^(https?://(?:.+?\.)?)m\.', r'\1', url) + webpage = self._download_webpage(desktop_url, video_id) error = self._html_search_regex( r']+id=["\']videoClosed["\'][^>]*>(.+?)', @@ -199,7 +215,7 @@ class XHamsterIE(XHamsterBase): class XHamsterEmbedIE(InfoExtractor): - _VALID_URL = r'https?://(?:(?:www|[a-z]{2})\.)?xhamster\.com/xembed\.php\?video=(?P\d+)' + _VALID_URL = r'https?://(?:.+?\.)?xhamster\.com/xembed\.php\?video=(?P\d+)' _TEST = { 'url': 'http://xhamster.com/xembed.php?video=3328539', 'info_dict': { @@ -235,30 +251,3 @@ class XHamsterEmbedIE(InfoExtractor): video_url = dict_get(vars, ('downloadLink', 'homepageLink', 'commentsLink', 'shareUrl')) return self.url_result(video_url, 'XHamster') - - -class XHamsterMobileIE(XHamsterBase): - _VALID_URL = XHamsterBase._VALID_URL_TEMPLATE % 'm\.' - - _TEST = { - 'url': 'https://m.xhamster.com/videos/cute-teen-jacqueline-solo-masturbation-8559111', - 'md5': 'e863ca8b0cc2e3d03ba3ef3c6288207c', - 'info_dict': { - 'id': '8559111', - 'display_id': 'cute-teen-jacqueline-solo-masturbation', - 'ext': 'mp4', - 'title': 'Cute Teen Jacqueline Solo Masturbation', - 'description': str, - 'upload_date': '20171117', - 'uploader': '10tz4d0114r5', - 'duration': 395, - 'age_limit': 18, - 'categories': ['Teen Dreams Channel', 'Amateur', 'Fingering', 'HD Videos', 'Masturbation', 'Small Tits', - 'Teens', 'Cute Teen', 'Solo', 'Solo Masturbation'] - } - } - - def _real_extract(self, url): - desktop_url = re.sub(r'^(https?://(?:(?:www|[a-z]{2})\.)?)m\.', r'\1', url) - - return self.url_result(desktop_url, 'XHamster') From c38ab682eb66acdc4aa78809d37669a5d26c117f Mon Sep 17 00:00:00 2001 From: Gabor Miklos Date: Tue, 5 Dec 2017 16:47:20 +0200 Subject: [PATCH 3/4] Actualized with master: added timestamp to tests --- youtube_dl/extractor/xhamster.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index c09044ac4..0f3bb4856 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -83,6 +83,7 @@ class XHamsterIE(InfoExtractor): 'display_id': 'cute-teen-jacqueline-solo-masturbation', 'ext': 'mp4', 'title': 'Cute Teen Jacqueline Solo Masturbation', + 'timestamp': 1510899603, 'description': str, 'upload_date': '20171117', 'uploader': '10tz4d0114r5', @@ -301,6 +302,7 @@ class XHamsterEmbedIE(InfoExtractor): 'id': '3328539', 'ext': 'mp4', 'title': 'Pen Masturbation', + 'timestamp': 1406581861, 'upload_date': '20140728', 'uploader': 'ManyakisArt', 'duration': 5, From 15aefeaa9a581fd406d07043d103cd8362d4b9e8 Mon Sep 17 00:00:00 2001 From: Gabor Miklos Date: Tue, 5 Dec 2017 18:25:28 +0200 Subject: [PATCH 4/4] Changed mobile site test to 'only_matching' --- youtube_dl/extractor/xhamster.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index 0f3bb4856..68652a22f 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -78,23 +78,7 @@ class XHamsterIE(InfoExtractor): }, { # mobile site 'url': 'https://m.xhamster.com/videos/cute-teen-jacqueline-solo-masturbation-8559111', - 'info_dict': { - 'id': '8559111', - 'display_id': 'cute-teen-jacqueline-solo-masturbation', - 'ext': 'mp4', - 'title': 'Cute Teen Jacqueline Solo Masturbation', - 'timestamp': 1510899603, - 'description': str, - 'upload_date': '20171117', - 'uploader': '10tz4d0114r5', - 'duration': 395, - 'age_limit': 18, - 'categories': ['Teen Dreams Channel', 'Amateur', 'Fingering', 'HD Videos', 'Masturbation', 'Small Tits', - 'Teens', 'Cute Teen', 'Solo', 'Solo Masturbation'] - }, - 'params': { - 'skip_download': True, - }, + 'only_matching': True, }, { 'url': 'https://xhamster.com/movies/2272726/amber_slayed_by_the_knight.html', 'only_matching': True,