From 6567bbe44a9930722318f79cacbc2d5ac38189ab Mon Sep 17 00:00:00 2001 From: Hamed Bahadorzadeh Date: Sun, 12 May 2019 00:48:56 +0430 Subject: [PATCH 1/4] Mp3, Podcast and playlist extraction added --- youtube_dl/extractor/radiojavan.py | 135 ++++++++++++++++++++++++++--- 1 file changed, 124 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/radiojavan.py b/youtube_dl/extractor/radiojavan.py index 3f74f0c01..77885ca12 100644 --- a/youtube_dl/extractor/radiojavan.py +++ b/youtube_dl/extractor/radiojavan.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import json import re from .common import InfoExtractor @@ -13,7 +14,7 @@ from ..utils import ( class RadioJavanIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?radiojavan\.com/videos/video/(?P[^/]+)/?' + _VALID_URL = r'https?:\/\/(?:www\.)?radiojavan\.com\/(?Pvideos\/video|mp3s\/mp3|playlists\/playlist\/mp3|podcasts\/podcast)\/(?P[^\/]+)\/?' _TEST = { 'url': 'http://www.radiojavan.com/videos/video/chaartaar-ashoobam', 'md5': 'e85208ffa3ca8b83534fca9fe19af95b', @@ -30,25 +31,39 @@ class RadioJavanIE(InfoExtractor): } def _real_extract(self, url): - video_id = self._match_id(url) + content_id = self._match_id(url) + m = self._VALID_URL_RE.match(url) + media_type = m.group('type') + if media_type == "videos/video": + return self.get_video_urls(url, content_id) + elif media_type == "mp3s/mp3": + return self.get_mp3_urls(url, content_id) + elif media_type == "podcasts/podcast": + return self.get_podcast_urls(url, content_id) + elif media_type == "playlists/playlist/mp3": + return self.get_playlist_urls(url, content_id) + + def get_download_host(self, url, host_url, content_id): download_host = self._download_json( - 'https://www.radiojavan.com/videos/video_host', video_id, - data=urlencode_postdata({'id': video_id}), + host_url, content_id, + data=urlencode_postdata({'id': content_id}), headers={ 'Content-Type': 'application/x-www-form-urlencoded', 'Referer': url, - }).get('host', 'https://host1.rjmusicmedia.com') - - webpage = self._download_webpage(url, video_id) + }).get("host") + return download_host + def get_video_urls(self, url, content_id): + download_host = self.get_download_host(url, 'https://www.radiojavan.com/videos/video_host', content_id) + webpage = self._download_webpage(url, content_id) formats = [] - for format_id, _, video_path in re.findall( + for format_id, _, media_path in re.findall( r'RJ\.video(?P\d+[pPkK])\s*=\s*(["\'])(?P(?:(?!\2).)+)\2', webpage): f = parse_resolution(format_id) f.update({ - 'url': urljoin(download_host, video_path), + 'url': urljoin(download_host, "media", media_path+".mp3"), 'format_id': format_id, }) formats.append(f) @@ -71,8 +86,8 @@ class RadioJavanIE(InfoExtractor): r'class="rating">([\d,]+) dislikes', webpage, 'dislike count', fatal=False)) - return { - 'id': video_id, + url = { + 'id': content_id, 'title': title, 'thumbnail': thumbnail, 'upload_date': upload_date, @@ -81,3 +96,101 @@ class RadioJavanIE(InfoExtractor): 'dislike_count': dislike_count, 'formats': formats, } + print(url) + return url + + def get_mp3_urls(self, url, content_id): + download_host = self.get_download_host(url, 'https://www.radiojavan.com/mp3s/mp3_host', content_id) + webpage = self._download_webpage(url, content_id) + formats = [] + for media_path in re.findall( + r'RJ.currentMP3Url\s*=\s*[\"\'](?P.+)[\"\'];', + webpage): + f = { + 'url': urljoin(download_host, "media/" + media_path + ".mp3"), + 'vcodec': 'none', + } + formats.append(f) + + title = self._og_search_title(webpage) + thumbnail = self._og_search_thumbnail(webpage) + + upload_date = unified_strdate(self._search_regex( + r'class="dateAdded">Date added: ([^<]+)<', + webpage, 'upload date', fatal=False)) + + view_count = str_to_int(self._search_regex( + r'class="views">Plays: ([\d,]+)', + webpage, 'view count', fatal=False)) + like_count = str_to_int(self._search_regex( + r'class="rating">([\d,]+) likes', + webpage, 'like count', fatal=False)) + dislike_count = str_to_int(self._search_regex( + r'class="rating">([\d,]+) dislikes', + webpage, 'dislike count', fatal=False)) + + url = { + 'id': content_id, + 'title': title, + 'thumbnail': thumbnail, + 'upload_date': upload_date, + 'view_count': view_count, + 'like_count': like_count, + 'dislike_count': dislike_count, + 'formats': formats, + } + print(url) + return url + + def get_playlist_urls(self, url, content_id): + webpage = self._download_webpage("https://www.radiojavan.com/mp3s/playlist_start?id="+content_id, content_id) + urls = [] + for mp3s in re.findall(r'RJ.relatedMP3\s*=\s*(?P\[.+\]);', webpage): + mp3s_info = json.loads(mp3s) + for mp3_info in mp3s_info: + url = self.get_mp3_urls("https://www.radiojavan.com/mp3s/mp3/"+mp3_info['next'], mp3_info['next']) + urls.append(url) + print(urls) + return urls + def get_podcast_urls(self, url, content_id): + download_host = self.get_download_host(url, 'https://www.radiojavan.com/podcasts/podcast_host', content_id) + webpage = self._download_webpage(url, content_id) + formats = [] + for media_path in re.findall( + r'RJ.currentMP3Url\s*=\s*["\'](?P.+)["\'];', + webpage): + f = { + 'url': urljoin(download_host, "media/"+media_path+".mp3"), + 'vcodec': 'none', + } + formats.append(f) + + title = self._og_search_title(webpage) + thumbnail = self._og_search_thumbnail(webpage) + + upload_date = unified_strdate(self._search_regex( + r'class="dateAdded">Date added: ([^<]+)<', + webpage, 'upload date', fatal=False)) + + view_count = str_to_int(self._search_regex( + r'class="views">Plays: ([\d,]+)', + webpage, 'view count', fatal=False)) + like_count = str_to_int(self._search_regex( + r'class="rating">([\d,]+) likes', + webpage, 'like count', fatal=False)) + dislike_count = str_to_int(self._search_regex( + r'class="rating">([\d,]+) dislikes', + webpage, 'dislike count', fatal=False)) + + url = { + 'id': content_id, + 'title': title, + 'thumbnail': thumbnail, + 'upload_date': upload_date, + 'view_count': view_count, + 'like_count': like_count, + 'dislike_count': dislike_count, + 'formats': formats, + } + print(url) + return url \ No newline at end of file From dcb8c210b404751fe58355790349bc66eb792ff9 Mon Sep 17 00:00:00 2001 From: Hamed Bahadorzadeh Date: Sun, 12 May 2019 01:35:08 +0430 Subject: [PATCH 2/4] Tests added and passed. Unnecessary prints removed. --- youtube_dl/extractor/radiojavan.py | 54 ++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/youtube_dl/extractor/radiojavan.py b/youtube_dl/extractor/radiojavan.py index 77885ca12..dcc143fea 100644 --- a/youtube_dl/extractor/radiojavan.py +++ b/youtube_dl/extractor/radiojavan.py @@ -15,7 +15,7 @@ from ..utils import ( class RadioJavanIE(InfoExtractor): _VALID_URL = r'https?:\/\/(?:www\.)?radiojavan\.com\/(?Pvideos\/video|mp3s\/mp3|playlists\/playlist\/mp3|podcasts\/podcast)\/(?P[^\/]+)\/?' - _TEST = { + _TESTS = [{ 'url': 'http://www.radiojavan.com/videos/video/chaartaar-ashoobam', 'md5': 'e85208ffa3ca8b83534fca9fe19af95b', 'info_dict': { @@ -28,7 +28,34 @@ class RadioJavanIE(InfoExtractor): 'like_count': int, 'dislike_count': int, } - } + }, + { + 'url': 'https://www.radiojavan.com/podcasts/podcast/Mohsens-House-92', + 'md5': '6ccde3249f86ede1dcbde0a441a6dc91', + 'info_dict': { + 'upload_date': '20190421', + 'id': 'Mohsens-House-92', + 'title': "Mohsen's House Podcast (Episode 92)", + 'dislike_count': int, + 'like_count': int, + 'view_count': int, + 'ext': 'mp3', + 'thumbnail': r're:^https?://.*\.jpe?g$', + } + },{ + 'url': 'https://www.radiojavan.com/mp3s/mp3/Sirvan-Khosravi-Dorost-Nemisham', + 'md5': '3fe3d839617ab3d41348bd4f1af04e70', + 'info_dict':{ + 'upload_date': '20190506', + 'dislike_count': int, + 'like_count': int, + 'view_count': int, + 'ext': 'mp3', + 'thumbnail': r're:^https?://.*\.jpe?g$', + 'title': 'Sirvan Khosravi - Dorost Nemisham', + 'id': 'Sirvan-Khosravi-Dorost-Nemisham' + } + }] def _real_extract(self, url): content_id = self._match_id(url) @@ -37,7 +64,7 @@ class RadioJavanIE(InfoExtractor): if media_type == "videos/video": return self.get_video_urls(url, content_id) - elif media_type == "mp3s/mp3": + elif media_type == "mp3s/mp3": return self.get_mp3_urls(url, content_id) elif media_type == "podcasts/podcast": return self.get_podcast_urls(url, content_id) @@ -63,7 +90,7 @@ class RadioJavanIE(InfoExtractor): webpage): f = parse_resolution(format_id) f.update({ - 'url': urljoin(download_host, "media", media_path+".mp3"), + 'url': urljoin(download_host, media_path), 'format_id': format_id, }) formats.append(f) @@ -96,7 +123,6 @@ class RadioJavanIE(InfoExtractor): 'dislike_count': dislike_count, 'formats': formats, } - print(url) return url def get_mp3_urls(self, url, content_id): @@ -139,19 +165,20 @@ class RadioJavanIE(InfoExtractor): 'dislike_count': dislike_count, 'formats': formats, } - print(url) return url def get_playlist_urls(self, url, content_id): - webpage = self._download_webpage("https://www.radiojavan.com/mp3s/playlist_start?id="+content_id, content_id) + webpage = self._download_webpage("https://www.radiojavan.com/mp3s/playlist_start?id=" + content_id, content_id) + title = self._og_search_title(webpage) + infopage = self._download_webpage("https://www.radiojavan.com/mp3s/playlist_start?id=" + content_id, content_id) urls = [] - for mp3s in re.findall(r'RJ.relatedMP3\s*=\s*(?P\[.+\]);', webpage): + for mp3s in re.findall(r'RJ.relatedMP3\s*=\s*(?P\[.+\]);', infopage): mp3s_info = json.loads(mp3s) - for mp3_info in mp3s_info: - url = self.get_mp3_urls("https://www.radiojavan.com/mp3s/mp3/"+mp3_info['next'], mp3_info['next']) + for mp3_info in mp3s_info: + url = self.get_mp3_urls("https://www.radiojavan.com/mp3s/mp3/" + mp3_info['next'], mp3_info['next']) urls.append(url) - print(urls) return urls + def get_podcast_urls(self, url, content_id): download_host = self.get_download_host(url, 'https://www.radiojavan.com/podcasts/podcast_host', content_id) webpage = self._download_webpage(url, content_id) @@ -160,7 +187,7 @@ class RadioJavanIE(InfoExtractor): r'RJ.currentMP3Url\s*=\s*["\'](?P.+)["\'];', webpage): f = { - 'url': urljoin(download_host, "media/"+media_path+".mp3"), + 'url': urljoin(download_host, "media/" + media_path + ".mp3"), 'vcodec': 'none', } formats.append(f) @@ -192,5 +219,4 @@ class RadioJavanIE(InfoExtractor): 'dislike_count': dislike_count, 'formats': formats, } - print(url) - return url \ No newline at end of file + return url From fc31de2581a69077b1c1377311a92153fa274402 Mon Sep 17 00:00:00 2001 From: Hamed Bahadorzadeh Date: Sun, 12 May 2019 01:48:18 +0430 Subject: [PATCH 3/4] string extraction corrected --- youtube_dl/extractor/radiojavan.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/radiojavan.py b/youtube_dl/extractor/radiojavan.py index dcc143fea..5d5f361b8 100644 --- a/youtube_dl/extractor/radiojavan.py +++ b/youtube_dl/extractor/radiojavan.py @@ -172,11 +172,13 @@ class RadioJavanIE(InfoExtractor): title = self._og_search_title(webpage) infopage = self._download_webpage("https://www.radiojavan.com/mp3s/playlist_start?id=" + content_id, content_id) urls = [] - for mp3s in re.findall(r'RJ.relatedMP3\s*=\s*(?P\[.+\]);', infopage): - mp3s_info = json.loads(mp3s) - for mp3_info in mp3s_info: - url = self.get_mp3_urls("https://www.radiojavan.com/mp3s/mp3/" + mp3_info['next'], mp3_info['next']) - urls.append(url) + mp3s = str(self._search_regex( + r'RJ.relatedMP3\s*=\s*(?P\[.+\]);', + infopage, 'mp3s', fatal=False)) + mp3s_info = json.loads(mp3s) + for mp3_info in mp3s_info: + url = self.get_mp3_urls("https://www.radiojavan.com/mp3s/mp3/" + mp3_info['next'], mp3_info['next']) + urls.append(url) return urls def get_podcast_urls(self, url, content_id): From 3a11950cd7510bd07139e64311b46922f25247f0 Mon Sep 17 00:00:00 2001 From: Hamed Bahadorzadeh Date: Sun, 12 May 2019 01:54:47 +0430 Subject: [PATCH 4/4] checked and corrected according to flake8 --- youtube_dl/extractor/radiojavan.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/radiojavan.py b/youtube_dl/extractor/radiojavan.py index 5d5f361b8..6d09efbcc 100644 --- a/youtube_dl/extractor/radiojavan.py +++ b/youtube_dl/extractor/radiojavan.py @@ -42,10 +42,10 @@ class RadioJavanIE(InfoExtractor): 'ext': 'mp3', 'thumbnail': r're:^https?://.*\.jpe?g$', } - },{ + }, { 'url': 'https://www.radiojavan.com/mp3s/mp3/Sirvan-Khosravi-Dorost-Nemisham', 'md5': '3fe3d839617ab3d41348bd4f1af04e70', - 'info_dict':{ + 'info_dict': { 'upload_date': '20190506', 'dislike_count': int, 'like_count': int, @@ -168,8 +168,6 @@ class RadioJavanIE(InfoExtractor): return url def get_playlist_urls(self, url, content_id): - webpage = self._download_webpage("https://www.radiojavan.com/mp3s/playlist_start?id=" + content_id, content_id) - title = self._og_search_title(webpage) infopage = self._download_webpage("https://www.radiojavan.com/mp3s/playlist_start?id=" + content_id, content_id) urls = [] mp3s = str(self._search_regex(