From 93dbe3f12fae10a2e556556c4a6aeb7526483647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20Le=20N=C3=A9grate?= Date: Sun, 15 Mar 2015 22:32:06 +0100 Subject: [PATCH 1/2] [mixcloud] Fix extraction of URL and like-count --- youtube_dl/extractor/mixcloud.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py index 1831c6749..63510ce89 100644 --- a/youtube_dl/extractor/mixcloud.py +++ b/youtube_dl/extractor/mixcloud.py @@ -73,14 +73,10 @@ class MixcloudIE(InfoExtractor): webpage = self._download_webpage(url, track_id) preview_url = self._search_regex( - r'\s(?:data-preview-url|m-preview)="([^"]+)"', webpage, 'preview url') - song_url = preview_url.replace('/previews/', '/c/originals/') - template_url = re.sub(r'(stream\d*)', 'stream%d', song_url) + r'\bm-play-on-spacebar\b.*\n?.*\bm-preview="([^"]+)"', webpage, 'preview url') + song_url = re.sub(r'\.mp3$', '.m4a', preview_url.replace('/previews/', '/c/m4a/64/')) + template_url = re.sub(r'(stream\d+)', 'stream%d', song_url) final_song_url = self._get_url(track_id, template_url) - if final_song_url is None: - self.to_screen('Trying with m4a extension') - template_url = template_url.replace('.mp3', '.m4a').replace('originals/', 'm4a/64/') - final_song_url = self._get_url(track_id, template_url) if final_song_url is None: raise ExtractorError('Unable to extract track url') @@ -99,8 +95,7 @@ class MixcloudIE(InfoExtractor): r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False) description = self._og_search_description(webpage) like_count = str_to_int(self._search_regex( - [r'([0-9]+)<'], + r'\bbutton-favorite\b.+m-ajax-toggle-count="([^"]+)"', webpage, 'like count', fatal=False)) view_count = str_to_int(self._search_regex( [r' Date: Mon, 16 Mar 2015 00:20:06 +0100 Subject: [PATCH 2/2] [mixcloud] Try preview server first, then further numbers --- test/test_mixcloud.py | 16 ++++++++++ youtube_dl/extractor/mixcloud.py | 53 +++++++++++++++++++++++++------- 2 files changed, 58 insertions(+), 11 deletions(-) create mode 100644 test/test_mixcloud.py diff --git a/test/test_mixcloud.py b/test/test_mixcloud.py new file mode 100644 index 000000000..56fbe4c87 --- /dev/null +++ b/test/test_mixcloud.py @@ -0,0 +1,16 @@ +import unittest +from youtube_dl.extractor.mixcloud import server_numbers + +class TestMixcloud(unittest.TestCase): + def test_server_numbers(self): + self.assertEqual([n for n in server_numbers(2, (1, 5))], + [2, 3, 1, 4, 5]) + self.assertEqual([n for n in server_numbers(1, (1, 5))], + [1, 2, 3, 4, 5]) + self.assertEqual([n for n in server_numbers(5, (1, 5))], + [5, 4, 3, 2, 1]) + self.assertEqual([n for n in server_numbers(-1, (1, 5))], + [-1, 1, 2, 3, 4, 5]) + +if __name__ == '__main__': + unittest.main() diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py index 63510ce89..254a71940 100644 --- a/youtube_dl/extractor/mixcloud.py +++ b/youtube_dl/extractor/mixcloud.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import re +import itertools from .common import InfoExtractor from ..compat import ( @@ -13,10 +14,11 @@ from ..utils import ( parse_iso8601, ) - class MixcloudIE(InfoExtractor): _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([^/]+)/([^/]+)' IE_NAME = 'mixcloud' + STREAM_PART_RE = r'\bstream(\d+)\b' + SERVER_NUMBERS_BOUNDARIES = (1, 30) _TESTS = [{ 'url': 'http://www.mixcloud.com/dholbach/cryptkeeper/', @@ -48,20 +50,22 @@ class MixcloudIE(InfoExtractor): }, }] - def _get_url(self, track_id, template_url): - server_count = 30 - for i in range(server_count): - url = template_url % i + def _get_url(self, track_id, song_url): + mobj = re.search(self.STREAM_PART_RE, song_url) + if mobj is None: + raise ExtractorError('Unexpected preview URL format: no stream%d') + server_nr = int(mobj.group(1)) + for nr in server_numbers(server_nr, self.SERVER_NUMBERS_BOUNDARIES): + url = re.sub(self.STREAM_PART_RE, 'stream%d' % nr, song_url) try: # We only want to know if the request succeed # don't download the whole file self._request_webpage( HEADRequest(url), track_id, - 'Checking URL %d/%d ...' % (i + 1, server_count + 1)) + 'Checking URL %d/%d ...' % (nr, self.SERVER_NUMBERS_BOUNDARIES[-1])) return url except ExtractorError: pass - return None def _real_extract(self, url): @@ -73,10 +77,10 @@ class MixcloudIE(InfoExtractor): webpage = self._download_webpage(url, track_id) preview_url = self._search_regex( - r'\bm-play-on-spacebar\b.*\n?.*\bm-preview="([^"]+)"', webpage, 'preview url') - song_url = re.sub(r'\.mp3$', '.m4a', preview_url.replace('/previews/', '/c/m4a/64/')) - template_url = re.sub(r'(stream\d+)', 'stream%d', song_url) - final_song_url = self._get_url(track_id, template_url) + r'\bm-play-on-spacebar\b.*\n?.*\bm-preview="([^"]+).mp3"', + webpage, 'preview url') + song_url = preview_url.replace('/previews/', '/c/m4a/64/') + '.m4a' + final_song_url = self._get_url(track_id, song_url) if final_song_url is None: raise ExtractorError('Unable to extract track url') @@ -117,3 +121,30 @@ class MixcloudIE(InfoExtractor): 'view_count': view_count, 'like_count': like_count, } + +def server_numbers(first, boundaries): + """ Server numbers to try in descending order of probable availability. + Starting from first (i.e. the number of the server hosting the preview file) + and going further and further up to the higher boundary and down to the + lower one in an alternating fashion. Namely: + + server_numbers(2, (1, 5)) + + # Where the preview server is 2, min number is 1 and max is 5. + # Yields: 2, 3, 1, 4, 5 + + Why not random numbers or increasing sequences? Since from what I've seen, + full length files seem to be hosted on servers whose number is closer to + that of the preview; to be confirmed. + """ + + if len(boundaries) != 2: + raise ValueError("boundaries should be a two-element tuple") + min, max = boundaries + highs = range(first + 1, max + 1) + lows = range(first - 1, min - 1, -1) + rest = filter(None, + itertools.chain.from_iterable(itertools.izip_longest(highs, lows))) + yield first + for n in rest: + yield n