From 72cb991f90bf4283f59e111d8140b478b6f8fabe Mon Sep 17 00:00:00 2001 From: kidburglar Date: Sun, 15 Jan 2017 17:00:29 +0100 Subject: [PATCH 1/6] Fixes #11572 Use cfscrape to avoid Crunchyroll error 503 --- youtube_dl/extractor/crunchyroll.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py index 559044352..f8bab638f 100644 --- a/youtube_dl/extractor/crunchyroll.py +++ b/youtube_dl/extractor/crunchyroll.py @@ -5,6 +5,7 @@ import re import json import base64 import zlib +import cfscrape from hashlib import sha1 from math import pow, sqrt, floor @@ -43,6 +44,12 @@ class CrunchyrollBaseIE(InfoExtractor): if username is None: return + # Scrape cookie from cloudfront and insert them + scraper = cfscrape.create_scraper() + tokens = scraper.get_tokens(self._LOGIN_URL, 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/47.0 (Chrome)') + self._set_crunchyroll_cookie('cf_clearance', tokens[0]['cf_clearance']) + self._set_crunchyroll_cookie('__cfduid', tokens[0]['__cfduid']) + login_page = self._download_webpage( self._LOGIN_URL, None, 'Downloading login page') @@ -103,6 +110,9 @@ class CrunchyrollBaseIE(InfoExtractor): request.add_header('Accept-Language', '*') return super(CrunchyrollBaseIE, self)._download_webpage(request, *args, **kwargs) + def _set_crunchyroll_cookie(self, name, value): + self._set_cookie('crunchyroll.com', name, value) + @staticmethod def _add_skip_wall(url): parsed_url = compat_urlparse.urlparse(url) From 076e0481f30322b67ba559c700b5713a08939c1b Mon Sep 17 00:00:00 2001 From: kidburglar Date: Sun, 15 Jan 2017 18:24:46 +0100 Subject: [PATCH 2/6] Fixes #11572 Not hardcode user-agent for cfscrape --- youtube_dl/extractor/crunchyroll.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py index f8bab638f..9d191de9e 100644 --- a/youtube_dl/extractor/crunchyroll.py +++ b/youtube_dl/extractor/crunchyroll.py @@ -24,6 +24,7 @@ from ..utils import ( lowercase_escape, remove_end, sanitized_Request, + std_headers, unified_strdate, urlencode_postdata, xpath_text, @@ -46,7 +47,7 @@ class CrunchyrollBaseIE(InfoExtractor): # Scrape cookie from cloudfront and insert them scraper = cfscrape.create_scraper() - tokens = scraper.get_tokens(self._LOGIN_URL, 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/47.0 (Chrome)') + tokens = scraper.get_tokens(self._LOGIN_URL, std_headers['User-Agent']) self._set_crunchyroll_cookie('cf_clearance', tokens[0]['cf_clearance']) self._set_crunchyroll_cookie('__cfduid', tokens[0]['__cfduid']) From b02249521b23c4a0fa5f5214e411c0e3a1270a09 Mon Sep 17 00:00:00 2001 From: kidburglar Date: Sun, 15 Jan 2017 22:32:19 +0100 Subject: [PATCH 3/6] Fixes #11572 Handle ImportError and display a message if you want use login function for CrunchyRoll --- youtube_dl/extractor/crunchyroll.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py index 9d191de9e..eb1e14fec 100644 --- a/youtube_dl/extractor/crunchyroll.py +++ b/youtube_dl/extractor/crunchyroll.py @@ -5,7 +5,11 @@ import re import json import base64 import zlib -import cfscrape +try: + import cfscrape + install_cfscrape_flag = True +except ImportError: + install_cfscrape_flag = False from hashlib import sha1 from math import pow, sqrt, floor @@ -45,6 +49,10 @@ class CrunchyrollBaseIE(InfoExtractor): if username is None: return + if install_cfscrape_flag == False: + print 'cfscrape not found. Please install it if you want use login function for CrunchyRoll.' + return False + # Scrape cookie from cloudfront and insert them scraper = cfscrape.create_scraper() tokens = scraper.get_tokens(self._LOGIN_URL, std_headers['User-Agent']) From 7ef4388f15e0cbc36a6337e10d618025cd973f77 Mon Sep 17 00:00:00 2001 From: kidburglar Date: Sun, 15 Jan 2017 22:54:30 +0100 Subject: [PATCH 4/6] Fixes #11572 Correct mistake in a comment --- youtube_dl/extractor/crunchyroll.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py index eb1e14fec..3c3474167 100644 --- a/youtube_dl/extractor/crunchyroll.py +++ b/youtube_dl/extractor/crunchyroll.py @@ -53,7 +53,7 @@ class CrunchyrollBaseIE(InfoExtractor): print 'cfscrape not found. Please install it if you want use login function for CrunchyRoll.' return False - # Scrape cookie from cloudfront and insert them + # Scrape cookie from cloudflare and insert them scraper = cfscrape.create_scraper() tokens = scraper.get_tokens(self._LOGIN_URL, std_headers['User-Agent']) self._set_crunchyroll_cookie('cf_clearance', tokens[0]['cf_clearance']) From 8fc2e7b461077a32e4833794560e7de874daa934 Mon Sep 17 00:00:00 2001 From: kidburglar Date: Sat, 21 Jan 2017 11:00:25 +0100 Subject: [PATCH 5/6] Fixes #11572 Change the flow of the verification is cfscrape is available --- youtube_dl/extractor/crunchyroll.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py index 3c3474167..cd02822ac 100644 --- a/youtube_dl/extractor/crunchyroll.py +++ b/youtube_dl/extractor/crunchyroll.py @@ -7,9 +7,9 @@ import base64 import zlib try: import cfscrape - install_cfscrape_flag = True + cfscrape_available = True except ImportError: - install_cfscrape_flag = False + cfscrape_available = False from hashlib import sha1 from math import pow, sqrt, floor @@ -45,20 +45,19 @@ class CrunchyrollBaseIE(InfoExtractor): _NETRC_MACHINE = 'crunchyroll' def _login(self): + if cfscrape_available: + # Scrape cookie from cloudflare and insert them + scraper = cfscrape.create_scraper() + tokens = scraper.get_tokens(self._LOGIN_URL, std_headers['User-Agent']) + self._set_crunchyroll_cookie('cf_clearance', tokens[0]['cf_clearance']) + self._set_crunchyroll_cookie('__cfduid', tokens[0]['__cfduid']) + else + print 'cfscrape not found. Please install it if you want use login function for CrunchyRoll.' + (username, password) = self._get_login_info() if username is None: return - if install_cfscrape_flag == False: - print 'cfscrape not found. Please install it if you want use login function for CrunchyRoll.' - return False - - # Scrape cookie from cloudflare and insert them - scraper = cfscrape.create_scraper() - tokens = scraper.get_tokens(self._LOGIN_URL, std_headers['User-Agent']) - self._set_crunchyroll_cookie('cf_clearance', tokens[0]['cf_clearance']) - self._set_crunchyroll_cookie('__cfduid', tokens[0]['__cfduid']) - login_page = self._download_webpage( self._LOGIN_URL, None, 'Downloading login page') From 10811770155643402090d4f7f85d51581e4c8342 Mon Sep 17 00:00:00 2001 From: kidburglar Date: Thu, 26 Jan 2017 18:24:23 +0100 Subject: [PATCH 6/6] Fixes #11572 Use self.report_waring to display warning about cfscrape if it's not found --- youtube_dl/extractor/crunchyroll.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py index cd02822ac..a4a1be19b 100644 --- a/youtube_dl/extractor/crunchyroll.py +++ b/youtube_dl/extractor/crunchyroll.py @@ -51,8 +51,8 @@ class CrunchyrollBaseIE(InfoExtractor): tokens = scraper.get_tokens(self._LOGIN_URL, std_headers['User-Agent']) self._set_crunchyroll_cookie('cf_clearance', tokens[0]['cf_clearance']) self._set_crunchyroll_cookie('__cfduid', tokens[0]['__cfduid']) - else - print 'cfscrape not found. Please install it if you want use login function for CrunchyRoll.' + else: + self.report_warning('cfscrape not found. Please install it if you want use login function for CrunchyRoll.') (username, password) = self._get_login_info() if username is None: