From 3f8ec99719cc91c9ec4fe3a6b32e4930e2c27c23 Mon Sep 17 00:00:00 2001 From: Avi Peretz Date: Sat, 22 Jun 2019 22:54:09 +0300 Subject: [PATCH 01/12] extrace page likes. --- youtube_dl/extractor/facebook.py | 50 ++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py index 1b91c9036..14cb54966 100644 --- a/youtube_dl/extractor/facebook.py +++ b/youtube_dl/extractor/facebook.py @@ -24,6 +24,7 @@ from ..utils import ( sanitized_Request, try_get, urlencode_postdata, + update_url_query ) @@ -481,6 +482,8 @@ class FacebookIE(InfoExtractor): 'like_count': likes_count, 'share_count': shares_count } + if uploader_id: + info_dict['uploader_like_count'] = FacebookAjax(self, webpage, uploader_id).page_likes return webpage, info_dict @@ -587,6 +590,53 @@ class FacebookTahoeData: return tahoe_request_data, tahoe_request_headers +class FacebookAjax: + HOVER_URL_TEMPLATE = 'https://www.facebook.com/ajax/hovercard/user.php?id=111&fb_dtsg_ag=x&endpoint=%2Fajax%2Fhovercard%2Fuser.php%3Fid%3D111&__a=1' + + def __init__(self, extractor, page, page_id): + self._page = page + self._page_id = page_id + self._extractor = extractor + self._hover_data = None + + def _get_hover_data(self): + if self._hover_data: + data = self._hover_data + else: + data = self._extractor._download_webpage( + self._get_request_url(self._page_id), self._page_id + ) + return '' if not data else data + + @property + def hover(self): + return self._get_hover_data() + + @property + def page_likes(self): + return parse_count( + self._extractor._search_regex(r'\/span>([\d,]+) likes', self.hover, 'uploader_likes', default=None) + ) + + def _get_request_url(self, page_id): + return update_url_query(self.HOVER_URL_TEMPLATE, + { + + 'id': page_id, + 'endpoint': '/ajax/hovercard/user.php?id=%s' % page_id, + '__a': 1, + '__pc': self._extractor._search_regex( + r'pkg_cohort["\']\s*:\s*["\'](.+?)["\']', self._page, + 'pkg cohort', default='PHASED:DEFAULT'), + '__rev': self._extractor._search_regex( + r'client_revision["\']\s*:\s*(\d+),', self._page, + 'client revision', default='3944515'), + 'fb_dtsg': self._extractor._search_regex( + r'"DTSGInitialData"\s*,\s*\[\]\s*,\s*{\s*"token"\s*:\s*"([^"]+)"', + self._page, 'dtsg token', default=''), + }) + + class FacebookPluginsVideoIE(InfoExtractor): _VALID_URL = r'https?://(?:[\w-]+\.)?facebook\.com/plugins/video\.php\?.*?\bhref=(?Phttps.+)' From c8a0a6d791501c7d15534405e7a70c6e3110a792 Mon Sep 17 00:00:00 2001 From: Avi Peretz Date: Sat, 22 Jun 2019 22:59:11 +0300 Subject: [PATCH 02/12] not fatal for now. --- youtube_dl/extractor/facebook.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py index 14cb54966..1021345bf 100644 --- a/youtube_dl/extractor/facebook.py +++ b/youtube_dl/extractor/facebook.py @@ -614,10 +614,13 @@ class FacebookAjax: @property def page_likes(self): - return parse_count( - self._extractor._search_regex(r'\/span>([\d,]+) likes', self.hover, 'uploader_likes', default=None) - ) - + try: + return parse_count( + self._extractor._search_regex(r'\/span>([\d,]+) likes', self.hover, 'uploader_likes', default=None) + ) + except: + return None + def _get_request_url(self, page_id): return update_url_query(self.HOVER_URL_TEMPLATE, { From 92e5646145dbaef5d6523dfc4d2a44cea9c4d57d Mon Sep 17 00:00:00 2001 From: Avi Peretz Date: Sun, 23 Jun 2019 09:57:17 +0300 Subject: [PATCH 03/12] print exception. --- youtube_dl/extractor/facebook.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py index 1021345bf..0c072a5e0 100644 --- a/youtube_dl/extractor/facebook.py +++ b/youtube_dl/extractor/facebook.py @@ -618,9 +618,9 @@ class FacebookAjax: return parse_count( self._extractor._search_regex(r'\/span>([\d,]+) likes', self.hover, 'uploader_likes', default=None) ) - except: - return None - + except Exception as e: + self._extractor.report_warning(self._page_id + str(e)) + def _get_request_url(self, page_id): return update_url_query(self.HOVER_URL_TEMPLATE, { From 0688086aea498a903d4ad93b98f90944ffbf3081 Mon Sep 17 00:00:00 2001 From: Avichai Cohen Date: Sun, 23 Jun 2019 12:13:48 +0300 Subject: [PATCH 04/12] upgrading version (to test new watcher) --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index a5def2a00..9a682d73c 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = 'vc.2019.06.19' +__version__ = 'vc.2019.06.20' From 75593b2c1b2b2ba23fbe7840bd0e7aed738198ef Mon Sep 17 00:00:00 2001 From: Avichai Cohen Date: Sun, 23 Jun 2019 14:04:52 +0300 Subject: [PATCH 05/12] changing version again to test the watcher --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 9a682d73c..a5def2a00 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = 'vc.2019.06.20' +__version__ = 'vc.2019.06.19' From 7b284e4b000c060a211b5329619f98affd7b83a9 Mon Sep 17 00:00:00 2001 From: Avichai Cohen Date: Sun, 23 Jun 2019 14:11:31 +0300 Subject: [PATCH 06/12] again --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index a5def2a00..9a682d73c 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = 'vc.2019.06.19' +__version__ = 'vc.2019.06.20' From b90bcd03fd4c9a7c1c178c95c1f7854bb09a2b5c Mon Sep 17 00:00:00 2001 From: Avichai Cohen Date: Sun, 23 Jun 2019 14:37:03 +0300 Subject: [PATCH 07/12] again --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 9a682d73c..f38d395fc 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = 'vc.2019.06.20' +__version__ = 'vc.2019.06.21' From a369e1aaa7a2096c82480b32dd99a398874b8b76 Mon Sep 17 00:00:00 2001 From: Avichai Cohen Date: Sun, 23 Jun 2019 16:12:28 +0300 Subject: [PATCH 08/12] upgrading again in order to test the watcher --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index f38d395fc..987778cd6 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = 'vc.2019.06.21' +__version__ = 'vc.2019.06.22' From 228de634855fa8d4929b80ed681c0747983746cd Mon Sep 17 00:00:00 2001 From: Avichai Cohen Date: Sun, 23 Jun 2019 16:17:30 +0300 Subject: [PATCH 09/12] again --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 987778cd6..ba1ccdf53 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = 'vc.2019.06.22' +__version__ = 'vc.2019.06.23' From 625a0b383af0c91cf28219d4c8774143ba135f9f Mon Sep 17 00:00:00 2001 From: Avichai Cohen Date: Sun, 23 Jun 2019 16:18:58 +0300 Subject: [PATCH 10/12] one more time --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index ba1ccdf53..11f895645 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = 'vc.2019.06.23' +__version__ = 'vc.2019.06.231' From ca1cf00cacd2aa2b53ad165b613cc7264ea1127d Mon Sep 17 00:00:00 2001 From: Avichai Cohen Date: Sun, 23 Jun 2019 16:23:50 +0300 Subject: [PATCH 11/12] again --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 11f895645..ba1ccdf53 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = 'vc.2019.06.231' +__version__ = 'vc.2019.06.23' From bb2ddf68d5131702d1cc63bcea42eceb80e3614e Mon Sep 17 00:00:00 2001 From: Avichai Cohen Date: Sun, 23 Jun 2019 16:28:51 +0300 Subject: [PATCH 12/12] again :-( --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index ba1ccdf53..d05d19391 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = 'vc.2019.06.23' +__version__ = 'vc.2019.06.232'