From 444d749924c0df4db50dfaf4d9a0237a53b74719 Mon Sep 17 00:00:00 2001 From: TRox1972 Date: Thu, 19 May 2016 02:20:11 +0200 Subject: [PATCH 1/8] [Flipagram] add new extractor saw there was another pull request for an extractor for the website, but it does not extract all the necessary metadata, and has been inactive for a period of time --- youtube_dl/extractor/common.py | 7 ++ youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/flipagram.py | 106 +++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+) create mode 100644 youtube_dl/extractor/flipagram.py diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 8a8c07226..4f82ed64d 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -833,6 +833,13 @@ class InfoExtractor(object): 'title': unescapeHTML(json_ld.get('headline')), 'description': unescapeHTML(json_ld.get('articleBody')), }) + elif item_type == 'VideoObject': + info.update({ + 'title': unescapeHTML(json_ld.get('name')), + 'description': unescapeHTML(json_ld.get('description')), + 'upload_date': unified_strdate(json_ld.get('upload_date')), + 'url': unescapeHTML(json_ld.get('contentUrl')), + }) return dict((k, v) for k, v in info.items() if v is not None) @staticmethod diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index efbe970fe..97eb1ba0e 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -240,6 +240,7 @@ from .fivemin import FiveMinIE from .fivetv import FiveTVIE from .fktv import FKTVIE from .flickr import FlickrIE +from .flipagram import FlipagramIE from .folketinget import FolketingetIE from .footyroom import FootyRoomIE from .formula1 import Formula1IE diff --git a/youtube_dl/extractor/flipagram.py b/youtube_dl/extractor/flipagram.py new file mode 100644 index 000000000..44e92f971 --- /dev/null +++ b/youtube_dl/extractor/flipagram.py @@ -0,0 +1,106 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import time +from .common import InfoExtractor + +from ..utils import ( + parse_iso8601, + unified_strdate, + int_or_none, +) + + +class FlipagramIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?flipagram\.com/f/(?P[^/?_]+)' + _TESTS = [{ + 'url': 'https://flipagram.com/f/myrWjW9RJw', + 'md5': '541988fb6c4c7c375215ea22a4a21841', + 'info_dict': { + 'id': 'myrWjW9RJw', + 'title': 'Flipagram by crystaldolce featuring King and Lionheart by Of Monsters and Men', + 'description': 'Herbie\'s first bannana🍌🐢🍌. #animals #pets #reptile #tortoise #sulcata #tort #justatreat #snacktime #bannanas #rescuepets #ofmonstersandmen @animals', + 'ext': 'mp4', + 'uploader': 'Crystal Dolce', + 'creator': 'Crystal Dolce', + 'uploader_id': 'crystaldolce', + } + }, { + 'url': 'https://flipagram.com/f/nyvTSJMKId', + 'only_matching': True, + }] + + def c_date_to_iso(self, c_date): + 'Convert dates in format \'04/25/2016 00:23:24 UTC\' to ISO8601.' + return time.strftime('%Y-%m-%dT%H:%M:%S', time.strptime(c_date, '%m/%d/%Y %H:%M:%S %Z')) + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + self.report_extraction(video_id) + user_data = self._parse_json(self._search_regex(r'window.reactH2O\s*=\s*({.+});', webpage, 'user data'), video_id) + content_data = self._search_json_ld(webpage, video_id) + + flipagram = user_data.get('flipagram', {}) + counts = flipagram.get('counts', {}) + user = flipagram.get('user', {}) + video = flipagram.get('video', {}) + + thumbnails = [] + for cover in flipagram.get('covers', []): + if not cover.get('url'): + continue + thumbnails.append({ + 'url': self._proto_relative_url(cover.get('url')), + 'width': int_or_none(cover.get('width')), + 'height': int_or_none(cover.get('height')), + }) + + comments = [] + for comment in user_data.get('comments', {}).get(video_id, {}).get('items', []): + text = comment.get('comment', []) + comments.append({ + 'author': comment.get('user', {}).get('name'), + 'author_id': comment.get('user', {}).get('username'), + 'id': comment.get('id'), + 'text': text[0] if text else '', + 'timestamp': parse_iso8601(self.c_date_to_iso(comment.get('created', ''))), + }) + + tags = [tag for item in flipagram['story'][1:] for tag in item] + + formats = [] + if flipagram.get('music', {}).get('track', {}).get('previewUrl', {}): + formats.append({ + 'url': flipagram.get('music').get('track').get('previewUrl'), + 'ext': 'm4a', + 'vcodec': 'none', + }) + + formats.append({ + 'url': video.get('url'), + 'ext': 'mp4', + 'width': int_or_none(video.get('width')), + 'height': int_or_none(video.get('height')), + 'filesize': int_or_none(video.get('size')), + }) + + return { + 'id': video_id, + 'title': content_data.get('title'), + 'formats': formats, + 'ext': 'mp4', + 'thumbnails': thumbnails, + 'description': content_data.get('description'), + 'uploader': user.get('name'), + 'creator': user.get('name'), + 'timestamp': parse_iso8601(flipagram.get('iso801Created')), + 'upload_date': unified_strdate(flipagram.get('created')), + 'uploader_id': user.get('username'), + 'view_count': int_or_none(counts.get('plays')), + 'repost_count': int_or_none(counts.get('reflips')), + 'comment_count': int_or_none(counts.get('comments')), + 'comments': comments, + 'tags': tags, + } From 64d851b398e43420e17628afd915db4301ec3270 Mon Sep 17 00:00:00 2001 From: TRox1972 Date: Wed, 25 May 2016 15:24:57 +0200 Subject: [PATCH 2/8] [Flipagram] add comments about comment retrieval --- youtube_dl/extractor/flipagram.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/youtube_dl/extractor/flipagram.py b/youtube_dl/extractor/flipagram.py index 44e92f971..43ba83929 100644 --- a/youtube_dl/extractor/flipagram.py +++ b/youtube_dl/extractor/flipagram.py @@ -57,6 +57,8 @@ class FlipagramIE(InfoExtractor): 'height': int_or_none(cover.get('height')), }) + # Note that this only retrieves comments that are initally loaded. + # For videos with large amounts of comments, most won't be retrieved. comments = [] for comment in user_data.get('comments', {}).get(video_id, {}).get('items', []): text = comment.get('comment', []) From 0e073ce26a3c540891279356bea0577107652c54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9stin=20Reed?= Date: Sat, 25 Jun 2016 18:05:49 +0200 Subject: [PATCH 3/8] [Flipagram] Simplify date format conversion for 'timestamp' --- youtube_dl/extractor/flipagram.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/flipagram.py b/youtube_dl/extractor/flipagram.py index 43ba83929..4866237d4 100644 --- a/youtube_dl/extractor/flipagram.py +++ b/youtube_dl/extractor/flipagram.py @@ -1,7 +1,8 @@ # coding: utf-8 from __future__ import unicode_literals -import time +import calendar +import datetime from .common import InfoExtractor from ..utils import ( @@ -30,9 +31,8 @@ class FlipagramIE(InfoExtractor): 'only_matching': True, }] - def c_date_to_iso(self, c_date): - 'Convert dates in format \'04/25/2016 00:23:24 UTC\' to ISO8601.' - return time.strftime('%Y-%m-%dT%H:%M:%S', time.strptime(c_date, '%m/%d/%Y %H:%M:%S %Z')) + def c_date_to_iso(self, date): + return calendar.timegm(datetime.datetime.strptime(date, '%m/%d/%Y %H:%M:%S %Z').utctimetuple()) def _real_extract(self, url): video_id = self._match_id(url) @@ -67,14 +67,14 @@ class FlipagramIE(InfoExtractor): 'author_id': comment.get('user', {}).get('username'), 'id': comment.get('id'), 'text': text[0] if text else '', - 'timestamp': parse_iso8601(self.c_date_to_iso(comment.get('created', ''))), + 'timestamp': self.c_date_to_iso(comment.get('created', '')), }) tags = [tag for item in flipagram['story'][1:] for tag in item] formats = [] if flipagram.get('music', {}).get('track', {}).get('previewUrl', {}): - formats.append({ + formats.append({ 'url': flipagram.get('music').get('track').get('previewUrl'), 'ext': 'm4a', 'vcodec': 'none', From d023fd66b23c2097abb703af44af217f83b1ed1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9stin=20Reed?= Date: Sat, 25 Jun 2016 18:21:28 +0200 Subject: [PATCH 4/8] [Flipagram] Remove 'ext' field as its already defined in 'formats' --- youtube_dl/extractor/flipagram.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/extractor/flipagram.py b/youtube_dl/extractor/flipagram.py index 4866237d4..527f1ddaa 100644 --- a/youtube_dl/extractor/flipagram.py +++ b/youtube_dl/extractor/flipagram.py @@ -92,7 +92,6 @@ class FlipagramIE(InfoExtractor): 'id': video_id, 'title': content_data.get('title'), 'formats': formats, - 'ext': 'mp4', 'thumbnails': thumbnails, 'description': content_data.get('description'), 'uploader': user.get('name'), From 5d3106356f9f4e63a9501a600899c2b6509aeef1 Mon Sep 17 00:00:00 2001 From: TRox1972 Date: Thu, 19 May 2016 02:20:11 +0200 Subject: [PATCH 5/8] [Flipagram] add new extractor saw there was another pull request for an extractor for the website, but it does not extract all the necessary metadata, and has been inactive for a period of time --- youtube_dl/extractor/common.py | 7 ++ youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/flipagram.py | 106 +++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+) create mode 100644 youtube_dl/extractor/flipagram.py diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 5a2603b50..20e9976ea 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -835,6 +835,13 @@ class InfoExtractor(object): 'title': unescapeHTML(json_ld.get('headline')), 'description': unescapeHTML(json_ld.get('articleBody')), }) + elif item_type == 'VideoObject': + info.update({ + 'title': unescapeHTML(json_ld.get('name')), + 'description': unescapeHTML(json_ld.get('description')), + 'upload_date': unified_strdate(json_ld.get('upload_date')), + 'url': unescapeHTML(json_ld.get('contentUrl')), + }) return dict((k, v) for k, v in info.items() if v is not None) @staticmethod diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 9f98a1490..10bc16998 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -251,6 +251,7 @@ from .fivemin import FiveMinIE from .fivetv import FiveTVIE from .fktv import FKTVIE from .flickr import FlickrIE +from .flipagram import FlipagramIE from .folketinget import FolketingetIE from .footyroom import FootyRoomIE from .formula1 import Formula1IE diff --git a/youtube_dl/extractor/flipagram.py b/youtube_dl/extractor/flipagram.py new file mode 100644 index 000000000..44e92f971 --- /dev/null +++ b/youtube_dl/extractor/flipagram.py @@ -0,0 +1,106 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import time +from .common import InfoExtractor + +from ..utils import ( + parse_iso8601, + unified_strdate, + int_or_none, +) + + +class FlipagramIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?flipagram\.com/f/(?P[^/?_]+)' + _TESTS = [{ + 'url': 'https://flipagram.com/f/myrWjW9RJw', + 'md5': '541988fb6c4c7c375215ea22a4a21841', + 'info_dict': { + 'id': 'myrWjW9RJw', + 'title': 'Flipagram by crystaldolce featuring King and Lionheart by Of Monsters and Men', + 'description': 'Herbie\'s first bannana🍌🐢🍌. #animals #pets #reptile #tortoise #sulcata #tort #justatreat #snacktime #bannanas #rescuepets #ofmonstersandmen @animals', + 'ext': 'mp4', + 'uploader': 'Crystal Dolce', + 'creator': 'Crystal Dolce', + 'uploader_id': 'crystaldolce', + } + }, { + 'url': 'https://flipagram.com/f/nyvTSJMKId', + 'only_matching': True, + }] + + def c_date_to_iso(self, c_date): + 'Convert dates in format \'04/25/2016 00:23:24 UTC\' to ISO8601.' + return time.strftime('%Y-%m-%dT%H:%M:%S', time.strptime(c_date, '%m/%d/%Y %H:%M:%S %Z')) + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + self.report_extraction(video_id) + user_data = self._parse_json(self._search_regex(r'window.reactH2O\s*=\s*({.+});', webpage, 'user data'), video_id) + content_data = self._search_json_ld(webpage, video_id) + + flipagram = user_data.get('flipagram', {}) + counts = flipagram.get('counts', {}) + user = flipagram.get('user', {}) + video = flipagram.get('video', {}) + + thumbnails = [] + for cover in flipagram.get('covers', []): + if not cover.get('url'): + continue + thumbnails.append({ + 'url': self._proto_relative_url(cover.get('url')), + 'width': int_or_none(cover.get('width')), + 'height': int_or_none(cover.get('height')), + }) + + comments = [] + for comment in user_data.get('comments', {}).get(video_id, {}).get('items', []): + text = comment.get('comment', []) + comments.append({ + 'author': comment.get('user', {}).get('name'), + 'author_id': comment.get('user', {}).get('username'), + 'id': comment.get('id'), + 'text': text[0] if text else '', + 'timestamp': parse_iso8601(self.c_date_to_iso(comment.get('created', ''))), + }) + + tags = [tag for item in flipagram['story'][1:] for tag in item] + + formats = [] + if flipagram.get('music', {}).get('track', {}).get('previewUrl', {}): + formats.append({ + 'url': flipagram.get('music').get('track').get('previewUrl'), + 'ext': 'm4a', + 'vcodec': 'none', + }) + + formats.append({ + 'url': video.get('url'), + 'ext': 'mp4', + 'width': int_or_none(video.get('width')), + 'height': int_or_none(video.get('height')), + 'filesize': int_or_none(video.get('size')), + }) + + return { + 'id': video_id, + 'title': content_data.get('title'), + 'formats': formats, + 'ext': 'mp4', + 'thumbnails': thumbnails, + 'description': content_data.get('description'), + 'uploader': user.get('name'), + 'creator': user.get('name'), + 'timestamp': parse_iso8601(flipagram.get('iso801Created')), + 'upload_date': unified_strdate(flipagram.get('created')), + 'uploader_id': user.get('username'), + 'view_count': int_or_none(counts.get('plays')), + 'repost_count': int_or_none(counts.get('reflips')), + 'comment_count': int_or_none(counts.get('comments')), + 'comments': comments, + 'tags': tags, + } From 7e1af5dc99fd2c6f5f31b1a642bfe2faf26f2f62 Mon Sep 17 00:00:00 2001 From: TRox1972 Date: Wed, 25 May 2016 15:24:57 +0200 Subject: [PATCH 6/8] [Flipagram] add comments about comment retrieval --- youtube_dl/extractor/flipagram.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/youtube_dl/extractor/flipagram.py b/youtube_dl/extractor/flipagram.py index 44e92f971..43ba83929 100644 --- a/youtube_dl/extractor/flipagram.py +++ b/youtube_dl/extractor/flipagram.py @@ -57,6 +57,8 @@ class FlipagramIE(InfoExtractor): 'height': int_or_none(cover.get('height')), }) + # Note that this only retrieves comments that are initally loaded. + # For videos with large amounts of comments, most won't be retrieved. comments = [] for comment in user_data.get('comments', {}).get(video_id, {}).get('items', []): text = comment.get('comment', []) From a2105ed5d0d1353e8bb0b480214ae22f7c8b72d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9stin=20Reed?= Date: Sat, 25 Jun 2016 18:05:49 +0200 Subject: [PATCH 7/8] [Flipagram] Simplify date format conversion for 'timestamp' --- youtube_dl/extractor/flipagram.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/flipagram.py b/youtube_dl/extractor/flipagram.py index 43ba83929..4866237d4 100644 --- a/youtube_dl/extractor/flipagram.py +++ b/youtube_dl/extractor/flipagram.py @@ -1,7 +1,8 @@ # coding: utf-8 from __future__ import unicode_literals -import time +import calendar +import datetime from .common import InfoExtractor from ..utils import ( @@ -30,9 +31,8 @@ class FlipagramIE(InfoExtractor): 'only_matching': True, }] - def c_date_to_iso(self, c_date): - 'Convert dates in format \'04/25/2016 00:23:24 UTC\' to ISO8601.' - return time.strftime('%Y-%m-%dT%H:%M:%S', time.strptime(c_date, '%m/%d/%Y %H:%M:%S %Z')) + def c_date_to_iso(self, date): + return calendar.timegm(datetime.datetime.strptime(date, '%m/%d/%Y %H:%M:%S %Z').utctimetuple()) def _real_extract(self, url): video_id = self._match_id(url) @@ -67,14 +67,14 @@ class FlipagramIE(InfoExtractor): 'author_id': comment.get('user', {}).get('username'), 'id': comment.get('id'), 'text': text[0] if text else '', - 'timestamp': parse_iso8601(self.c_date_to_iso(comment.get('created', ''))), + 'timestamp': self.c_date_to_iso(comment.get('created', '')), }) tags = [tag for item in flipagram['story'][1:] for tag in item] formats = [] if flipagram.get('music', {}).get('track', {}).get('previewUrl', {}): - formats.append({ + formats.append({ 'url': flipagram.get('music').get('track').get('previewUrl'), 'ext': 'm4a', 'vcodec': 'none', From b77bd403eeea98f0fd3ca1141c5c2cac514e89a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A9stin=20Reed?= Date: Sat, 25 Jun 2016 18:21:28 +0200 Subject: [PATCH 8/8] [Flipagram] Remove 'ext' field as its already defined in 'formats' --- youtube_dl/extractor/flipagram.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/extractor/flipagram.py b/youtube_dl/extractor/flipagram.py index 4866237d4..527f1ddaa 100644 --- a/youtube_dl/extractor/flipagram.py +++ b/youtube_dl/extractor/flipagram.py @@ -92,7 +92,6 @@ class FlipagramIE(InfoExtractor): 'id': video_id, 'title': content_data.get('title'), 'formats': formats, - 'ext': 'mp4', 'thumbnails': thumbnails, 'description': content_data.get('description'), 'uploader': user.get('name'),