From 8cddcb03222eeb6ca4bb1e8a441dcfb7c3449d5a Mon Sep 17 00:00:00 2001 From: hodayabu Date: Mon, 20 Apr 2020 21:07:13 +0300 Subject: [PATCH 1/6] video info platform tiktok --- test/ci/test_tiktok.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/ci/test_tiktok.py b/test/ci/test_tiktok.py index 7f0294c2a..2fe6cc4b9 100644 --- a/test/ci/test_tiktok.py +++ b/test/ci/test_tiktok.py @@ -8,7 +8,8 @@ class TikTokTestYoutubeDl(unittest.TestCase): params = {} ydl = youtube_dl.YoutubeDL(params) info = ydl.extract_info(url, download=False) - self.assertEquals(info['share_count'], 110) + self.assertEquals(info['share_count'], 121) + From 17880cbaca971d631304ed6cfbe7c1365e8b4110 Mon Sep 17 00:00:00 2001 From: hodayabu Date: Tue, 21 Apr 2020 14:32:32 +0300 Subject: [PATCH 2/6] tiktok fix code --- test/ci/test_tiktok.py | 21 +++++++++--- youtube_dl/extractor/tiktok.py | 62 ++++++++++------------------------ 2 files changed, 35 insertions(+), 48 deletions(-) diff --git a/test/ci/test_tiktok.py b/test/ci/test_tiktok.py index 2fe6cc4b9..5e050e6e2 100644 --- a/test/ci/test_tiktok.py +++ b/test/ci/test_tiktok.py @@ -4,13 +4,26 @@ import youtube_dl class TikTokTestYoutubeDl(unittest.TestCase): def test_meta_data(self): - url = 'https://www.tiktok.com/@danieltbraun/video/6817099671043853574' + url = 'https://www.tiktok.com/@oriangaon/video/6807126376001441030' params = {} ydl = youtube_dl.YoutubeDL(params) info = ydl.extract_info(url, download=False) - self.assertEquals(info['share_count'], 121) - - + self.assertEquals(info['id'], '6807126376001441030') + self.assertEquals(info['url'], 'https://www.tiktok.com/@oriangaon/video/6807126376001441030') + self.assertEquals(info['title'], '#foryou #foyou Mmmmm....,,') + self.assertEquals(info['uploader'], 'Oriangaon') + self.assertEquals(info['timestamp'], 1584907616) + self.assertEquals(info['thumbnail'], + 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/d1a8fbd3e42dda3a1baa01ee9edad289') + self.assertGreaterEqual(info['view_count'], 79864) + self.assertEquals(info['uploader_id'], '6772113344733955077') + self.assertFalse(info['is_live']) + self.assertEquals(info['live_status'], 'not_live') + self.assertGreaterEqual(info['like_count'], 2213) + self.assertGreaterEqual(info['share_count'], 109) + self.assertGreaterEqual(info['comment_count'], 40) + self.assertEquals(info['duration'], 10) + self.assertEquals(info['ext'], 'mp.4') if __name__ == '__main__': diff --git a/youtube_dl/extractor/tiktok.py b/youtube_dl/extractor/tiktok.py index 140db07bc..9d742465e 100644 --- a/youtube_dl/extractor/tiktok.py +++ b/youtube_dl/extractor/tiktok.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals import ast from bs4 import BeautifulSoup -import requests import json from .common import InfoExtractor from ..utils import ( @@ -11,11 +10,7 @@ from ..utils import ( int_or_none, str_or_none, try_get, - url_or_none, -) - - -# add to requirements.txt- bs4, newspaper, requests + url_or_none) class TikTokBaseIE(InfoExtractor): @@ -70,8 +65,6 @@ class TikTokBaseIE(InfoExtractor): } - - class TikTokIE(TikTokBaseIE): _VALID_URL = r'''(?x) https?:// @@ -103,55 +96,34 @@ class TikTokIE(TikTokBaseIE): }] def _real_extract(self, url): - video_id = url.split('/')[-1] - - # extract meta data using the official api - # Response json contains: provider url, title, html, author_namee, height, thumbnail_width, width, version, - # author_url, thumbnail_height, thumbnail_url, type, provider_name (tiktok) - + video_id = self._match_id(url) json_api = self._download_json('https://www.tiktok.com/oembed?url=' + url, video_id) - # extract metadata with beautifulSoup webpage = self._download_webpage(url, video_id) soup = BeautifulSoup(webpage, features="html.parser") - h2 = soup.find_all("h2", {"class": "jsx-1038045583 jsx-3192540912 jsx-2150087249 video-meta-count"}) - data = h2[0].text.split(' ') - likes_count = self.numeric_convert(data[0]) - comments_count = self.numeric_convert(data[3]) json_next_data = soup.find(id='__NEXT_DATA__') props = json_next_data.contents[0] json_data_encode = json.dumps(props.encode('utf-8')) ast_le = ast.literal_eval(json_data_encode) data_dict = json.loads(ast_le) - timestamp = self.numeric_convert(data_dict['props']['pageProps']['videoData']['itemInfos']['createTime']) - shares = data_dict['props']['pageProps']['videoData']['itemInfos']['shareCount'] - views = data_dict['props']['pageProps']['videoData']['itemInfos']['playCount'] - duration = data_dict['props']['pageProps']['videoData']['itemInfos']['video']['videoMeta']['duration'] - provider_id = data_dict['props']['pageProps']['videoData']['itemInfos']['authorId'] - - # TO-DO- check on formats + item_info = data_dict['props']['pageProps']['videoData']['itemInfos'] + timestamp = int(item_info['createTime']) + shares = item_info['shareCount'] + views = item_info['playCount'] + duration = item_info['video']['videoMeta']['duration'] + provider_id = item_info['authorId'] + comments_count = item_info['commentCount'] + likes_count = item_info['diggCount'] return self.info_dict(video_id, str(url), json_api['title'], json_api['author_name'], timestamp, json_api['thumbnail_url'], - views, provider_id, False, 'not_live', likes_count, shares, '', comments_count,duration) + views, provider_id, False, 'not_live', likes_count, shares, '', comments_count, duration) - def numeric_convert(self, unicode): - if 'K' in unicode: - unicode=unicode[:-1] - return int(float(unicode)*1000) - if 'M' in unicode: - unicode=unicode[:-1] - return int(float(unicode)*100000) - else: - return int(unicode) - - - - def info_dict (self, video_id, url, video_title, - uploader, timestamp, thumbnail, - view_count, uploader_id, is_live, live_status - , likes_count, shares_count, subtitles, comment_count, duration): + def info_dict(self, video_id, url, video_title, + uploader, timestamp, thumbnail, + view_count, uploader_id, is_live, live_status + , likes_count, shares_count, subtitles, comment_count, duration): info_dict = { 'id': video_id, 'url': url, @@ -167,11 +139,13 @@ class TikTokIE(TikTokBaseIE): 'share_count': shares_count, 'subtitles': subtitles, 'comment_count': comment_count, - 'duration': duration + 'duration': duration, + 'ext':'mp.4' } return info_dict + class TikTokUserIE(TikTokBaseIE): _VALID_URL = r'''(?x) https?:// From b9f1ba3c017ae7b235d7b1efd9be36f2802069af Mon Sep 17 00:00:00 2001 From: hodayabu Date: Tue, 21 Apr 2020 17:57:08 +0300 Subject: [PATCH 3/6] video info platform tiktok --- youtube_dl/extractor/tiktok.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/tiktok.py b/youtube_dl/extractor/tiktok.py index 9d742465e..4be1ef562 100644 --- a/youtube_dl/extractor/tiktok.py +++ b/youtube_dl/extractor/tiktok.py @@ -118,12 +118,12 @@ class TikTokIE(TikTokBaseIE): return self.info_dict(video_id, str(url), json_api['title'], json_api['author_name'], timestamp, json_api['thumbnail_url'], - views, provider_id, False, 'not_live', likes_count, shares, '', comments_count, duration) + views, provider_id, False, 'not_live', likes_count, shares, '', comments_count, duration, json_api['html']) def info_dict(self, video_id, url, video_title, uploader, timestamp, thumbnail, view_count, uploader_id, is_live, live_status - , likes_count, shares_count, subtitles, comment_count, duration): + , likes_count, shares_count, subtitles, comment_count, duration, embed_code): info_dict = { 'id': video_id, 'url': url, @@ -140,8 +140,8 @@ class TikTokIE(TikTokBaseIE): 'subtitles': subtitles, 'comment_count': comment_count, 'duration': duration, - 'ext':'mp.4' - + 'ext':'mp.4', + 'embed_code': embed_code } return info_dict From 06b483099c8d4b854f551fa16a1a3c2f1be98db8 Mon Sep 17 00:00:00 2001 From: bhodaya Date: Sun, 26 Apr 2020 09:19:15 +0300 Subject: [PATCH 4/6] add embed code --- test/ci/test_tiktok.py | 1 + youtube_dl/extractor/tiktok.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ci/test_tiktok.py b/test/ci/test_tiktok.py index 5e050e6e2..c0a93d61f 100644 --- a/test/ci/test_tiktok.py +++ b/test/ci/test_tiktok.py @@ -24,6 +24,7 @@ class TikTokTestYoutubeDl(unittest.TestCase): self.assertGreaterEqual(info['comment_count'], 40) self.assertEquals(info['duration'], 10) self.assertEquals(info['ext'], 'mp.4') + self.assertGreater(len(info['embed_code']),0) if __name__ == '__main__': diff --git a/youtube_dl/extractor/tiktok.py b/youtube_dl/extractor/tiktok.py index 4be1ef562..4a2d3b94f 100644 --- a/youtube_dl/extractor/tiktok.py +++ b/youtube_dl/extractor/tiktok.py @@ -1,4 +1,3 @@ -# coding: utf-8 from __future__ import unicode_literals import ast from bs4 import BeautifulSoup From 3165ef58a5eb4345c6e46aab210e601ce90f666e Mon Sep 17 00:00:00 2001 From: bhodaya Date: Sun, 3 May 2020 16:03:06 +0300 Subject: [PATCH 5/6] download video --- test/ci/test_tiktok.py | 11 ++++++ youtube_dl/extractor/tiktok.py | 71 ++++++++++------------------------ 2 files changed, 32 insertions(+), 50 deletions(-) diff --git a/test/ci/test_tiktok.py b/test/ci/test_tiktok.py index c0a93d61f..e92981283 100644 --- a/test/ci/test_tiktok.py +++ b/test/ci/test_tiktok.py @@ -1,4 +1,6 @@ import unittest +import os.path + import youtube_dl @@ -26,6 +28,15 @@ class TikTokTestYoutubeDl(unittest.TestCase): self.assertEquals(info['ext'], 'mp.4') self.assertGreater(len(info['embed_code']),0) + def test_download_video(self): + url = 'https://www.tiktok.com/@ballislife/video/6783617809113943301' + params = {} + ydl = youtube_dl.YoutubeDL(params) + info = ydl.extract_info(url, download=True) + self.assertTrue(os.path.exists("Imagine lebron freaking out over something you did! #foryou #ballislife #lebron #nba-6783617809113943301.mp.4")) + + + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/extractor/tiktok.py b/youtube_dl/extractor/tiktok.py index 4a2d3b94f..023b40ab6 100644 --- a/youtube_dl/extractor/tiktok.py +++ b/youtube_dl/extractor/tiktok.py @@ -4,63 +4,31 @@ from bs4 import BeautifulSoup import json from .common import InfoExtractor from ..utils import ( - compat_str, ExtractorError, - int_or_none, - str_or_none, - try_get, - url_or_none) +) class TikTokBaseIE(InfoExtractor): def _extract_aweme(self, data): - video = data['video'] - description = str_or_none(try_get(data, lambda x: x['desc'])) - width = int_or_none(try_get(data, lambda x: video['width'])) - height = int_or_none(try_get(data, lambda x: video['height'])) + video = data['props']['pageProps']['metaParams'] + description = video['description'] + video_meta=data['props']['pageProps']['videoData']['itemInfos']['video'] + width = video_meta['videoMeta']['width'] + height = video_meta['videoMeta']['height'] + format_urls=video_meta['urls'] - format_urls = set() formats = [] - for format_id in ( - 'play_addr_lowbr', 'play_addr', 'play_addr_h264', - 'download_addr'): - for format in try_get( - video, lambda x: x[format_id]['url_list'], list) or []: - format_url = url_or_none(format) - if not format_url: - continue - if format_url in format_urls: - continue - format_urls.add(format_url) - formats.append({ - 'url': format_url, - 'ext': 'mp4', - 'height': height, - 'width': width, - }) + for format in format_urls: + formats.append({ + 'url': format, + 'ext': 'mp4', + 'height': height, + 'width': width, + }) self._sort_formats(formats) - - thumbnail = url_or_none(try_get( - video, lambda x: x['cover']['url_list'][0], compat_str)) - uploader = try_get(data, lambda x: x['author']['nickname'], compat_str) - timestamp = int_or_none(data.get('create_time')) - comment_count = int_or_none(data.get('comment_count')) or int_or_none( - try_get(data, lambda x: x['statistics']['comment_count'])) - repost_count = int_or_none(try_get( - data, lambda x: x['statistics']['share_count'])) - - aweme_id = data['aweme_id'] - return { - 'id': aweme_id, - 'title': uploader or aweme_id, 'description': description, - 'thumbnail': thumbnail, - 'uploader': uploader, - 'timestamp': timestamp, - 'comment_count': comment_count, - 'repost_count': repost_count, - 'formats': formats, + 'formats': formats } @@ -115,14 +83,16 @@ class TikTokIE(TikTokBaseIE): comments_count = item_info['commentCount'] likes_count = item_info['diggCount'] + entry=self._extract_aweme(data_dict) + return self.info_dict(video_id, str(url), json_api['title'], json_api['author_name'], timestamp, json_api['thumbnail_url'], - views, provider_id, False, 'not_live', likes_count, shares, '', comments_count, duration, json_api['html']) + views, provider_id, False, 'not_live', likes_count, shares, '', comments_count, duration, json_api['html'], entry['formats']) def info_dict(self, video_id, url, video_title, uploader, timestamp, thumbnail, view_count, uploader_id, is_live, live_status - , likes_count, shares_count, subtitles, comment_count, duration, embed_code): + , likes_count, shares_count, subtitles, comment_count, duration, embed_code, format): info_dict = { 'id': video_id, 'url': url, @@ -140,7 +110,8 @@ class TikTokIE(TikTokBaseIE): 'comment_count': comment_count, 'duration': duration, 'ext':'mp.4', - 'embed_code': embed_code + 'embed_code': embed_code, + 'format': format } return info_dict From b7103c6c5fd02e5c24d0d713d1b2d564f98e3caf Mon Sep 17 00:00:00 2001 From: bhodaya Date: Sun, 3 May 2020 20:12:06 +0300 Subject: [PATCH 6/6] download video --- test/ci/test_tiktok.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ci/test_tiktok.py b/test/ci/test_tiktok.py index e92981283..0aad8e420 100644 --- a/test/ci/test_tiktok.py +++ b/test/ci/test_tiktok.py @@ -34,7 +34,7 @@ class TikTokTestYoutubeDl(unittest.TestCase): ydl = youtube_dl.YoutubeDL(params) info = ydl.extract_info(url, download=True) self.assertTrue(os.path.exists("Imagine lebron freaking out over something you did! #foryou #ballislife #lebron #nba-6783617809113943301.mp.4")) - + os.remove("Imagine lebron freaking out over something you did! #foryou #ballislife #lebron #nba-6783617809113943301.mp.4")