Merge branch 'master' of https://github.com/rg3/youtube-dl

2016-08-03 09:06:08 +02:00 · 2016-08-03 09:06:08 +02:00 · 982bfd8395
commit 982bfd8395
parent 8ddff4c81a 899d2bea63
21 changed files with 541 additions and 93 deletions
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@ -6,8 +6,8 @@

 ---

-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.28*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.28**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.08.01*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.08.01**

 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2016.07.28
+[debug] youtube-dl version 2016.08.01
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}
--- a/271
+++ b/271
@ -0,0 +1,271 @@
+version 2016.08.01
+
+Fixed/improved extractors
+- [yandexmusic:track] Adapt to changes in track location JSON (#10193)
+- [bloomberg] Support another form of player (#10187)
+- [limelight] Skip DRM protected videos
+- [safari] Relax regular expressions for URL matching (#10202)
+- [cwtv] Add support for cwtvpr.com (#10196)
+
+version 2016.07.30
+
+Fixed/improved extractors
+- [twitch:clips] Sort formats
+- [tv2] Use m3u8_native
+- [tv2:article] Fix video detection (#10188)
+- rtve (#10076)
+- [dailymotion:playlist] Optimize download archive processing (#10180)
+
+
+version 2016.07.28
+
+Fixed/improved extractors
+- shared (#10170)
+- soundcloud (#10179)
+- twitch (#9767)
+
+
+version 2016.07.26.2
+
+Fixed/improved extractors
+- smotri
+- camdemy
+- mtv
+- comedycentral
+- cmt
+- cbc
+- mgtv
+- orf
+
+
+version 2016.07.24
+
+New extractors
+- arkena (#8682)
+- lcp (#8682)
+
+Fixed/improved extractors
+- facebook (#10151)
+- dailymail
+- telegraaf
+- dcn
+- onet
+- tvp
+
+Miscellaneous
+- Support $Time$ in DASH manifests
+
+
+version 2016.07.22
+
+New extractors
+- odatv (#9285)
+
+Fixed/improved extractors
+- bbc
+- youjizz (#10131)
+- youtube (#10140)
+- pornhub (#10138)
+- eporner (#10139)
+
+
+version 2016.07.17
+
+New extractors
+- nintendo (#9986)
+- streamable (#9122)
+
+Fixed/improved extractors
+- ard (#10095)
+- mtv
+- comedycentral (#10101)
+- viki (#10098)
+- spike (#10106)
+
+Miscellaneous
+- Improved twitter player detection (#10090)
+
+
+version 2016.07.16
+
+New extractors
+- ninenow (#5181)
+
+Fixed/improved extractors
+- rtve (#10076)
+- brightcove
+- 3qsdn
+- syfy (#9087, #3820, #2388)
+- youtube (#10083)
+
+Miscellaneous
+- Fix subtitle embedding for video-only and audio-only files (#10081)
+
+
+version 2016.07.13
+
+New extractors
+- rudo
+
+Fixed/improved extractors
+- biobiochiletv
+- tvplay
+- dbtv
+- brightcove
+- tmz
+- youtube (#10059)
+- shahid (#10062)
+- vk
+- ellentv (#10067)
+
+
+version 2016.07.11
+
+New Extractors
+- roosterteeth (#9864)
+
+Fixed/improved extractors
+- miomio (#9605)
+- vuclip
+- youtube
+- vidzi (#10058)
+
+
+version 2016.07.09.2
+
+Fixed/improved extractors
+- vimeo (#1638)
+- facebook (#10048)
+- lynda (#10047)
+- animeondemand
+
+Fixed/improved features
+- Embedding subtitles no longer throws an error with problematic inputs (#9063)
+
+
+version 2016.07.09.1
+
+Fixed/improved extractors
+- youtube
+- ard
+- srmediatek (#9373)
+
+
+version 2016.07.09
+
+New extractors
+- Flipagram (#9898)
+
+Fixed/improved extractors
+- telecinco
+- toutv
+- radiocanada
+- tweakers (#9516)
+- lynda
+- nick (#7542)
+- polskieradio (#10028)
+- le
+- facebook (#9851)
+- mgtv
+- animeondemand (#10031)
+
+Fixed/improved features
+- `--postprocessor-args` and `--downloader-args` now accepts non-ASCII inputs
+  on non-Windows systems
+
+
+version 2016.07.07
+
+New extractors
+- kamcord (#10001)
+
+Fixed/improved extractors
+- spiegel (#10018)
+- metacafe (#8539, #3253)
+- onet (#9950)
+- francetv (#9955)
+- brightcove (#9965)
+- daum (#9972)
+
+
+version 2016.07.06
+
+Fixed/improved extractors
+- youtube (#10007, #10009)
+- xuite
+- stitcher
+- spiegel
+- slideshare
+- sandia
+- rtvnh
+- prosiebensat1
+- onionstudios
+
+
+version 2016.07.05
+
+Fixed/improved extractors
+- brightcove
+- yahoo (#9995)
+- pornhub (#9997)
+- iqiyi
+- kaltura (#5557)
+- la7
+- Changed features
+- Rename --cn-verfication-proxy to --geo-verification-proxy
+Miscellaneous
+- Add script for displaying downloads statistics
+
+
+version 2016.07.03.1
+
+Fixed/improved extractors
+- theplatform
+- aenetworks
+- nationalgeographic
+- hrti (#9482)
+- facebook (#5701)
+- buzzfeed (#5701)
+- rai (#8617, #9157, #9232, #8552, #8551)
+- nationalgeographic (#9991)
+- iqiyi
+
+
+version 2016.07.03
+
+New extractors
+- hrti (#9482)
+
+Fixed/improved extractors
+- vk (#9981)
+- facebook (#9938)
+- xtube (#9953, #9961)
+
+
+version 2016.07.02
+
+New extractors
+- fusion (#9958)
+
+Fixed/improved extractors
+- twitch (#9975)
+- vine (#9970)
+- periscope (#9967)
+- pornhub (#8696)
+
+
+version 2016.07.01
+
+New extractors
+- 9c9media
+- ctvnews (#2156)
+- ctv (#4077)
+
+Fixed/Improved extractors
+- rds
+- meta (#8789)
+- pornhub (#9964)
+- sixplay (#2183)
+
+New features
+- Accept quoted strings across multiple lines (#9940)
--- a/4
+++ b/4
@ -94,7 +94,7 @@ _EXTRACTOR_FILES != find youtube_dl/extractor -iname '*.py' -and -not -iname 'la
 youtube_dl/extractor/lazy_extractors.py: devscripts/make_lazy_extractors.py devscripts/lazy_load_template.py $(_EXTRACTOR_FILES)
 	$(PYTHON) devscripts/make_lazy_extractors.py $@

-youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
+youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish ChangeLog
 	@tar -czf youtube-dl.tar.gz --transform "s|^|youtube-dl/|" --owner 0 --group 0 \
 		--exclude '*.DS_Store' \
 		--exclude '*.kate-swp' \
@ -107,7 +107,7 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-
 		--exclude 'docs/_build' \
 		-- \
 		bin devscripts test youtube_dl docs \
-		LICENSE README.md README.txt \
+		ChangeLog LICENSE README.md README.txt \
 		Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion \
 		youtube-dl.zsh youtube-dl.fish setup.py \
 		youtube-dl
--- a/README.md
+++ b/README.md
@ -424,7 +424,7 @@ which means you can modify it, redistribute it or use it however you like.

 # CONFIGURATION

-You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux and OS X, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. Note that by default configuration file may not exist so you may need to create it yourself.
+You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux and OS X, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config/youtube-dl.conf`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. Note that by default configuration file may not exist so you may need to create it yourself.

 For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime, use a proxy and save all videos under `Movies` directory in your home directory:
 ```
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@ -71,9 +71,12 @@ fi
 /bin/echo -e "\n### Changing version in version.py..."
 sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py

+/bin/echo -e "\n### Changing version in ChangeLog..."
+sed -i "s/<unreleased>/$version/" ChangeLog
+
 /bin/echo -e "\n### Committing documentation, templates and youtube_dl/version.py..."
 make README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md supportedsites
-git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md docs/supportedsites.md youtube_dl/version.py
+git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md docs/supportedsites.md youtube_dl/version.py ChangeLog
 git commit $gpg_sign_commits -m "release $version"

 /bin/echo -e "\n### Now tagging, signing and pushing..."
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@ -48,6 +48,9 @@ class TestInfoExtractor(unittest.TestCase):
        self.assertEqual(ie._og_search_property('foobar', html), 'Foo')
        self.assertEqual(ie._og_search_property('test1', html), 'foo > < bar')
        self.assertEqual(ie._og_search_property('test2', html), 'foo >//< bar')
+        self.assertEqual(ie._og_search_property(('test0', 'test1'), html), 'foo > < bar')
+        self.assertRaises(RegexNotFoundError, ie._og_search_property, 'test0', html, None, fatal=True)
+        self.assertRaises(RegexNotFoundError, ie._og_search_property, ('test0', 'test00'), html, None, fatal=True)

    def test_html_search_meta(self):
        ie = self.ie
--- a/test/test_verbose_output.py
+++ b/test/test_verbose_output.py
@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import unittest
+
+import sys
+import os
+import subprocess
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+class TestVerboseOutput(unittest.TestCase):
+    def test_private_info_arg(self):
+        outp = subprocess.Popen(
+            [
+                sys.executable, 'youtube_dl/__main__.py', '-v',
+                '--username', 'johnsmith@gmail.com',
+                '--password', 'secret',
+            ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        sout, serr = outp.communicate()
+        self.assertTrue('--username' in serr)
+        self.assertTrue('johnsmith' not in serr)
+        self.assertTrue('--password' in serr)
+        self.assertTrue('secret' not in serr)
+
+    def test_private_info_shortarg(self):
+        outp = subprocess.Popen(
+            [
+                sys.executable, 'youtube_dl/__main__.py', '-v',
+                '-u', 'johnsmith@gmail.com',
+                '-p', 'secret',
+            ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        sout, serr = outp.communicate()
+        self.assertTrue('-u' in serr)
+        self.assertTrue('johnsmith' not in serr)
+        self.assertTrue('-p' in serr)
+        self.assertTrue('secret' not in serr)
+
+    def test_private_info_eq(self):
+        outp = subprocess.Popen(
+            [
+                sys.executable, 'youtube_dl/__main__.py', '-v',
+                '--username=johnsmith@gmail.com',
+                '--password=secret',
+            ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        sout, serr = outp.communicate()
+        self.assertTrue('--username' in serr)
+        self.assertTrue('johnsmith' not in serr)
+        self.assertTrue('--password' in serr)
+        self.assertTrue('secret' not in serr)
+
+    def test_private_info_shortarg_eq(self):
+        outp = subprocess.Popen(
+            [
+                sys.executable, 'youtube_dl/__main__.py', '-v',
+                '-u=johnsmith@gmail.com',
+                '-p=secret',
+            ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        sout, serr = outp.communicate()
+        self.assertTrue('-u' in serr)
+        self.assertTrue('johnsmith' not in serr)
+        self.assertTrue('-p' in serr)
+        self.assertTrue('secret' not in serr)
+
+if __name__ == '__main__':
+    unittest.main()
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals

 import re
@ -20,6 +21,18 @@ class BloombergIE(InfoExtractor):
        'params': {
            'format': 'best[format_id^=hds]',
        },
+    }, {
+        # video ID in BPlayer(...)
+        'url': 'http://www.bloomberg.com/features/2016-hello-world-new-zealand/',
+        'info_dict': {
+            'id': '938c7e72-3f25-4ddb-8b85-a9be731baa74',
+            'ext': 'flv',
+            'title': 'Meet the Real-Life Tech Wizards of Middle Earth',
+            'description': 'Hello World, Episode 1: New Zealand’s freaky AI babies, robot exoskeletons, and a virtual you.',
+        },
+        'params': {
+            'format': 'best[format_id^=hds]',
+        },
    }, {
        'url': 'http://www.bloomberg.com/news/articles/2015-11-12/five-strange-things-that-have-been-happening-in-financial-markets',
        'only_matching': True,
@ -33,7 +46,11 @@ class BloombergIE(InfoExtractor):
        webpage = self._download_webpage(url, name)
        video_id = self._search_regex(
            r'["\']bmmrId["\']\s*:\s*(["\'])(?P<url>.+?)\1',
-            webpage, 'id', group='url')
+            webpage, 'id', group='url', default=None)
+        if not video_id:
+            bplayer_data = self._parse_json(self._search_regex(
+                r'BPlayer\(null,\s*({[^;]+})\);', webpage, 'id'), name)
+            video_id = bplayer_data['id']
        title = re.sub(': Video$', '', self._og_search_title(webpage))

        embed_info = self._download_json(
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@ -1,6 +1,7 @@
 from __future__ import unicode_literals

 from .mtv import MTVServicesInfoExtractor
+from .common import InfoExtractor


 class ComedyCentralIE(MTVServicesInfoExtractor):
@ -96,3 +97,22 @@ class ComedyCentralTVIE(MTVServicesInfoExtractor):
            webpage, 'mrss url', group='url')

        return self._get_videos_info_from_url(mrss_url, video_id)
+
+
+class ComedyCentralShortnameIE(InfoExtractor):
+    _VALID_URL = r'^:(?P<id>tds|thedailyshow)$'
+    _TESTS = [{
+        'url': ':tds',
+        'only_matching': True,
+    }, {
+        'url': ':thedailyshow',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        shortcut_map = {
+            'tds': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes',
+            'thedailyshow': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes',
+        }
+        return self.url_result(shortcut_map[video_id])
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@ -727,9 +727,14 @@ class InfoExtractor(object):
                    [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(prop)

    def _og_search_property(self, prop, html, name=None, **kargs):
+        if not isinstance(prop, (list, tuple)):
+            prop = [prop]
        if name is None:
-            name = 'OpenGraph %s' % prop
-        escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs)
+            name = 'OpenGraph %s' % prop[0]
+        og_regexes = []
+        for p in prop:
+            og_regexes.extend(self._og_regexes(p))
+        escaped = self._search_regex(og_regexes, html, name, flags=re.DOTALL, **kargs)
        if escaped is None:
            return None
        return unescapeHTML(escaped)
--- a/youtube_dl/extractor/cwtv.py
+++ b/youtube_dl/extractor/cwtv.py
@ -9,7 +9,7 @@ from ..utils import (


 class CWTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?cw(?:tv|seed)\.com/(?:shows/)?(?:[^/]+/){2}\?.*\bplay=(?P<id>[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})'
+    _VALID_URL = r'https?://(?:www\.)?cw(?:tv(?:pr)?|seed)\.com/(?:shows/)?(?:[^/]+/)+[^?]*\?.*\b(?:play|watch)=(?P<id>[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})'
    _TESTS = [{
        'url': 'http://cwtv.com/shows/arrow/legends-of-yesterday/?play=6b15e985-9345-4f60-baf8-56e96be57c63',
        'info_dict': {
@ -51,6 +51,12 @@ class CWTVIE(InfoExtractor):
    }, {
        'url': 'http://cwtv.com/thecw/chroniclesofcisco/?play=8adebe35-f447-465f-ab52-e863506ff6d6',
        'only_matching': True,
+    }, {
+        'url': 'http://cwtvpr.com/the-cw/video?watch=9eee3f60-ef4e-440b-b3b2-49428ac9c54e',
+        'only_matching': True,
+    }, {
+        'url': 'http://cwtv.com/shows/arrow/legends-of-yesterday/?watch=6b15e985-9345-4f60-baf8-56e96be57c63',
+        'only_matching': True,
    }]

    def _real_extract(self, url):
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@ -331,7 +331,9 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):

            for video_id in re.findall(r'data-xid="(.+?)"', webpage):
                if video_id not in video_ids:
-                    yield self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
+                    yield self.url_result(
+                        'http://www.dailymotion.com/video/%s' % video_id,
+                        DailymotionIE.ie_key(), video_id)
                    video_ids.add(video_id)

            if re.search(self._MORE_PAGES_INDICATOR, webpage) is None:
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@ -159,6 +159,7 @@ from .coub import CoubIE
 from .collegerama import CollegeRamaIE
 from .comedycentral import (
    ComedyCentralIE,
+    ComedyCentralShortnameIE,
    ComedyCentralTVIE,
    ToshIE,
 )
--- a/youtube_dl/extractor/limelight.py
+++ b/youtube_dl/extractor/limelight.py
@ -37,11 +37,12 @@ class LimelightBaseIE(InfoExtractor):

        for stream in streams:
            stream_url = stream.get('url')
-            if not stream_url:
+            if not stream_url or stream.get('drmProtected'):
                continue
-            if '.f4m' in stream_url:
+            ext = determine_ext(stream_url)
+            if ext == 'f4m':
                formats.extend(self._extract_f4m_formats(
-                    stream_url, video_id, fatal=False))
+                    stream_url, video_id, f4m_id='hds', fatal=False))
            else:
                fmt = {
                    'url': stream_url,
@ -50,7 +51,7 @@ class LimelightBaseIE(InfoExtractor):
                    'fps': float_or_none(stream.get('videoFrameRate')),
                    'width': int_or_none(stream.get('videoWidthInPixels')),
                    'height': int_or_none(stream.get('videoHeightInPixels')),
-                    'ext': determine_ext(stream_url)
+                    'ext': ext,
                }
                rtmp = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>.+))/(?P<playpath>mp4:.+)$', stream_url)
                if rtmp:
@ -68,18 +69,23 @@ class LimelightBaseIE(InfoExtractor):

        for mobile_url in mobile_urls:
            media_url = mobile_url.get('mobileUrl')
-            if not media_url:
-                continue
            format_id = mobile_url.get('targetMediaPlatform')
-            if determine_ext(media_url) == 'm3u8':
+            if not media_url or format_id == 'Widevine':
+                continue
+            ext = determine_ext(media_url)
+            if ext == 'm3u8':
                formats.extend(self._extract_m3u8_formats(
                    media_url, video_id, 'mp4', 'm3u8_native',
                    m3u8_id=format_id, fatal=False))
+            elif ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    stream_url, video_id, f4m_id=format_id, fatal=False))
            else:
                formats.append({
                    'url': media_url,
                    'format_id': format_id,
                    'preference': -1,
+                    'ext': ext,
                })

        self._sort_formats(formats)
@ -145,7 +151,7 @@ class LimelightMediaIE(LimelightBaseIE):
        'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
        'info_dict': {
            'id': '3ffd040b522b4485b6d84effc750cd86',
-            'ext': 'flv',
+            'ext': 'mp4',
            'title': 'HaP and the HB Prince Trailer',
            'description': 'md5:8005b944181778e313d95c1237ddb640',
            'thumbnail': 're:^https?://.*\.jpeg$',
@ -154,7 +160,7 @@ class LimelightMediaIE(LimelightBaseIE):
            'upload_date': '20090604',
        },
        'params': {
-            # rtmp download
+            # m3u8 download
            'skip_download': True,
        },
    }, {
@ -164,7 +170,6 @@ class LimelightMediaIE(LimelightBaseIE):
            'id': 'a3e00274d4564ec4a9b29b9466432335',
            'ext': 'flv',
            'title': '3Play Media Overview Video',
-            'description': '',
            'thumbnail': 're:^https?://.*\.jpeg$',
            'duration': 78.101,
            'timestamp': 1338929955,
--- a/youtube_dl/extractor/ntvru.py
+++ b/youtube_dl/extractor/ntvru.py
@ -11,10 +11,9 @@ from ..utils import (

 class NTVRuIE(InfoExtractor):
    IE_NAME = 'ntv.ru'
-    _VALID_URL = r'https?://(?:www\.)?ntv\.ru/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?ntv\.ru/(?:[^/]+/)*(?P<id>[^/?#&]+)'

-    _TESTS = [
-        {
+    _TESTS = [{
        'url': 'http://www.ntv.ru/novosti/863142/',
        'md5': 'ba7ea172a91cb83eb734cad18c10e723',
        'info_dict': {
@ -25,8 +24,7 @@ class NTVRuIE(InfoExtractor):
            'thumbnail': 're:^http://.*\.jpg',
            'duration': 136,
        },
-        },
-        {
+    }, {
        'url': 'http://www.ntv.ru/video/novosti/750370/',
        'md5': 'adecff79691b4d71e25220a191477124',
        'info_dict': {
@ -37,8 +35,7 @@ class NTVRuIE(InfoExtractor):
            'thumbnail': 're:^http://.*\.jpg',
            'duration': 172,
        },
-        },
-        {
+    }, {
        'url': 'http://www.ntv.ru/peredacha/segodnya/m23700/o232416',
        'md5': '82dbd49b38e3af1d00df16acbeab260c',
        'info_dict': {
@ -49,8 +46,7 @@ class NTVRuIE(InfoExtractor):
            'thumbnail': 're:^http://.*\.jpg',
            'duration': 1496,
        },
-        },
-        {
+    }, {
        'url': 'http://www.ntv.ru/kino/Koma_film',
        'md5': 'f825770930937aa7e5aca0dc0d29319a',
        'info_dict': {
@ -61,8 +57,7 @@ class NTVRuIE(InfoExtractor):
            'thumbnail': 're:^http://.*\.jpg',
            'duration': 5592,
        },
-        },
-        {
+    }, {
        'url': 'http://www.ntv.ru/serial/Delo_vrachey/m31760/o233916/',
        'md5': '9320cd0e23f3ea59c330dc744e06ff3b',
        'info_dict': {
@ -73,8 +68,7 @@ class NTVRuIE(InfoExtractor):
            'thumbnail': 're:^http://.*\.jpg',
            'duration': 2590,
        },
-        },
-    ]
+    }]

    _VIDEO_ID_REGEXES = [
        r'<meta property="og:url" content="http://www\.ntv\.ru/video/(\d+)',
@ -87,11 +81,21 @@ class NTVRuIE(InfoExtractor):

        webpage = self._download_webpage(url, video_id)

-        video_id = self._html_search_regex(self._VIDEO_ID_REGEXES, webpage, 'video id')
+        video_url = self._og_search_property(
+            ('video', 'video:iframe'), webpage, default=None)
+        if video_url:
+            video_id = self._search_regex(
+                r'https?://(?:www\.)?ntv\.ru/video/(?:embed/)?(\d+)',
+                video_url, 'video id', default=None)
+
+        if not video_id:
+            video_id = self._html_search_regex(
+                self._VIDEO_ID_REGEXES, webpage, 'video id')

        player = self._download_xml(
            'http://www.ntv.ru/vi%s/' % video_id,
            video_id, 'Downloading video XML')
+
        title = clean_html(xpath_text(player, './data/title', 'title', fatal=True))
        description = clean_html(xpath_text(player, './data/description', 'description'))

--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@ -113,6 +113,8 @@ class RTVEALaCartaIE(InfoExtractor):
        png = self._download_webpage(png_request, video_id, 'Downloading url information')
        video_url = _decrypt_url(png)
        if not video_url.endswith('.f4m'):
+            if '?' not in video_url:
+                video_url = video_url.replace('resources/', 'auth/resources/')
            video_url = video_url.replace('.net.rtve', '.multimedia.cdn.rtve')

        subtitles = None
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@ -75,7 +75,7 @@ class SafariBaseIE(InfoExtractor):
 class SafariIE(SafariBaseIE):
    IE_NAME = 'safari'
    IE_DESC = 'safaribooksonline.com online video'
-    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/library/view/[^/]+/(?P<course_id>[^/]+)/(?P<part>part\d+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/library/view/[^/]+/(?P<course_id>[^/]+)/(?P<part>[^/?#&]+)\.html'

    _TESTS = [{
        'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
@ -92,6 +92,9 @@ class SafariIE(SafariBaseIE):
        # non-digits in course id
        'url': 'https://www.safaribooksonline.com/library/view/create-a-nodejs/100000006A0210/part00.html',
        'only_matching': True,
+    }, {
+        'url': 'https://www.safaribooksonline.com/library/view/learning-path-red/9780134664057/RHCE_Introduction.html',
+        'only_matching': True,
    }]

    def _real_extract(self, url):
@ -132,12 +135,15 @@ class SafariIE(SafariBaseIE):

 class SafariApiIE(SafariBaseIE):
    IE_NAME = 'safari:api'
-    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>part\d+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?safaribooksonline\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>[^/?#&]+)\.html'

-    _TEST = {
+    _TESTS = [{
        'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
        'only_matching': True,
-    }
+    }, {
+        'url': 'https://www.safaribooksonline.com/api/v1/book/9780134664057/chapter/RHCE_Introduction.html',
+        'only_matching': True,
+    }]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
--- a/youtube_dl/extractor/tv2.py
+++ b/youtube_dl/extractor/tv2.py
@ -8,6 +8,7 @@ from ..utils import (
    determine_ext,
    int_or_none,
    float_or_none,
+    js_to_json,
    parse_iso8601,
    remove_end,
 )
@ -54,10 +55,11 @@ class TV2IE(InfoExtractor):
                ext = determine_ext(video_url)
                if ext == 'f4m':
                    formats.extend(self._extract_f4m_formats(
-                        video_url, video_id, f4m_id=format_id))
+                        video_url, video_id, f4m_id=format_id, fatal=False))
                elif ext == 'm3u8':
                    formats.extend(self._extract_m3u8_formats(
-                        video_url, video_id, 'mp4', m3u8_id=format_id))
+                        video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id=format_id, fatal=False))
                elif ext == 'ism' or video_url.endswith('.ism/Manifest'):
                    pass
                else:
@ -105,7 +107,7 @@ class TV2ArticleIE(InfoExtractor):
        'url': 'http://www.tv2.no/2015/05/16/nyheter/alesund/krim/pingvin/6930542',
        'info_dict': {
            'id': '6930542',
-            'title': 'Russen hetses etter pingvintyveri – innrømmer å ha åpnet luken på buret',
+            'title': 'Russen hetses etter pingvintyveri - innrømmer å ha åpnet luken på buret',
            'description': 'md5:339573779d3eea3542ffe12006190954',
        },
        'playlist_count': 2,
@ -119,9 +121,23 @@ class TV2ArticleIE(InfoExtractor):

        webpage = self._download_webpage(url, playlist_id)

+        # Old embed pattern (looks unused nowadays)
+        assets = re.findall(r'data-assetid=["\'](\d+)', webpage)
+
+        if not assets:
+            # New embed pattern
+            for v in re.findall('TV2ContentboxVideo\(({.+?})\)', webpage):
+                video = self._parse_json(
+                    v, playlist_id, transform_source=js_to_json, fatal=False)
+                if not video:
+                    continue
+                asset = video.get('assetId')
+                if asset:
+                    assets.append(asset)
+
        entries = [
-            self.url_result('http://www.tv2.no/v/%s' % video_id, 'TV2')
-            for video_id in re.findall(r'data-assetid="(\d+)"', webpage)]
+            self.url_result('http://www.tv2.no/v/%s' % asset_id, 'TV2')
+            for asset_id in assets]

        title = remove_end(self._og_search_title(webpage), ' - TV2.no')
        description = remove_end(self._og_search_description(webpage), ' - TV2.no')
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@ -75,6 +75,12 @@ class YandexMusicTrackIE(YandexMusicBaseIE):
            % storage_dir,
            track_id, 'Downloading track location JSON')

+        # Each string is now wrapped in a list, this is probably only temporarily thus
+        # supporting both scenarios (see https://github.com/rg3/youtube-dl/issues/10193)
+        for k, v in data.items():
+            if v and isinstance(v, list):
+                data[k] = v[0]
+
        key = hashlib.md5(('XGRlBW9FXlekgbPrRHuSiA' + data['path'][1:] + data['s']).encode('utf-8')).hexdigest()
        storage = storage_dir.split('.')

--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@ -2,6 +2,7 @@ from __future__ import unicode_literals

 import os.path
 import optparse
+import re
 import sys

 from .downloader.external import list_external_downloaders
@ -93,8 +94,18 @@ def parseOpts(overrideArguments=None):
        setattr(parser.values, option.dest, value.split(','))

    def _hide_login_info(opts):
-        opts = list(opts)
-        for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
+        PRIVATE_OPTS = ['-p', '--password', '-u', '--username', '--video-password']
+        eqre = re.compile('^(?P<key>' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$')
+
+        def _scrub_eq(o):
+            m = eqre.match(o)
+            if m:
+                return m.group('key') + '=PRIVATE'
+            else:
+                return o
+
+        opts = list(map(_scrub_eq, opts))
+        for private_opt in PRIVATE_OPTS:
            try:
                i = opts.index(private_opt)
                opts[i + 1] = 'PRIVATE'
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@ -1,3 +1,3 @@
 from __future__ import unicode_literals

-__version__ = '2016.07.28'
+__version__ = '2016.08.01'