move mpd multisegement parsing into dash downloader

youtube_dl/downloader/dash.py

@@ -2,12 +2,18 @@ from __future__ import unicode_literals
 
 import os
 import re
+import math
 
 from .fragment import FragmentFD
-from ..compat import compat_urllib_error
+from ..compat import (
+    compat_urllib_error,
+    compat_etree_fromstring,
+)
 from ..utils import (
     sanitize_open,
     encodeFilename,
+    xpath_with_ns,
+    parse_duration,
 )
 
 
@@ -18,10 +24,129 @@ class DashSegmentsFD(FragmentFD):
 
     FD_NAME = 'dashsegments'
 
+    def _extract_multisegment_info(self, mpd_doc, mpd_url, representation_id):
+        mobj = re.search(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag)
+        namespace = mobj.group(1) if mobj else None
+        ns_map = {'mpd': namespace} if namespace else {}
+
+        def _add_ns(path):
+            return xpath_with_ns(path, ns_map) if ns_map else path
+
+        period, adaptation_set, representation = None, None, None
+        found = False
+        for p in mpd_doc.findall(_add_ns('mpd:Period')):
+            for a in p.findall(_add_ns('mpd:AdaptationSet')):
+                for r in a.findall(_add_ns('mpd:Representation')):
+                    if r.get('id') == representation_id:
+                        period, adaptation_set, representation = p, a, r
+                        found = True
+                        break
+                if found:
+                    break
+            if found:
+                break
+
+        mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
+        period_duration = parse_duration(period.get('duration')) or mpd_duration
+
+        mpd_base_url = mpd_url.rpartition('/')[0]
+        base_url = ''
+        for element in (representation, adaptation_set, period, mpd_doc):
+            base_url_e = element.find(_add_ns('mpd:BaseURL'))
+            if base_url_e is not None:
+                base_url = base_url_e.text + base_url
+                if re.match(r'^https?://', base_url):
+                    break
+        if mpd_base_url and not re.match(r'^https?://', base_url):
+            if not mpd_base_url.endswith('/') and not base_url.startswith('/'):
+                mpd_base_url += '/'
+            base_url = mpd_base_url + base_url
+
+        ms_info = {
+            'base_url': base_url,
+            'start_number': 1,
+            'timescale': 1,
+        }
+
+        for element in (period, adaptation_set, representation):
+            segment_list = element.find(_add_ns('mpd:SegmentList'))
+            if segment_list is not None:
+                segment_urls_e = segment_list.findall(_add_ns('mpd:SegmentURL'))
+                if segment_urls_e:
+                    ms_info['segment_urls'] = [segment.attrib['media'] for segment in segment_urls_e]
+                initialization = segment_list.find(_add_ns('mpd:Initialization'))
+                if initialization is not None:
+                    ms_info['initialization_url'] = initialization.attrib['sourceURL']
+            else:
+                segment_template = element.find(_add_ns('mpd:SegmentTemplate'))
+                if segment_template is not None:
+                    start_number = segment_template.get('startNumber')
+                    if start_number:
+                        ms_info['start_number'] = int(start_number)
+                    timescale = segment_template.get('timescale')
+                    if timescale:
+                        ms_info['timescale'] = int(timescale)
+                    segment_timeline = segment_template.find(_add_ns('mpd:SegmentTimeline'))
+                    if segment_timeline is not None:
+                        s_e = segment_timeline.findall(_add_ns('mpd:S'))
+                        if s_e:
+                            current_start_time = 0
+                            ms_info['segment_start_times'] = []
+                            for s in s_e:
+                                for i in int(s.get('r', '0')) + 1:
+                                    ms_info['segment_start_times'].append(current_start_time)
+                                    current_start_time += int(s['d']) / ms_info['timescale']
+                    else:
+                        segment_duration = segment_template.get('duration')
+                        if segment_duration:
+                            ms_info['segment_duration'] = int(segment_duration)
+                    media_template = segment_template.get('media')
+                    if media_template:
+                        ms_info['media_template'] = media_template
+                    initialization = segment_template.get('initialization')
+                    if initialization:
+                        ms_info['initialization_url'] = initialization
+                    else:
+                        initialization = segment_template.find(_add_ns('mpd:Initialization'))
+                        if initialization is not None:
+                            ms_info['initialization_url'] = initialization.attrib['sourceURL']
+
+        if 'segment_urls' not in ms_info and 'media_template' in ms_info:
+            if 'segment_start_times' not in ms_info and 'segment_duration':
+                segment_duration = float(ms_info['segment_duration']) / float(ms_info['timescale'])
+                ms_info['segment_start_times'] = [i * segment_duration for i in range(
+                    int(math.ceil(float(period_duration) / segment_duration)))]
+            media_template = ms_info['media_template']
+            media_template = media_template.replace('$RepresentationID$', representation_id)
+            media_template = re.sub(r'\$(Time|Number|Bandwidth)\$', r'%(\1)d', media_template)
+            media_template = re.sub(r'\$(Time|Number|Bandwidth)%(\d+)d\$', r'%(\1)\2d', media_template)
+            media_template.replace('$$', '$')
+            ms_info['segment_urls'] = [
+                media_template % {
+                    'Number': segment_number,
+                    'Bandwidth': representation.attrib.get('bandwidth'),
+                    'Time': segement_start_time,
+                } for segment_number, segement_start_time in enumerate(
+                    ms_info['segment_start_times'],
+                    ms_info['start_number'])]
+        return ms_info
+
     def real_download(self, filename, info_dict):
-        base_url = info_dict['url']
+        mpd_url = info_dict['url']
-        segment_urls = [info_dict['segment_urls'][0]] if self.params.get('test', False) else info_dict['segment_urls']
+        params = info_dict['_downloader_params']
-        initialization_url = info_dict.get('initialization_url')
+        mpd_doc = params.get('mpd')
+        is_live = mpd_doc is None
+        if is_live:
+            self.to_screen('[%s] Downloading MPD manifest' % self.FD_NAME)
+            urlh = self.ydl.urlopen(mpd_url)
+            mpd_url = urlh.geturl()
+            mpd_doc = compat_etree_fromstring(urlh.read().decode('utf-8'))
+
+        representation_id = params.get('representation_id')
+        ms_info = self._extract_multisegment_info(mpd_doc, mpd_url, representation_id)
+        segment_urls = [ms_info['segment_urls'][0]] if self.params.get('test', False) else ms_info['segment_urls']
+        initialization_url = ms_info.get('initialization_url')
+        base_url = ms_info.get('base_url')
 
         ctx = {
             'filename': filename,

youtube_dl/extractor/common.py

@@ -24,6 +24,8 @@ from ..compat import (
     compat_urllib_parse_urlencode,
     compat_urllib_request,
     compat_urlparse,
+    compat_urllib_parse_urlparse,
+    compat_urllib_parse,
 )
 from ..downloader.f4m import remove_encrypted_media
 from ..utils import (
@@ -1405,21 +1407,20 @@ class InfoExtractor(object):
         return entries
 
     def _extract_mpd_formats(self, mpd_url, video_id, mpd_id=None, note=None, errnote=None, fatal=True, formats_dict={}):
-        res = self._download_webpage_handle(
+        mpd_doc = self._download_xml(
             mpd_url, video_id,
             note=note or 'Downloading MPD manifest',
             errnote=errnote or 'Failed to download MPD manifest',
             fatal=fatal)
-        if res is False:
+
+        if mpd_doc is False:
             return []
-        mpd, urlh = res
-        mpd_base_url = re.match(r'https?://.+/', urlh.geturl()).group()
 
-        return self._parse_mpd_formats(
+        return self._parse_mpd_formats(mpd_doc, mpd_id, mpd_url, formats_dict=formats_dict)
-            compat_etree_fromstring(mpd.encode('utf-8')), mpd_id, mpd_base_url, formats_dict=formats_dict)
 
-    def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}):
+    def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_url='', formats_dict={}):
-        if mpd_doc.get('type') == 'dynamic':
+        mpd_type = mpd_doc.get('type', 'static')
+        if mpd_type == 'dynamic':
             return []
 
         namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace', default=None)
@@ -1430,63 +1431,24 @@ class InfoExtractor(object):
         def is_drm_protected(element):
             return element.find(_add_ns('ContentProtection')) is not None
 
-        def extract_multisegment_info(element, ms_parent_info):
+        def has_multisegment(element):
-            ms_info = ms_parent_info.copy()
+            return element.find(_add_ns('SegmentList')) is not None or element.find(_add_ns('SegmentTemplate')) is not None
-            segment_list = element.find(_add_ns('SegmentList'))
-            if segment_list is not None:
-                segment_urls_e = segment_list.findall(_add_ns('SegmentURL'))
-                if segment_urls_e:
-                    ms_info['segment_urls'] = [segment.attrib['media'] for segment in segment_urls_e]
-                initialization = segment_list.find(_add_ns('Initialization'))
-                if initialization is not None:
-                    ms_info['initialization_url'] = initialization.attrib['sourceURL']
-            else:
-                segment_template = element.find(_add_ns('SegmentTemplate'))
-                if segment_template is not None:
-                    start_number = segment_template.get('startNumber')
-                    if start_number:
-                        ms_info['start_number'] = int(start_number)
-                    segment_timeline = segment_template.find(_add_ns('SegmentTimeline'))
-                    if segment_timeline is not None:
-                        s_e = segment_timeline.findall(_add_ns('S'))
-                        if s_e:
-                            ms_info['total_number'] = 0
-                            for s in s_e:
-                                ms_info['total_number'] += 1 + int(s.get('r', '0'))
-                    else:
-                        timescale = segment_template.get('timescale')
-                        if timescale:
-                            ms_info['timescale'] = int(timescale)
-                        segment_duration = segment_template.get('duration')
-                        if segment_duration:
-                            ms_info['segment_duration'] = int(segment_duration)
-                    media_template = segment_template.get('media')
-                    if media_template:
-                        ms_info['media_template'] = media_template
-                    initialization = segment_template.get('initialization')
-                    if initialization:
-                        ms_info['initialization_url'] = initialization
-                    else:
-                        initialization = segment_template.find(_add_ns('Initialization'))
-                        if initialization is not None:
-                            ms_info['initialization_url'] = initialization.attrib['sourceURL']
-            return ms_info
 
-        mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
+        mpd_base_url = mpd_url.rpartition('/')[0]
+        parsed_mpd_url = None
+        if mpd_base_url:
+            parsed_mpd_url = compat_urllib_parse_urlparse(mpd_url)
         formats = []
         for period in mpd_doc.findall(_add_ns('Period')):
-            period_duration = parse_duration(period.get('duration')) or mpd_duration
+            is_ms = has_multisegment(period)
-            period_ms_info = extract_multisegment_info(period, {
-                'start_number': 1,
-                'timescale': 1,
-            })
             for adaptation_set in period.findall(_add_ns('AdaptationSet')):
                 if is_drm_protected(adaptation_set):
                     continue
-                adaption_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info)
+                is_ms = is_ms or has_multisegment(adaptation_set)
                 for representation in adaptation_set.findall(_add_ns('Representation')):
                     if is_drm_protected(representation):
                         continue
+                    is_ms = is_ms or has_multisegment(representation)
                     representation_attrib = adaptation_set.attrib.copy()
                     representation_attrib.update(representation.attrib)
                     # According to page 41 of ISO/IEC 29001-1:2014, @mimeType is mandatory
@@ -1496,24 +1458,12 @@ class InfoExtractor(object):
                         # TODO implement WebVTT downloading
                         pass
                     elif content_type == 'video' or content_type == 'audio':
-                        base_url = ''
+                        representation_id = representation.attrib['id']
-                        for element in (representation, adaptation_set, period, mpd_doc):
-                            base_url_e = element.find(_add_ns('BaseURL'))
-                            if base_url_e is not None:
-                                base_url = base_url_e.text + base_url
-                                if re.match(r'^https?://', base_url):
-                                    break
-                        if mpd_base_url and not re.match(r'^https?://', base_url):
-                            if not mpd_base_url.endswith('/') and not base_url.startswith('/'):
-                                mpd_base_url += '/'
-                            base_url = mpd_base_url + base_url
-                        representation_id = representation_attrib.get('id')
                         lang = representation_attrib.get('lang')
                         url_el = representation.find(_add_ns('BaseURL'))
                         filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
                         f = {
                             'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
-                            'url': base_url,
                             'ext': mimetype2ext(mime_type),
                             'width': int_or_none(representation_attrib.get('width')),
                             'height': int_or_none(representation_attrib.get('height')),
@@ -1526,35 +1476,42 @@ class InfoExtractor(object):
                             'format_note': 'DASH %s' % content_type,
                             'filesize': filesize,
                         }
-                        representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
+                        if is_ms:
-                        if 'segment_urls' not in representation_ms_info and 'media_template' in representation_ms_info:
+                            f['_downloader_params'] = {
-                            if 'total_number' not in representation_ms_info and 'segment_duration':
+                                'representation_id': representation_id,
-                                segment_duration = float(representation_ms_info['segment_duration']) / float(representation_ms_info['timescale'])
+                            }
-                                representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
+                            if parsed_mpd_url:
-                            media_template = representation_ms_info['media_template']
+                                fragment_parts = ((period, 'id', 'period'), (adaptation_set, 'id', 'as'), (adaptation_set, 'group', 'track'))
-                            media_template = media_template.replace('$RepresentationID$', representation_id)
+                                params = {}
-                            media_template = re.sub(r'\$(Number|Bandwidth)\$', r'%(\1)d', media_template)
+                                for ele, attr, frag_attr in fragment_parts:
-                            media_template = re.sub(r'\$(Number|Bandwidth)%(\d+)\$', r'%(\1)\2d', media_template)
+                                    frag_val = ele.attrib.get(attr)
-                            media_template.replace('$$', '$')
+                                    if frag_val:
-                            representation_ms_info['segment_urls'] = [
+                                        params[frag_attr] = frag_val
-                                media_template % {
+                                fragment = compat_urllib_parse.urlencode(params)
-                                    'Number': segment_number,
+                                fragment_mpd_url = parsed_mpd_url._replace(fragment=fragment).geturl()
-                                    'Bandwidth': representation_attrib.get('bandwidth')}
-                                for segment_number in range(
-                                    representation_ms_info['start_number'],
-                                    representation_ms_info['total_number'] + representation_ms_info['start_number'])]
-                        if 'segment_urls' in representation_ms_info:
                                 f.update({
-                                'segment_urls': representation_ms_info['segment_urls'],
+                                    'url': fragment_mpd_url,
                                     'protocol': 'http_dash_segments',
                                 })
-                            if 'initialization_url' in representation_ms_info:
+                            if mpd_type != 'dynamic':
-                                initialization_url = representation_ms_info['initialization_url'].replace('$RepresentationID$', representation_id)
+                                f['_downloader_params'].update({
-                                f.update({
+                                    'mpd': mpd_doc,
-                                    'initialization_url': initialization_url,
+                                })
+                        else:
+                            base_url = ''
+                            for element in (representation, adaptation_set, period, mpd_doc):
+                                base_url_e = element.find(_add_ns('BaseURL'))
+                                if base_url_e is not None:
+                                    base_url = base_url_e.text + base_url
+                                    if re.match(r'^https?://', base_url):
+                                        break
+                            if mpd_base_url and not re.match(r'^https?://', base_url):
+                                if not mpd_base_url.endswith('/') and not base_url.startswith('/'):
+                                    mpd_base_url += '/'
+                                base_url = mpd_base_url + base_url
+                            f.update({
+                                'url': base_url,
                             })
-                                if not f.get('url'):
-                                    f['url'] = initialization_url
                         try:
                             existing_format = next(
                                 fo for fo in formats

youtube_dl/extractor/generic.py

@@ -1356,7 +1356,7 @@ class GenericIE(InfoExtractor):
                 return self.playlist_result(self._parse_xspf(doc, video_id), video_id)
             elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
                 info_dict['formats'] = self._parse_mpd_formats(
-                    doc, video_id, mpd_base_url=url.rpartition('/')[0])
+                    doc, video_id, mpd_url=url)
                 self._sort_formats(info_dict['formats'])
                 return info_dict
             elif re.match(r'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc.tag):