Hello community, here is the log from the commit of package youtube-dl for openSUSE:Factory checked in at 2020-06-07 21:38:42 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/youtube-dl (Old) and /work/SRC/openSUSE:Factory/.youtube-dl.new.3606 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "youtube-dl" Sun Jun 7 21:38:42 2020 rev:134 rq:812121 version:2020.06.06 Changes: -------- --- /work/SRC/openSUSE:Factory/youtube-dl/python-youtube-dl.changes 2020-06-02 14:37:11.159245653 +0200 +++ /work/SRC/openSUSE:Factory/.youtube-dl.new.3606/python-youtube-dl.changes 2020-06-07 21:38:54.133561154 +0200 @@ -1,0 +2,8 @@ +Sat Jun 6 23:01:22 UTC 2020 - Jan Engelhardt <jeng...@inai.de> + +- Update to release 2020.06.06 + * jwplatform: Improve embeds extraction + * brightcove: Fix subtitles extraction + * twitch: Pass v5 accept header and fix thumbnails extraction + +------------------------------------------------------------------- youtube-dl.changes: same change Old: ---- youtube-dl-2020.05.29.tar.gz youtube-dl-2020.05.29.tar.gz.sig New: ---- youtube-dl-2020.06.06.tar.gz youtube-dl-2020.06.06.tar.gz.sig ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-youtube-dl.spec ++++++ --- /var/tmp/diff_new_pack.5ZMKh2/_old 2020-06-07 21:38:55.989567038 +0200 +++ /var/tmp/diff_new_pack.5ZMKh2/_new 2020-06-07 21:38:55.993567050 +0200 @@ -19,7 +19,7 @@ %define modname youtube-dl %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-youtube-dl -Version: 2020.05.29 +Version: 2020.06.06 Release: 0 Summary: A Python module for downloading from video sites for offline watching License: SUSE-Public-Domain AND CC-BY-SA-3.0 ++++++ youtube-dl.spec ++++++ --- /var/tmp/diff_new_pack.5ZMKh2/_old 2020-06-07 21:38:56.017567126 +0200 +++ /var/tmp/diff_new_pack.5ZMKh2/_new 2020-06-07 21:38:56.017567126 +0200 @@ -17,7 +17,7 @@ Name: youtube-dl -Version: 2020.05.29 +Version: 2020.06.06 Release: 0 Summary: A tool for downloading from video sites for offline watching License: SUSE-Public-Domain AND CC-BY-SA-3.0 ++++++ youtube-dl-2020.05.29.tar.gz -> youtube-dl-2020.06.06.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/ChangeLog new/youtube-dl/ChangeLog --- old/youtube-dl/ChangeLog 2020-05-28 22:33:09.000000000 +0200 +++ new/youtube-dl/ChangeLog 2020-06-05 20:51:34.000000000 +0200 @@ -1,3 +1,19 @@ +version 2020.06.06 + +Extractors +* [tele5] Bypass geo restriction ++ [jwplatform] Add support for bypass geo restriction +* [tele5] Prefer jwplatform over nexx (#25533) +* [twitch:stream] Expect 400 and 410 HTTP errors from API +* [twitch:stream] Fix extraction (#25528) +* [twitch] Fix thumbnails extraction (#25531) ++ [twitch] Pass v5 Accept HTTP header (#25531) +* [brightcove] Fix subtitles extraction (#25540) ++ [malltv] Add support for sk.mall.tv (#25445) +* [periscope] Fix untitled broadcasts (#25482) +* [jwplatform] Improve embeds extraction (#25467) + + version 2020.05.29 Core Binary files old/youtube-dl/youtube-dl and new/youtube-dl/youtube-dl differ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/brightcove.py new/youtube-dl/youtube_dl/extractor/brightcove.py --- old/youtube-dl/youtube_dl/extractor/brightcove.py 2020-05-27 14:19:55.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/brightcove.py 2020-06-01 23:04:34.000000000 +0200 @@ -5,32 +5,34 @@ import re import struct -from .common import InfoExtractor from .adobepass import AdobePassIE +from .common import InfoExtractor from ..compat import ( compat_etree_fromstring, + compat_HTTPError, compat_parse_qs, compat_urllib_parse_urlparse, compat_urlparse, compat_xml_parse_error, - compat_HTTPError, ) from ..utils import ( - ExtractorError, + clean_html, extract_attributes, + ExtractorError, find_xpath_attr, fix_xml_ampersands, float_or_none, - js_to_json, int_or_none, + js_to_json, + mimetype2ext, parse_iso8601, smuggle_url, + str_or_none, unescapeHTML, unsmuggle_url, - update_url_query, - clean_html, - mimetype2ext, UnsupportedError, + update_url_query, + url_or_none, ) @@ -553,10 +555,16 @@ subtitles = {} for text_track in json_data.get('text_tracks', []): - if text_track.get('src'): - subtitles.setdefault(text_track.get('srclang'), []).append({ - 'url': text_track['src'], - }) + if text_track.get('kind') != 'captions': + continue + text_track_url = url_or_none(text_track.get('src')) + if not text_track_url: + continue + lang = (str_or_none(text_track.get('srclang')) + or str_or_none(text_track.get('label')) or 'en').lower() + subtitles.setdefault(lang, []).append({ + 'url': text_track_url, + }) is_live = False duration = float_or_none(json_data.get('duration'), 1000) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/jwplatform.py new/youtube-dl/youtube_dl/extractor/jwplatform.py --- old/youtube-dl/youtube_dl/extractor/jwplatform.py 2020-05-27 14:19:55.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/jwplatform.py 2020-06-01 23:04:34.000000000 +0200 @@ -4,6 +4,7 @@ import re from .common import InfoExtractor +from ..utils import unsmuggle_url class JWPlatformIE(InfoExtractor): @@ -32,10 +33,14 @@ @staticmethod def _extract_urls(webpage): return re.findall( - r'<(?:script|iframe)[^>]+?src=["\']((?:https?:)?//content\.jwplatform\.com/players/[a-zA-Z0-9]{8})', + r'<(?:script|iframe)[^>]+?src=["\']((?:https?:)?//(?:content\.jwplatform|cdn\.jwplayer)\.com/players/[a-zA-Z0-9]{8})', webpage) def _real_extract(self, url): + url, smuggled_data = unsmuggle_url(url, {}) + self._initialize_geo_bypass({ + 'countries': smuggled_data.get('geo_countries'), + }) video_id = self._match_id(url) json_data = self._download_json('https://cdn.jwplayer.com/v2/media/' + video_id, video_id) return self._parse_jwplayer_data(json_data, video_id) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/malltv.py new/youtube-dl/youtube_dl/extractor/malltv.py --- old/youtube-dl/youtube_dl/extractor/malltv.py 2020-05-27 14:19:55.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/malltv.py 2020-06-01 23:04:34.000000000 +0200 @@ -8,7 +8,7 @@ class MallTVIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?mall\.tv/(?:[^/]+/)*(?P<id>[^/?#&]+)' + _VALID_URL = r'https?://(?:(?:www|sk)\.)?mall\.tv/(?:[^/]+/)*(?P<id>[^/?#&]+)' _TESTS = [{ 'url': 'https://www.mall.tv/18-miliard-pro-neziskovky-opravdu-jsou-sportovci-nebo-clovek-v-tisni-pijavice', 'md5': '1c4a37f080e1f3023103a7b43458e518', @@ -26,6 +26,9 @@ }, { 'url': 'https://www.mall.tv/kdo-to-plati/18-miliard-pro-neziskovky-opravdu-jsou-sportovci-nebo-clovek-v-tisni-pijavice', 'only_matching': True, + }, { + 'url': 'https://sk.mall.tv/gejmhaus/reklamacia-nehreje-vyrobnik-tepla-alebo-spekacka', + 'only_matching': True, }] def _real_extract(self, url): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/periscope.py new/youtube-dl/youtube_dl/extractor/periscope.py --- old/youtube-dl/youtube_dl/extractor/periscope.py 2020-05-27 14:19:55.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/periscope.py 2020-06-01 23:04:34.000000000 +0200 @@ -18,7 +18,7 @@ item_id, query=query) def _parse_broadcast_data(self, broadcast, video_id): - title = broadcast['status'] + title = broadcast.get('status') or 'Periscope Broadcast' uploader = broadcast.get('user_display_name') or broadcast.get('username') title = '%s - %s' % (uploader, title) if uploader else title is_live = broadcast.get('state').lower() == 'running' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/tele5.py new/youtube-dl/youtube_dl/extractor/tele5.py --- old/youtube-dl/youtube_dl/extractor/tele5.py 2020-05-27 14:19:55.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/tele5.py 2020-06-01 23:04:34.000000000 +0200 @@ -6,18 +6,16 @@ from .common import InfoExtractor from .jwplatform import JWPlatformIE from .nexx import NexxIE -from ..compat import ( - compat_str, - compat_urlparse, -) +from ..compat import compat_urlparse from ..utils import ( NO_DEFAULT, - try_get, + smuggle_url, ) class Tele5IE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?tele5\.de/(?:[^/]+/)*(?P<id>[^/?#&]+)' + _GEO_COUNTRIES = ['DE'] _TESTS = [{ 'url': 'https://www.tele5.de/mediathek/filme-online/videos?vid=1549416', 'info_dict': { @@ -31,6 +29,21 @@ 'skip_download': True, }, }, { + # jwplatform, nexx unavailable + 'url': 'https://www.tele5.de/filme/ghoul-das-geheimnis-des-friedhofmonsters/', + 'info_dict': { + 'id': 'WJuiOlUp', + 'ext': 'mp4', + 'upload_date': '20200603', + 'timestamp': 1591214400, + 'title': 'Ghoul - Das Geheimnis des Friedhofmonsters', + 'description': 'md5:42002af1d887ff3d5b2b3ca1f8137d97', + }, + 'params': { + 'skip_download': True, + }, + 'add_ie': [JWPlatformIE.ie_key()], + }, { 'url': 'https://www.tele5.de/kalkofes-mattscheibe/video-clips/politik-und-gesellschaft?ve_id=1551191', 'only_matching': True, }, { @@ -88,15 +101,8 @@ if not jwplatform_id: jwplatform_id = extract_id(JWPLATFORM_ID_RE, 'jwplatform id') - media = self._download_json( - 'https://cdn.jwplayer.com/v2/media/' + jwplatform_id, - display_id) - nexx_id = try_get( - media, lambda x: x['playlist'][0]['nexx_id'], compat_str) - - if nexx_id: - return nexx_result(nexx_id) - return self.url_result( - 'jwplatform:%s' % jwplatform_id, ie=JWPlatformIE.ie_key(), - video_id=jwplatform_id) + smuggle_url( + 'jwplatform:%s' % jwplatform_id, + {'geo_countries': self._GEO_COUNTRIES}), + ie=JWPlatformIE.ie_key(), video_id=jwplatform_id) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/twitch.py new/youtube-dl/youtube_dl/extractor/twitch.py --- old/youtube-dl/youtube_dl/extractor/twitch.py 2020-05-27 14:19:55.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/twitch.py 2020-06-01 23:04:34.000000000 +0200 @@ -21,6 +21,8 @@ orderedSet, parse_duration, parse_iso8601, + qualities, + str_or_none, try_get, unified_timestamp, update_url_query, @@ -50,8 +52,14 @@ def _call_api(self, path, item_id, *args, **kwargs): headers = kwargs.get('headers', {}).copy() - headers['Client-ID'] = self._CLIENT_ID - kwargs['headers'] = headers + headers.update({ + 'Accept': 'application/vnd.twitchtv.v5+json; charset=UTF-8', + 'Client-ID': self._CLIENT_ID, + }) + kwargs.update({ + 'headers': headers, + 'expected_status': (400, 410), + }) response = self._download_json( '%s/%s' % (self._API_BASE, path), item_id, *args, **compat_kwargs(kwargs)) @@ -186,12 +194,27 @@ is_live = False else: is_live = None + _QUALITIES = ('small', 'medium', 'large') + quality_key = qualities(_QUALITIES) + thumbnails = [] + preview = info.get('preview') + if isinstance(preview, dict): + for thumbnail_id, thumbnail_url in preview.items(): + thumbnail_url = url_or_none(thumbnail_url) + if not thumbnail_url: + continue + if thumbnail_id not in _QUALITIES: + continue + thumbnails.append({ + 'url': thumbnail_url, + 'preference': quality_key(thumbnail_id), + }) return { 'id': info['_id'], 'title': info.get('title') or 'Untitled Broadcast', 'description': info.get('description'), 'duration': int_or_none(info.get('length')), - 'thumbnail': info.get('preview'), + 'thumbnails': thumbnails, 'uploader': info.get('channel', {}).get('display_name'), 'uploader_id': info.get('channel', {}).get('name'), 'timestamp': parse_iso8601(info.get('recorded_at')), @@ -572,10 +595,18 @@ else super(TwitchStreamIE, cls).suitable(url)) def _real_extract(self, url): - channel_id = self._match_id(url) + channel_name = self._match_id(url) + + access_token = self._call_api( + 'api/channels/%s/access_token' % channel_name, channel_name, + 'Downloading access token JSON') + + token = access_token['token'] + channel_id = compat_str(self._parse_json( + token, channel_name)['channel_id']) stream = self._call_api( - 'kraken/streams/%s?stream_type=all' % channel_id.lower(), + 'kraken/streams/%s?stream_type=all' % channel_id, channel_id, 'Downloading stream JSON').get('stream') if not stream: @@ -585,11 +616,9 @@ # (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing # an invalid m3u8 URL. Working around by use of original channel name from stream # JSON and fallback to lowercase if it's not available. - channel_id = stream.get('channel', {}).get('name') or channel_id.lower() - - access_token = self._call_api( - 'api/channels/%s/access_token' % channel_id, channel_id, - 'Downloading channel access token') + channel_name = try_get( + stream, lambda x: x['channel']['name'], + compat_str) or channel_name.lower() query = { 'allow_source': 'true', @@ -600,11 +629,11 @@ 'playlist_include_framerate': 'true', 'segment_preference': '4', 'sig': access_token['sig'].encode('utf-8'), - 'token': access_token['token'].encode('utf-8'), + 'token': token.encode('utf-8'), } formats = self._extract_m3u8_formats( '%s/api/channel/hls/%s.m3u8?%s' - % (self._USHER_BASE, channel_id, compat_urllib_parse_urlencode(query)), + % (self._USHER_BASE, channel_name, compat_urllib_parse_urlencode(query)), channel_id, 'mp4') self._prefer_source(formats) @@ -627,8 +656,8 @@ }) return { - 'id': compat_str(stream['_id']), - 'display_id': channel_id, + 'id': str_or_none(stream.get('_id')) or channel_id, + 'display_id': channel_name, 'title': title, 'description': description, 'thumbnails': thumbnails, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/extractor/twitter.py new/youtube-dl/youtube_dl/extractor/twitter.py --- old/youtube-dl/youtube_dl/extractor/twitter.py 2020-05-27 14:19:55.000000000 +0200 +++ new/youtube-dl/youtube_dl/extractor/twitter.py 2020-06-01 23:04:34.000000000 +0200 @@ -578,6 +578,18 @@ IE_NAME = 'twitter:broadcast' _VALID_URL = TwitterBaseIE._BASE_REGEX + r'i/broadcasts/(?P<id>[0-9a-zA-Z]{13})' + _TEST = { + # untitled Periscope video + 'url': 'https://twitter.com/i/broadcasts/1yNGaQLWpejGj', + 'info_dict': { + 'id': '1yNGaQLWpejGj', + 'ext': 'mp4', + 'title': 'Andrea May Sahouri - Periscope Broadcast', + 'uploader': 'Andrea May Sahouri', + 'uploader_id': '1PXEdBZWpGwKe', + }, + } + def _real_extract(self, url): broadcast_id = self._match_id(url) broadcast = self._call_api( diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/youtube-dl/youtube_dl/version.py new/youtube-dl/youtube_dl/version.py --- old/youtube-dl/youtube_dl/version.py 2020-05-28 22:33:09.000000000 +0200 +++ new/youtube-dl/youtube_dl/version.py 2020-06-05 20:51:34.000000000 +0200 @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2020.05.29' +__version__ = '2020.06.06'