[ie/apple] Rework extractors (#17277)

* Add API fallback for ApplePodcastsIE
* Fix AppleConnectIE token extraction
* Refactor and add AppleBaseIE

Closes #17266
Authored by: bashonly

yt_dlp/extractor/appleconnect.py

@@ -1,39 +1,22 @@
-import time
+from .applepodcasts import AppleBaseIE
-
-from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
-    extract_attributes,
     float_or_none,
-    jwt_decode_hs256,
-    jwt_encode,
     parse_resolution,
     qualities,
     unified_strdate,
     update_url,
     url_or_none,
-    urljoin,
 )
 from ..utils.traversal import (
-    find_element,
     require,
     traverse_obj,
 )
 
 
-class AppleConnectIE(InfoExtractor):
+class AppleConnectIE(AppleBaseIE):
     IE_NAME = 'apple:music:connect'
     IE_DESC = 'Apple Music Connect'
 
-    _BASE_URL = 'https://music.apple.com'
-    _QUALITIES = {
-        'provisionalUploadVideo': None,
-        'sdVideo': 480,
-        'sdVideoWithPlusAudio': 480,
-        'sd480pVideo': 480,
-        '720pHdVideo': 720,
-        '1080pHdVideo': 1080,
-    }
     _VALID_URL = r'https?://music\.apple\.com/[\w-]+/post/(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://music.apple.com/us/post/1018290019',
@@ -59,29 +42,16 @@ class AppleConnectIE(InfoExtractor):
         },
     }]
 
-    _jwt = None
+    _BASE_URL = 'https://music.apple.com'
-
+    _JWT_KEY_ID = 'WebPlayKid'
-    @staticmethod
+    _QUALITIES = {
-    def _jwt_is_expired(token):
+        'provisionalUploadVideo': None,
-        return jwt_decode_hs256(token)['exp'] - time.time() < 120
+        'sdVideo': 480,
-
+        'sdVideoWithPlusAudio': 480,
-    def _get_token(self, webpage, video_id):
+        'sd480pVideo': 480,
-        if self._jwt and not self._jwt_is_expired(self._jwt):
+        '720pHdVideo': 720,
-            return self._jwt
+        '1080pHdVideo': 1080,
-
+    }
-        js_url = traverse_obj(webpage, (
-            {find_element(tag='script', attr='crossorigin', value='', html=True)},
-            {extract_attributes}, 'src', {urljoin(self._BASE_URL)}, {require('JS URL')}))
-        js = self._download_webpage(
-            js_url, video_id, 'Downloading token JS', 'Unable to download token JS')
-
-        header = jwt_encode({}, '', headers={'alg': 'ES256', 'kid': 'WebPlayKid'}).split('.')[0]
-        self._jwt = self._search_regex(
-            fr'(["\'])(?P<jwt>{header}(?:\.[\w-]+){{2}})\1', js, 'JSON Web Token', group='jwt')
-        if self._jwt_is_expired(self._jwt):
-            raise ExtractorError('The fetched token is already expired')
-
-        return self._jwt
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

yt_dlp/extractor/applepodcasts.py

@@ -1,15 +1,55 @@
+import time
+
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     clean_html,
     clean_podcast_url,
     int_or_none,
+    jwt_decode_hs256,
+    jwt_encode,
     parse_iso8601,
+    try_call,
+    update_url,
+    url_or_none,
+    urljoin,
 )
 from ..utils.traversal import traverse_obj
 
 
-class ApplePodcastsIE(InfoExtractor):
+class AppleBaseIE(InfoExtractor):
-    _VALID_URL = r'https?://podcasts\.apple\.com/(?:[^/]+/)?podcast(?:/[^/]+){1,2}.*?\bi=(?P<id>\d+)'
+    """Subclasses must set _BASE_URL and _JWT_KEY_ID"""
+
+    _jwt_cache = {}
+
+    @staticmethod
+    def _jwt_is_expired(token):
+        return jwt_decode_hs256(token)['exp'] - time.time() < 120
+
+    def _get_token(self, webpage, episode_id):
+        if self._jwt_cache.get(self._BASE_URL) and not self._jwt_is_expired(self._jwt_cache[self._BASE_URL]):
+            return self._jwt
+
+        js_path = self._search_regex(
+            r'<script [^>]*\bsrc="(/assets/index~[0-9a-f]+\.js)">', webpage, 'JS asset path')
+        js_code = self._download_webpage(
+            urljoin(self._BASE_URL, js_path), episode_id,
+            'Downloading JS asset', 'Unable to download JS asset')
+
+        header = jwt_encode({}, '', headers={'typ': 'JWT', 'alg': 'ES256', 'kid': self._JWT_KEY_ID}).split('.')[0]
+        self._jwt_cache[self._BASE_URL] = self._search_regex(
+            fr'(["\'])(?P<jwt>{header}(?:\.[\w-]+){{2}})\1', js_code, 'JSON Web Token', group='jwt')
+        if self._jwt_is_expired(self._jwt_cache[self._BASE_URL]):
+            raise ExtractorError('The fetched token is already expired')
+
+        return self._jwt_cache[self._BASE_URL]
+
+
+class ApplePodcastsIE(AppleBaseIE):
+    IE_NAME = 'apple:podcasts'
+    IE_DESC = 'Apple Podcasts'
+
+    _VALID_URL = r'https?://podcasts\.apple\.com/(?P<country>[^/?#]+/)?podcast(?:/[^/?#]+){1,2}/?\?(?:[^#]+&)?i=(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://podcasts.apple.com/us/podcast/urbana-podcast-724-by-david-penn/id1531349107?i=1000748574256',
         'md5': 'f8a6f92735d0cfbd5e6a7294151e28d8',
@@ -23,7 +63,7 @@ class ApplePodcastsIE(InfoExtractor):
             'timestamp': 1770400801,
             'duration': 3602,
             'series': 'Urbana Radio Show',
-            'thumbnail': 're:.+[.](png|jpe?g|webp)',
+            'thumbnail': r're:https://.+/.+\.jpg',
         },
     }, {
         'url': 'https://podcasts.apple.com/us/podcast/207-whitney-webb-returns/id1135137367?i=1000482637777',
@@ -39,7 +79,7 @@ class ApplePodcastsIE(InfoExtractor):
             'timestamp': 1593932400,
             'duration': 5369,
             'series': 'The Tim Dillon Show',
-            'thumbnail': 're:.+[.](png|jpe?g|webp)',
+            'thumbnail': r're:https://.+/.+\.jpg',
         },
     }, {
         'url': 'https://podcasts.apple.com/podcast/207-whitney-webb-returns/id1135137367?i=1000482637777',
@@ -52,15 +92,55 @@ class ApplePodcastsIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    def _real_extract(self, url):
+    _BASE_URL = 'https://podcasts.apple.com'
-        episode_id = self._match_id(url)
+    _JWT_KEY_ID = 'C4J7GBP74H'
-        webpage = self._download_webpage(url, episode_id)
+
+    def _extract_podcast_from_api(self, webpage, episode_id, country_code):
+        data = self._download_json(
+            f'https://amp-api.podcasts.apple.com/v1/catalog/{country_code or "us"}/podcast-episodes/{episode_id}',
+            episode_id, headers={
+                'Authorization': f'Bearer {self._get_token(webpage, episode_id)}',
+                'Origin': self._BASE_URL,
+            },
+            query={
+                # XXX: if video is available, try adding the params 'with=entitlements,hlsVideo'
+                'extend': 'fullDescription',
+                'include': 'podcast',
+                'l': 'en-US',
+            })['data'][0]
+
+        thumb_info = traverse_obj(data, ('attributes', 'artwork', {
+            'url': ('url', {url_or_none}),
+            'h': ('height', {int_or_none}),
+            'w': ('width', {int_or_none}),
+        }))
+
+        return {
+            'id': episode_id,
+            **traverse_obj(data, {
+                'title': ('attributes', 'name', {str}),
+                'description': ('attributes', 'fullDescription', {clean_html}),
+                'url': ('attributes', 'assetUrl', {clean_podcast_url}, {update_url(scheme='https')}),
+                'timestamp': ('attributes', 'releaseDateTime', {parse_iso8601}),
+                'duration': ('attributes', 'durationInMilliseconds', {int_or_none(scale=1000)}),
+                'episode': ('attributes', 'name', {str}),
+                'episode_number': ('attributes', 'episodeNumber', {int_or_none}),
+                'series': ('relationships', 'podcast', 'data', 0, 'attributes', 'name', {str}),
+            }),
+            'thumbnail': try_call(lambda: thumb_info.pop('url').format(f='jpg', **thumb_info)),
+            'vcodec': 'none',
+        }
+
+    def _extract_podcast_from_webpage(self, webpage, episode_id):
         server_data = self._search_json(
             r'<script [^>]*\bid=["\']serialized-server-data["\'][^>]*>', webpage,
-            'server data', episode_id)['data'][0]['data']
+            'server data', episode_id, default=None)
         model_data = traverse_obj(server_data, (
-            'headerButtonItems', lambda _, v: v['$kind'] == 'share' and v['modelType'] == 'EpisodeLockup',
+            'data', 0, 'data', 'headerButtonItems',
+            lambda _, v: v['$kind'] == 'share' and v['modelType'] == 'EpisodeLockup',
             'model', {dict}, any))
+        if not model_data:
+            return None
 
         return {
             'id': episode_id,
@@ -77,3 +157,12 @@ class ApplePodcastsIE(InfoExtractor):
             'thumbnail': self._og_search_thumbnail(webpage),
             'vcodec': 'none',
         }
+
+    def _real_extract(self, url):
+        episode_id, country_code = self._match_valid_url(url).group('id', 'country')
+        # Webpage may be unavailable, see https://github.com/yt-dlp/yt-dlp/issues/17266
+        webpage = self._download_webpage(url, episode_id, expected_status=500)
+
+        return (
+            self._extract_podcast_from_webpage(webpage, episode_id)
+            or self._extract_podcast_from_api(webpage, episode_id, country_code))

yt_dlp/extractor/instagram.py

@@ -42,7 +42,6 @@ def _id_to_pk(shortcode):
 
 
 class InstagramBaseIE(InfoExtractor):
-    _API_BASE_URL = 'https://i.instagram.com/api/v1'
     _BASE_URL = 'https://www.instagram.com/'
     _APP_IDS = {
         'ios': '124024574287414',
@@ -75,6 +74,12 @@ class InstagramBaseIE(InfoExtractor):
     def _is_web_app(self):
         return self._app_id == self._APP_IDS['web']
 
+    @property
+    def _API_BASE_URL(self):
+        if not self._is_web_app:
+            return 'https://i.instagram.com/api/v1'
+        return 'https://www.instagram.com/api/v1'
+
     @property
     def _api_headers(self):
         return {
@@ -87,7 +92,8 @@ class InstagramBaseIE(InfoExtractor):
 
     @staticmethod
     def _is_login_redirect(url):
-        return urllib.parse.urlparse(url).path.startswith('/accounts/login')
+        path = urllib.parse.urlparse(url).path
+        return path.startswith('/accounts/login') or path == '/'
 
     def _get_count(self, media, kind, *keys):
         return traverse_obj(