[ie/apple] Rework extractors (#17277)

* Add API fallback for ApplePodcastsIE
* Fix AppleConnectIE token extraction
* Refactor and add AppleBaseIE

Closes #17266
Authored by: bashonly

README.md

@@ -1859,7 +1859,7 @@ The following extractors use this feature:
 #### youtube
 * `lang`: Prefer translated metadata (`title`, `description` etc) of this language code (case-sensitive). By default, the video primary language metadata is preferred, with a fallback to `en` translated. See [youtube/_base.py](https://github.com/yt-dlp/yt-dlp/blob/415b4c9f955b1a0391204bd24a7132590e7b3bdb/yt_dlp/extractor/youtube/_base.py#L402-L409) for the list of supported content language codes
 * `skip`: One or more of `hls`, `dash` or `translated_subs` to skip extraction of the m3u8 manifests, dash manifests and [auto-translated subtitles](https://github.com/yt-dlp/yt-dlp/issues/4090#issuecomment-1158102032) respectively
-* `player_client`: Clients to extract video data from. The currently available clients are `web`, `web_safari`, `web_embedded`, `web_music`, `web_creator`, `mweb`, `ios`, `visionos`, `android`, `android_vr`, `tv`, `tv_downgraded`, and `tv_simply`. By default, `android_vr,web_safari` is used. If no JavaScript runtime/engine is available, then only `android_vr` is used. If logged-in cookies are passed to yt-dlp, then `tv_downgraded,web_safari` is used for free accounts and `tv_downgraded,web_creator` is used for premium accounts. The `web_music` client is added for `music.youtube.com` URLs when logged-in cookies are used. The `web_embedded` client is added for age-restricted videos but only successfully works around the age-restriction sometimes (e.g. if the video is embeddable), and may be added as a fallback if `android_vr` or `visionos` is unable to access a video. The `web_creator` client is added for age-restricted videos if account age-verification is required. Some clients, such as `web_creator` and `web_music`, require a `po_token` for their formats to be downloadable. Some clients, such as `web_creator`, will only work with authentication. Not all clients support authentication via cookies. You can use `default` for the default clients, or you can use `all` for all clients (not recommended). You can prefix a client with `-` to exclude it, e.g. `youtube:player_client=default,-web_safari`
+* `player_client`: Clients to extract video data from. The currently available clients are `web`, `web_safari`, `web_embedded`, `web_music`, `web_creator`, `mweb`, `ios`, `visionos`, `android`, `android_vr`, `tv`, `tv_downgraded`, and `tv_simply`. By default, `visionos,android_vr,web` is used. If no JavaScript runtime/engine is available, then `web` is omitted. If logged-in cookies are passed to yt-dlp, then `tv_downgraded,web` is used for free accounts and `tv_downgraded,web_creator,web` is used for premium accounts. The `web_music` client is added for `music.youtube.com` URLs when logged-in cookies are used. The `web_embedded` client is added for age-restricted videos but only successfully works around the age-restriction sometimes (e.g. if the video is embeddable). The `tv_downgraded` client may be added as a fallback if `android_vr` or `visionos` is unable to access a video. The `web_creator` client is added for age-restricted videos if account age-verification is required. Some clients, such as `web_creator` and `web_music`, require a `po_token` for their formats to be downloadable. Some clients, such as `web_creator`, will only work with authentication. Not all clients support authentication via cookies. You can use `default` for the default clients, or you can use `all` for all clients (not recommended). You can prefix a client with `-` to exclude it, e.g. `youtube:player_client=default,-web`
 * `player_skip`: Skip some network requests that are generally needed for robust extraction. One or more of `configs` (skip client configs), `webpage` (skip initial webpage), `js` (skip js player), `initial_data` (skip initial data/next ep request). While these options can help reduce the number of requests needed or avoid some rate-limiting, they could cause issues such as missing formats or metadata.  See [#860](https://github.com/yt-dlp/yt-dlp/pull/860) and [#12826](https://github.com/yt-dlp/yt-dlp/issues/12826) for more details
 * `webpage_skip`: Skip extraction of embedded webpage data. One or both of `player_response`, `initial_data`. These options are for testing purposes and don't skip any network requests. Neither is skipped by default; however, if a `player_js_version` value other than `actual` is used, then `webpage_skip=player_response` is implied
 * `webpage_client`: Client to use for the video webpage request. One of `web` or `web_safari` (default)

yt_dlp/extractor/appleconnect.py

@@ -1,39 +1,22 @@
-import time
-
-from .common import InfoExtractor
+from .applepodcasts import AppleBaseIE
 from ..utils import (
-    ExtractorError,
-    extract_attributes,
     float_or_none,
-    jwt_decode_hs256,
-    jwt_encode,
     parse_resolution,
     qualities,
     unified_strdate,
     update_url,
     url_or_none,
-    urljoin,
 )
 from ..utils.traversal import (
-    find_element,
     require,
     traverse_obj,
 )
 
 
-class AppleConnectIE(InfoExtractor):
+class AppleConnectIE(AppleBaseIE):
     IE_NAME = 'apple:music:connect'
     IE_DESC = 'Apple Music Connect'
 
-    _BASE_URL = 'https://music.apple.com'
-    _QUALITIES = {
-        'provisionalUploadVideo': None,
-        'sdVideo': 480,
-        'sdVideoWithPlusAudio': 480,
-        'sd480pVideo': 480,
-        '720pHdVideo': 720,
-        '1080pHdVideo': 1080,
-    }
     _VALID_URL = r'https?://music\.apple\.com/[\w-]+/post/(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://music.apple.com/us/post/1018290019',
@@ -59,29 +42,16 @@ class AppleConnectIE(InfoExtractor):
         },
     }]
 
-    _jwt = None
-
-    @staticmethod
-    def _jwt_is_expired(token):
-        return jwt_decode_hs256(token)['exp'] - time.time() < 120
-
-    def _get_token(self, webpage, video_id):
-        if self._jwt and not self._jwt_is_expired(self._jwt):
-            return self._jwt
-
-        js_url = traverse_obj(webpage, (
-            {find_element(tag='script', attr='crossorigin', value='', html=True)},
-            {extract_attributes}, 'src', {urljoin(self._BASE_URL)}, {require('JS URL')}))
-        js = self._download_webpage(
-            js_url, video_id, 'Downloading token JS', 'Unable to download token JS')
-
-        header = jwt_encode({}, '', headers={'alg': 'ES256', 'kid': 'WebPlayKid'}).split('.')[0]
-        self._jwt = self._search_regex(
-            fr'(["\'])(?P<jwt>{header}(?:\.[\w-]+){{2}})\1', js, 'JSON Web Token', group='jwt')
-        if self._jwt_is_expired(self._jwt):
-            raise ExtractorError('The fetched token is already expired')
-
-        return self._jwt
+    _BASE_URL = 'https://music.apple.com'
+    _JWT_KEY_ID = 'WebPlayKid'
+    _QUALITIES = {
+        'provisionalUploadVideo': None,
+        'sdVideo': 480,
+        'sdVideoWithPlusAudio': 480,
+        'sd480pVideo': 480,
+        '720pHdVideo': 720,
+        '1080pHdVideo': 1080,
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

yt_dlp/extractor/applepodcasts.py

@@ -1,15 +1,55 @@
+import time
+
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     clean_html,
     clean_podcast_url,
     int_or_none,
+    jwt_decode_hs256,
+    jwt_encode,
     parse_iso8601,
+    try_call,
+    update_url,
+    url_or_none,
+    urljoin,
 )
 from ..utils.traversal import traverse_obj
 
 
-class ApplePodcastsIE(InfoExtractor):
-    _VALID_URL = r'https?://podcasts\.apple\.com/(?:[^/]+/)?podcast(?:/[^/]+){1,2}.*?\bi=(?P<id>\d+)'
+class AppleBaseIE(InfoExtractor):
+    """Subclasses must set _BASE_URL and _JWT_KEY_ID"""
+
+    _jwt_cache = {}
+
+    @staticmethod
+    def _jwt_is_expired(token):
+        return jwt_decode_hs256(token)['exp'] - time.time() < 120
+
+    def _get_token(self, webpage, episode_id):
+        if self._jwt_cache.get(self._BASE_URL) and not self._jwt_is_expired(self._jwt_cache[self._BASE_URL]):
+            return self._jwt
+
+        js_path = self._search_regex(
+            r'<script [^>]*\bsrc="(/assets/index~[0-9a-f]+\.js)">', webpage, 'JS asset path')
+        js_code = self._download_webpage(
+            urljoin(self._BASE_URL, js_path), episode_id,
+            'Downloading JS asset', 'Unable to download JS asset')
+
+        header = jwt_encode({}, '', headers={'typ': 'JWT', 'alg': 'ES256', 'kid': self._JWT_KEY_ID}).split('.')[0]
+        self._jwt_cache[self._BASE_URL] = self._search_regex(
+            fr'(["\'])(?P<jwt>{header}(?:\.[\w-]+){{2}})\1', js_code, 'JSON Web Token', group='jwt')
+        if self._jwt_is_expired(self._jwt_cache[self._BASE_URL]):
+            raise ExtractorError('The fetched token is already expired')
+
+        return self._jwt_cache[self._BASE_URL]
+
+
+class ApplePodcastsIE(AppleBaseIE):
+    IE_NAME = 'apple:podcasts'
+    IE_DESC = 'Apple Podcasts'
+
+    _VALID_URL = r'https?://podcasts\.apple\.com/(?P<country>[^/?#]+/)?podcast(?:/[^/?#]+){1,2}/?\?(?:[^#]+&)?i=(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://podcasts.apple.com/us/podcast/urbana-podcast-724-by-david-penn/id1531349107?i=1000748574256',
         'md5': 'f8a6f92735d0cfbd5e6a7294151e28d8',
@@ -23,7 +63,7 @@ class ApplePodcastsIE(InfoExtractor):
             'timestamp': 1770400801,
             'duration': 3602,
             'series': 'Urbana Radio Show',
-            'thumbnail': 're:.+[.](png|jpe?g|webp)',
+            'thumbnail': r're:https://.+/.+\.jpg',
         },
     }, {
         'url': 'https://podcasts.apple.com/us/podcast/207-whitney-webb-returns/id1135137367?i=1000482637777',
@@ -39,7 +79,7 @@ class ApplePodcastsIE(InfoExtractor):
             'timestamp': 1593932400,
             'duration': 5369,
             'series': 'The Tim Dillon Show',
-            'thumbnail': 're:.+[.](png|jpe?g|webp)',
+            'thumbnail': r're:https://.+/.+\.jpg',
         },
     }, {
         'url': 'https://podcasts.apple.com/podcast/207-whitney-webb-returns/id1135137367?i=1000482637777',
@@ -52,15 +92,55 @@ class ApplePodcastsIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    def _real_extract(self, url):
-        episode_id = self._match_id(url)
-        webpage = self._download_webpage(url, episode_id)
+    _BASE_URL = 'https://podcasts.apple.com'
+    _JWT_KEY_ID = 'C4J7GBP74H'
+
+    def _extract_podcast_from_api(self, webpage, episode_id, country_code):
+        data = self._download_json(
+            f'https://amp-api.podcasts.apple.com/v1/catalog/{country_code or "us"}/podcast-episodes/{episode_id}',
+            episode_id, headers={
+                'Authorization': f'Bearer {self._get_token(webpage, episode_id)}',
+                'Origin': self._BASE_URL,
+            },
+            query={
+                # XXX: if video is available, try adding the params 'with=entitlements,hlsVideo'
+                'extend': 'fullDescription',
+                'include': 'podcast',
+                'l': 'en-US',
+            })['data'][0]
+
+        thumb_info = traverse_obj(data, ('attributes', 'artwork', {
+            'url': ('url', {url_or_none}),
+            'h': ('height', {int_or_none}),
+            'w': ('width', {int_or_none}),
+        }))
+
+        return {
+            'id': episode_id,
+            **traverse_obj(data, {
+                'title': ('attributes', 'name', {str}),
+                'description': ('attributes', 'fullDescription', {clean_html}),
+                'url': ('attributes', 'assetUrl', {clean_podcast_url}, {update_url(scheme='https')}),
+                'timestamp': ('attributes', 'releaseDateTime', {parse_iso8601}),
+                'duration': ('attributes', 'durationInMilliseconds', {int_or_none(scale=1000)}),
+                'episode': ('attributes', 'name', {str}),
+                'episode_number': ('attributes', 'episodeNumber', {int_or_none}),
+                'series': ('relationships', 'podcast', 'data', 0, 'attributes', 'name', {str}),
+            }),
+            'thumbnail': try_call(lambda: thumb_info.pop('url').format(f='jpg', **thumb_info)),
+            'vcodec': 'none',
+        }
+
+    def _extract_podcast_from_webpage(self, webpage, episode_id):
         server_data = self._search_json(
             r'<script [^>]*\bid=["\']serialized-server-data["\'][^>]*>', webpage,
-            'server data', episode_id)['data'][0]['data']
+            'server data', episode_id, default=None)
         model_data = traverse_obj(server_data, (
-            'headerButtonItems', lambda _, v: v['$kind'] == 'share' and v['modelType'] == 'EpisodeLockup',
+            'data', 0, 'data', 'headerButtonItems',
+            lambda _, v: v['$kind'] == 'share' and v['modelType'] == 'EpisodeLockup',
             'model', {dict}, any))
+        if not model_data:
+            return None
 
         return {
             'id': episode_id,
@@ -77,3 +157,12 @@ class ApplePodcastsIE(InfoExtractor):
             'thumbnail': self._og_search_thumbnail(webpage),
             'vcodec': 'none',
         }
+
+    def _real_extract(self, url):
+        episode_id, country_code = self._match_valid_url(url).group('id', 'country')
+        # Webpage may be unavailable, see https://github.com/yt-dlp/yt-dlp/issues/17266
+        webpage = self._download_webpage(url, episode_id, expected_status=500)
+
+        return (
+            self._extract_podcast_from_webpage(webpage, episode_id)
+            or self._extract_podcast_from_api(webpage, episode_id, country_code))

yt_dlp/extractor/instagram.py

@@ -42,7 +42,6 @@ def _id_to_pk(shortcode):
 
 
 class InstagramBaseIE(InfoExtractor):
-    _API_BASE_URL = 'https://i.instagram.com/api/v1'
     _BASE_URL = 'https://www.instagram.com/'
     _APP_IDS = {
         'ios': '124024574287414',
@@ -75,6 +74,12 @@ class InstagramBaseIE(InfoExtractor):
     def _is_web_app(self):
         return self._app_id == self._APP_IDS['web']
 
+    @property
+    def _API_BASE_URL(self):
+        if not self._is_web_app:
+            return 'https://i.instagram.com/api/v1'
+        return 'https://www.instagram.com/api/v1'
+
     @property
     def _api_headers(self):
         return {
@@ -87,7 +92,8 @@ class InstagramBaseIE(InfoExtractor):
 
     @staticmethod
     def _is_login_redirect(url):
-        return urllib.parse.urlparse(url).path.startswith('/accounts/login')
+        path = urllib.parse.urlparse(url).path
+        return path.startswith('/accounts/login') or path == '/'
 
     def _get_count(self, media, kind, *keys):
         return traverse_obj(

yt_dlp/extractor/vimeo.py

@@ -49,7 +49,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         'Cannot download embed-only video without embedding URL. Please call yt-dlp '
         'with the URL of the page that embeds this video.')
 
-    _DEFAULT_CLIENT = 'macos'
+    _DEFAULT_CLIENT = 'web'
     _DEFAULT_AUTHED_CLIENT = 'web'
     _CLIENT_HEADERS = {
         'Accept': 'application/vnd.vimeo.*+json; version=3.4.10',
@@ -89,7 +89,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         },
         'macos': {
             'CACHE_KEY': 'oauth-token-macos',
-            'CACHE_ONLY': False,
+            'CACHE_ONLY': True,
             'VIEWER_JWT': False,
             'REQUIRES_AUTH': False,
             'AUTH': 'NDc1N2JlN2Y5ZjZmMjU3NzE3NTRkZTg1NmY2YzU2MTI0OTFlNjJiYjpwVUNDWUlBZmZqSHhQcndBYWxGMzgyYys2NkN5d1JrREJZZXdPcEdsU05tdjFlVVo2aE1lYk9GcWE3ZW9KVldlYnFlOWh5Vno5UWtpUGJ5empYZFBpYkFwV0FFTnB5VWV4ZEh3aHZnRUNEL0VySnBzTmFraDdNbS9nMXhWanhIcw==',

yt_dlp/extractor/youtube/_base.py

@@ -107,6 +107,7 @@ INNERTUBE_CLIENTS = {
         **WEB_PO_TOKEN_POLICIES,
     },
     # Safari UA returns pre-merged video+audio 144p/240p/360p/720p/1080p HLS formats
+    # Since 2026.07, HLS formats are only returned with some logged-in or "trusted" sessions
     'web_safari': {
         'INNERTUBE_CONTEXT': {
             'client': {
@@ -223,6 +224,7 @@ INNERTUBE_CLIENTS = {
     },
     # "Made for kids" videos aren't available with this client
     # Using a clientVersion>1.65 may return SABR streams only
+    # Since 2026.07, intermittent/selective POT enforcement has been observed for non-HLS formats
     'android_vr': {
         'INNERTUBE_CONTEXT': {
             'client': {
@@ -238,6 +240,24 @@ INNERTUBE_CLIENTS = {
         },
         'INNERTUBE_CONTEXT_CLIENT_NAME': 28,
         'REQUIRE_JS_PLAYER': False,
+        'GVS_PO_TOKEN_POLICY': {
+            StreamingProtocol.HTTPS: GvsPoTokenPolicy(
+                required=True,
+                recommended=True,
+                not_required_with_player_token=True,
+            ),
+            StreamingProtocol.DASH: GvsPoTokenPolicy(
+                required=True,
+                recommended=True,
+                not_required_with_player_token=True,
+            ),
+            StreamingProtocol.HLS: GvsPoTokenPolicy(
+                required=False,
+                recommended=True,
+                not_required_with_player_token=True,
+            ),
+        },
+        'PLAYER_PO_TOKEN_POLICY': PlayerPoTokenPolicy(required=False, recommended=True),
     },
     # iOS clients have HLS live streams. Setting device model to get 60fps formats.
     # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
@@ -340,7 +360,6 @@ INNERTUBE_CLIENTS = {
             },
         },
         'INNERTUBE_CONTEXT_CLIENT_NAME': 7,
-        'REQUIRE_AUTH': True,
         'SUPPORTS_COOKIES': True,
     },
     'tv_simply': {

yt_dlp/extractor/youtube/_video.py

@@ -28,6 +28,7 @@ from .jsc._director import initialize_jsc_director
 from .jsc.provider import JsChallengeRequest, JsChallengeType, NChallengeInput, SigChallengeInput
 from .pot._director import initialize_pot_director
 from .pot.provider import PoTokenContext, PoTokenRequest
+from ...networking import HEADRequest
 from ...utils import (
     NO_DEFAULT,
     ExtractorError,
@@ -139,13 +140,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     _RETURN_TYPE = 'video'  # XXX: How to handle multifeed?
 
     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'srt', 'vtt')
-    _DEFAULT_CLIENTS = ('android_vr', 'web_safari')
-    _DEFAULT_JSLESS_CLIENTS = ('android_vr',)
-    _DEFAULT_AUTHED_CLIENTS = ('tv_downgraded', 'web_safari')
+    _DEFAULT_CLIENTS = ('visionos', 'android_vr', 'web')
+    _DEFAULT_JSLESS_CLIENTS = ('visionos', 'android_vr')
+    _DEFAULT_AUTHED_CLIENTS = ('tv_downgraded', 'web')
     # Premium does not require POT (except for subtitles)
-    _DEFAULT_PREMIUM_CLIENTS = ('tv_downgraded', 'web_creator')
+    _DEFAULT_PREMIUM_CLIENTS = ('tv_downgraded', 'web_creator', 'web')
     _WEBPAGE_CLIENTS = ('web', 'web_safari')
-    _DEFAULT_WEBPAGE_CLIENT = 'web_safari'
+    _DEFAULT_WEBPAGE_CLIENT = 'web'
 
     _GEO_BYPASS = False
 
@@ -2036,8 +2037,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 last_seq = last_seq_cache[cache_key]
             else:
                 try:
-                    urlh = self._request_webpage(base_url, None, note=False, errnote=False, fatal=False)
-                except ExtractorError:
+                    urlh = self._request_webpage(
+                        HEADRequest(base_url), None,
+                        note=False, errnote='Fragment request failed')
+                except ExtractorError as e:
+                    self.write_debug(e.msg)
                     urlh = None
                 last_seq = try_get(urlh, lambda x: int_or_none(x.headers['X-Head-Seqnum']))
                 if urlh:
@@ -2900,6 +2904,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     def _is_unplayable(player_response):
         return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'UNPLAYABLE'
 
+    @staticmethod
+    def _is_error_response(player_response):
+        return traverse_obj(player_response, ('playabilityStatus', 'status')) == 'ERROR'
+
     def _extract_player_response(self, client, video_id, webpage_ytcfg, player_ytcfg, player_url, initial_pr, visitor_data, data_sync_id, po_token):
         headers = self.generate_api_headers(
             ytcfg=player_ytcfg,
@@ -3133,12 +3141,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
             if (
                 # Is this a "made for kids" video that can't be downloaded with android_vr/visionos?
-                client in {'android_vr', 'visionos'} and self._is_unplayable(pr)
+                client in {'android_vr', 'visionos'}
+                and (self._is_unplayable(pr) or self._is_error_response(pr))
                 and webpage and 'made for kids' in webpage
                 # ...and is a JS runtime is available?
                 and any(p.is_available() for p in self._jsc_director.providers.values())
             ):
-                append_client('web_embedded')
+                append_client('tv_downgraded')
 
             # web_embedded can work around age-gate and age-verification for some embeddable videos
             if self._is_agegated(pr) and variant != 'web_embedded':