diff --git a/README.md b/README.md index 648a759e..d69a705b 100644 --- a/README.md +++ b/README.md @@ -788,7 +788,7 @@ We are immensely grateful to all the wonderful people who have contributed to th ## Contact -LuanRT - [@thesciencephile][twitter] - luan.lrt4@gmail.com +LuanRT - [@thesciencephile][twitter] - luanrt@thatsciencephile.com Project Link: [https://github.com/LuanRT/YouTube.js][project] diff --git a/deno.ts b/deno.ts index 1225fd4b..660de98f 100644 --- a/deno.ts +++ b/deno.ts @@ -1,3 +1,3 @@ export * from './deno/src/platform/deno.ts'; import Innertube from './deno/src/platform/deno.ts'; -export default Innertube; \ No newline at end of file +export default Innertube; diff --git a/deno/package.json b/deno/package.json index 02343731..eb244022 100644 --- a/deno/package.json +++ b/deno/package.json @@ -1,6 +1,6 @@ { "name": "youtubei.js", - "version": "5.8.0", + "version": "6.0.0", "description": "A wrapper around YouTube's private API. Supports YouTube, YouTube Music, YouTube Kids and YouTube Studio (WIP).", "type": "module", "types": "./dist/src/platform/lib.d.ts", @@ -56,7 +56,8 @@ "Wykerd (https://github.com/wykerd/)", "MasterOfBob777 (https://github.com/MasterOfBob777)", "patrickkfkan (https://github.com/patrickkfkan)", - "akkadaska (https://github.com/akkadaska)" + "akkadaska (https://github.com/akkadaska)", + "Absidue (https://github.com/absidue)" ], "directories": { "test": "./test", @@ -68,10 +69,10 @@ "lint": "npx eslint ./src", "lint:fix": "npx eslint --fix ./src", "build": "npm run build:parser-map && npm run build:proto && npm run build:esm && npm run bundle:node && npm run bundle:browser && npm run bundle:browser:prod", - "build:parser-map": "node ./scripts/build-parser-map.cjs", + "build:parser-map": "node ./scripts/gen-parser-map.mjs", "build:proto": "npx pb-gen-ts --entry-path=\"src/proto\" --out-dir=\"src/proto/generated\" --ext-in-import=\".js\"", "build:esm": "npx tsc", - "build:deno": "npx cpy ./src ./deno && npx cpy ./package.json ./deno && npx replace \".ts';\" \".ts';\" ./deno -r && npx replace '.js\";' '.ts\";' ./deno -r && npx replace \"'https://esm.sh/linkedom';\" \"'https://esm.sh/linkedom';\" ./deno -r && npx replace \"'https://esm.sh/jintr';\" \"'https://esm.sh/jintr';\" ./deno -r && npx replace \"new Jinter\" \"new Jinter\" ./deno -r", + "build:deno": "npx cpy ./src ./deno && npx esbuild ./src/utils/DashManifest.tsx --keep-names --format=esm --platform=neutral --target=es2020 --outfile=./deno/src/utils/DashManifest.js && npx cpy ./package.json ./deno && npx replace \".ts';\" \".ts';\" ./deno -r && npx replace '.js\";' '.ts\";' ./deno -r && npx replace \"'./DashManifest.js';\" \"'./DashManifest.js';\" ./deno -r && npx replace \"'https://esm.sh/jintr';\" \"'https://esm.sh/jintr';\" ./deno -r", "bundle:node": "npx esbuild ./dist/src/platform/node.js --bundle --target=node10 --keep-names --format=cjs --platform=node --outfile=./bundle/node.cjs --external:jintr --external:undici --external:linkedom --external:tslib --sourcemap --banner:js=\"/* eslint-disable */\"", "bundle:browser": "npx esbuild ./dist/src/platform/web.js --banner:js=\"/* eslint-disable */\" --bundle --target=chrome58 --keep-names --format=esm --sourcemap --define:global=globalThis --conditions=module --outfile=./bundle/browser.js --platform=browser", "bundle:browser:prod": "npm run bundle:browser -- --outfile=./bundle/browser.min.js --minify", @@ -85,11 +86,11 @@ "license": "MIT", "dependencies": { "jintr": "^1.1.0", - "linkedom": "^0.14.12", "tslib": "^2.5.0", "undici": "^5.19.1" }, "devDependencies": { + "@types/glob": "^8.1.0", "@types/jest": "^28.1.7", "@types/node": "^17.0.45", "@typescript-eslint/eslint-plugin": "^5.30.6", diff --git a/deno/src/Innertube.ts b/deno/src/Innertube.ts index 409ffc1a..90829891 100644 --- a/deno/src/Innertube.ts +++ b/deno/src/Innertube.ts @@ -19,7 +19,7 @@ import { Kids, Music, Studio } from './core/clients/index.ts'; import { AccountManager, InteractionManager, PlaylistManager } from './core/managers/index.ts'; import { Feed, TabbedFeed } from './core/mixins/index.ts'; -import Proto from './proto/index.ts'; +import * as Proto from './proto/index.ts'; import * as Constants from './utils/Constants.ts'; import { InnertubeError, generateRandomString, throwIfMissing } from './utils/Utils.ts'; @@ -38,11 +38,11 @@ import { GetUnseenCountEndpoint } from './core/endpoints/notification/index.ts'; import type { ApiResponse } from './core/Actions.ts'; import type { IBrowseResponse, IParsedResponse } from './parser/types/index.ts'; import type { INextRequest } from './types/index.ts'; -import type { DownloadOptions, FormatOptions } from './utils/FormatUtils.ts'; +import type { DownloadOptions, FormatOptions } from './types/FormatUtils.ts'; export type InnertubeConfig = SessionOptions; -export type InnerTubeClient = 'WEB' | 'ANDROID' | 'YTMUSIC_ANDROID' | 'YTMUSIC' | 'YTSTUDIO_ANDROID' | 'TV_EMBEDDED' | 'YTKIDS' +export type InnerTubeClient = 'WEB' | 'iOS' | 'ANDROID' | 'YTMUSIC_ANDROID' | 'YTMUSIC' | 'YTSTUDIO_ANDROID' | 'TV_EMBEDDED' | 'YTKIDS'; export type SearchFilters = Partial<{ upload_date: 'all' | 'hour' | 'today' | 'week' | 'month' | 'year'; diff --git a/deno/src/core/Session.ts b/deno/src/core/Session.ts index 661b109d..387e617c 100644 --- a/deno/src/core/Session.ts +++ b/deno/src/core/Session.ts @@ -3,7 +3,7 @@ import EventEmitterLike from '../utils/EventEmitterLike.ts'; import Actions from './Actions.ts'; import Player from './Player.ts'; -import Proto from '../proto/index.ts'; +import * as Proto from '../proto/index.ts'; import type { ICache } from '../types/Cache.ts'; import type { FetchFunction } from '../types/PlatformShim.ts'; import HTTPClient from '../utils/HTTPClient.ts'; @@ -16,6 +16,7 @@ export enum ClientType { WEB = 'WEB', KIDS = 'WEB_KIDS', MUSIC = 'WEB_REMIX', + IOS = 'iOS', ANDROID = 'ANDROID', ANDROID_MUSIC = 'ANDROID_MUSIC', ANDROID_CREATOR = 'ANDROID_CREATOR', diff --git a/deno/src/core/clients/Music.ts b/deno/src/core/clients/Music.ts index dafd97f4..0d22d229 100644 --- a/deno/src/core/clients/Music.ts +++ b/deno/src/core/clients/Music.ts @@ -18,7 +18,7 @@ import PlaylistPanel from '../../parser/classes/PlaylistPanel.ts'; import SearchSuggestionsSection from '../../parser/classes/SearchSuggestionsSection.ts'; import SectionList from '../../parser/classes/SectionList.ts'; import Tab from '../../parser/classes/Tab.ts'; -import Proto from '../../proto/index.ts'; +import * as Proto from '../../proto/index.ts'; import type { ObservedArray, YTNode } from '../../parser/helpers.ts'; import type { MusicSearchFilters } from '../../types/index.ts'; @@ -329,7 +329,7 @@ export default class Music { if (!page.contents) throw new InnertubeError('Unexpected response', page); - if (page.contents.item().key('type').string() === 'Message') + if (page.contents.item().type === 'Message') throw new InnertubeError(page.contents.item().as(Message).text.toString(), video_id); const section_list = page.contents.item().as(SectionList).contents; diff --git a/deno/src/core/clients/Studio.ts b/deno/src/core/clients/Studio.ts index 9e496c9a..32aa32a7 100644 --- a/deno/src/core/clients/Studio.ts +++ b/deno/src/core/clients/Studio.ts @@ -1,4 +1,4 @@ -import Proto from '../../proto/index.ts'; +import * as Proto from '../../proto/index.ts'; import * as Constants from '../../utils/Constants.ts'; import { InnertubeError, MissingParamError, Platform } from '../../utils/Utils.ts'; diff --git a/deno/src/core/managers/AccountManager.ts b/deno/src/core/managers/AccountManager.ts index 4e08ed08..801523fa 100644 --- a/deno/src/core/managers/AccountManager.ts +++ b/deno/src/core/managers/AccountManager.ts @@ -3,7 +3,7 @@ import Analytics from '../../parser/youtube/Analytics.ts'; import Settings from '../../parser/youtube/Settings.ts'; import TimeWatched from '../../parser/youtube/TimeWatched.ts'; -import Proto from '../../proto/index.ts'; +import * as Proto from '../../proto/index.ts'; import { InnertubeError } from '../../utils/Utils.ts'; import { Account, BrowseEndpoint, Channel } from '../endpoints/index.ts'; diff --git a/deno/src/core/managers/InteractionManager.ts b/deno/src/core/managers/InteractionManager.ts index 41b433df..854535bb 100644 --- a/deno/src/core/managers/InteractionManager.ts +++ b/deno/src/core/managers/InteractionManager.ts @@ -1,4 +1,4 @@ -import Proto from '../../proto/index.ts'; +import * as Proto from '../../proto/index.ts'; import type Actions from '../Actions.ts'; import type { ApiResponse } from '../Actions.ts'; diff --git a/deno/src/core/mixins/MediaInfo.ts b/deno/src/core/mixins/MediaInfo.ts index 990989b7..ed7622ec 100644 --- a/deno/src/core/mixins/MediaInfo.ts +++ b/deno/src/core/mixins/MediaInfo.ts @@ -1,14 +1,15 @@ import type { ApiResponse } from '../Actions.ts'; import type Actions from '../Actions.ts'; import * as Constants from '../../utils/Constants.ts'; -import type { DownloadOptions, FormatFilter, FormatOptions, URLTransformer } from '../../utils/FormatUtils.ts'; -import FormatUtils from '../../utils/FormatUtils.ts'; +import type { DownloadOptions, FormatFilter, FormatOptions, URLTransformer } from '../../types/FormatUtils.ts'; +import * as FormatUtils from '../../utils/FormatUtils.ts'; import { InnertubeError } from '../../utils/Utils.ts'; import type Format from '../../parser/classes/misc/Format.ts'; import type { INextResponse, IPlayerResponse } from '../../parser/index.ts'; import Parser from '../../parser/index.ts'; import type { DashOptions } from '../../types/DashOptions.ts'; import PlayerStoryboardSpec from '../../parser/classes/PlayerStoryboardSpec.ts'; +import { getStreamingInfo } from '../../utils/StreamingInfo.ts'; export default class MediaInfo { #page: [IPlayerResponse, INextResponse?]; @@ -52,6 +53,21 @@ export default class MediaInfo { return FormatUtils.toDash(this.streaming_data, url_transformer, format_filter, this.#cpn, this.#actions.session.player, this.#actions, storyboards); } + /** + * Get a cleaned up representation of the adaptive_formats + */ + getStreamingInfo(url_transformer?: URLTransformer, format_filter?: FormatFilter) { + return getStreamingInfo( + this.streaming_data, + url_transformer, + format_filter, + this.cpn, + this.#actions.session.player, + this.#actions, + this.#page[0].storyboards?.is(PlayerStoryboardSpec) ? this.#page[0].storyboards : undefined + ); + } + /** * Selects the format that best matches the given options. * @param options - Options diff --git a/deno/src/parser/README.md b/deno/src/parser/README.md index 108e1d95..52bad8f6 100644 --- a/deno/src/parser/README.md +++ b/deno/src/parser/README.md @@ -310,7 +310,7 @@ const example_data = { // The first argument is the name of the class, the second is the data you have for the node. // It will return a class that extends YTNode. -const Example = Generator.YTNodeGenerator.generateRuntimeClass('Example', example_data); +const Example = Generator.generateRuntimeClass('Example', example_data); // You may now use this class as you would any other node. const example = new Example(example_data); diff --git a/deno/src/parser/classes/GridShow.ts b/deno/src/parser/classes/GridShow.ts index 63a62409..d7cbcdf9 100644 --- a/deno/src/parser/classes/GridShow.ts +++ b/deno/src/parser/classes/GridShow.ts @@ -1,6 +1,6 @@ import { YTNode, type ObservedArray } from '../helpers.ts'; import type { RawNode } from '../index.ts'; -import Parser from '../parser.ts'; +import * as Parser from '../parser.ts'; import Author from './misc/Author.ts'; import Text from './misc/Text.ts'; import NavigationEndpoint from './NavigationEndpoint.ts'; diff --git a/deno/src/parser/classes/GuideCollapsibleEntry.ts b/deno/src/parser/classes/GuideCollapsibleEntry.ts index aa16cd809..562eef11 100644 --- a/deno/src/parser/classes/GuideCollapsibleEntry.ts +++ b/deno/src/parser/classes/GuideCollapsibleEntry.ts @@ -1,4 +1,4 @@ -import Parser from '../parser.ts'; +import * as Parser from '../parser.ts'; import GuideEntry from './GuideEntry.ts'; import type { RawNode } from '../index.ts'; import { type ObservedArray, YTNode } from '../helpers.ts'; diff --git a/deno/src/parser/classes/GuideCollapsibleSectionEntry.ts b/deno/src/parser/classes/GuideCollapsibleSectionEntry.ts index 248768bd..48ad0ff0 100644 --- a/deno/src/parser/classes/GuideCollapsibleSectionEntry.ts +++ b/deno/src/parser/classes/GuideCollapsibleSectionEntry.ts @@ -1,4 +1,4 @@ -import Parser from '../parser.ts'; +import * as Parser from '../parser.ts'; import type { RawNode } from '../index.ts'; import { type ObservedArray, YTNode } from '../helpers.ts'; diff --git a/deno/src/parser/classes/GuideSection.ts b/deno/src/parser/classes/GuideSection.ts index 483fbba6..457e930d 100644 --- a/deno/src/parser/classes/GuideSection.ts +++ b/deno/src/parser/classes/GuideSection.ts @@ -1,5 +1,5 @@ import Text from './misc/Text.ts'; -import Parser from '../parser.ts'; +import * as Parser from '../parser.ts'; import { type ObservedArray, YTNode } from '../helpers.ts'; import type { RawNode } from '../index.ts'; diff --git a/deno/src/parser/classes/MusicCarouselShelf.ts b/deno/src/parser/classes/MusicCarouselShelf.ts index 750b36bb..b81b93f8 100644 --- a/deno/src/parser/classes/MusicCarouselShelf.ts +++ b/deno/src/parser/classes/MusicCarouselShelf.ts @@ -2,6 +2,7 @@ import { YTNode, type ObservedArray } from '../helpers.ts'; import Parser, { type RawNode } from '../index.ts'; import MusicCarouselShelfBasicHeader from './MusicCarouselShelfBasicHeader.ts'; +import MusicMultiRowListItem from './MusicMultiRowListItem.ts'; import MusicNavigationButton from './MusicNavigationButton.ts'; import MusicResponsiveListItem from './MusicResponsiveListItem.ts'; import MusicTwoRowItem from './MusicTwoRowItem.ts'; @@ -10,13 +11,13 @@ export default class MusicCarouselShelf extends YTNode { static type = 'MusicCarouselShelf'; header: MusicCarouselShelfBasicHeader | null; - contents: ObservedArray; + contents: ObservedArray; num_items_per_column?: number; constructor(data: RawNode) { super(); this.header = Parser.parseItem(data.header, MusicCarouselShelfBasicHeader); - this.contents = Parser.parseArray(data.contents, [ MusicTwoRowItem, MusicResponsiveListItem, MusicNavigationButton ]); + this.contents = Parser.parseArray(data.contents, [ MusicTwoRowItem, MusicResponsiveListItem, MusicMultiRowListItem, MusicNavigationButton ]); if (Reflect.has(data, 'numItemsPerColumn')) { this.num_items_per_column = parseInt(data.numItemsPerColumn); diff --git a/deno/src/parser/classes/MusicMultiRowListItem.ts b/deno/src/parser/classes/MusicMultiRowListItem.ts new file mode 100644 index 00000000..973a24dc --- /dev/null +++ b/deno/src/parser/classes/MusicMultiRowListItem.ts @@ -0,0 +1,44 @@ +import { YTNode } from '../helpers.ts'; +import { Parser, type RawNode } from '../index.ts'; +import { Text } from '../misc.ts'; + +import Menu from './menus/Menu.ts'; +import MusicItemThumbnailOverlay from './MusicItemThumbnailOverlay.ts'; +import MusicThumbnail from './MusicThumbnail.ts'; +import NavigationEndpoint from './NavigationEndpoint.ts'; + +export default class MusicMultiRowListItem extends YTNode { + static type = 'MusicMultiRowListItem'; + + thumbnail: MusicThumbnail | null; + overlay: MusicItemThumbnailOverlay | null; + on_tap: NavigationEndpoint; + menu: Menu | null; + subtitle: Text; + title: Text; + second_title?: Text; + description?: Text; + display_style?: string; + + constructor(data: RawNode) { + super(); + this.thumbnail = Parser.parseItem(data.thumbnail, MusicThumbnail); + this.overlay = Parser.parseItem(data.overlay, MusicItemThumbnailOverlay); + this.on_tap = new NavigationEndpoint(data.onTap); + this.menu = Parser.parseItem(data.menu, Menu); + this.subtitle = new Text(data.subtitle); + this.title = new Text(data.title); + + if (Reflect.has(data, 'secondTitle')) { + this.second_title = new Text(data.secondTitle); + } + + if (Reflect.has(data, 'description')) { + this.description = new Text(data.description); + } + + if (Reflect.has(data, 'displayStyle')) { + this.display_style = data.displayStyle; + } + } +} \ No newline at end of file diff --git a/deno/src/parser/classes/MusicResponsiveListItem.ts b/deno/src/parser/classes/MusicResponsiveListItem.ts index 2366bafa..46ccfb8a 100644 --- a/deno/src/parser/classes/MusicResponsiveListItem.ts +++ b/deno/src/parser/classes/MusicResponsiveListItem.ts @@ -1,5 +1,10 @@ // TODO: Clean up and refactor this. +import { YTNode } from '../helpers.ts'; +import { isTextRun, timeToSeconds } from '../../utils/Utils.ts'; +import type { ObservedArray } from '../helpers.ts'; +import type { RawNode } from '../index.ts'; + import Parser from '../index.ts'; import MusicItemThumbnailOverlay from './MusicItemThumbnailOverlay.ts'; import MusicResponsiveListItemFixedColumn from './MusicResponsiveListItemFixedColumn.ts'; @@ -9,11 +14,6 @@ import NavigationEndpoint from './NavigationEndpoint.ts'; import Menu from './menus/Menu.ts'; import Text from './misc/Text.ts'; -import { isTextRun, timeToSeconds } from '../../utils/Utils.ts'; -import type { ObservedArray } from '../helpers.ts'; -import { YTNode } from '../helpers.ts'; -import type { RawNode } from '../index.ts'; - export default class MusicResponsiveListItem extends YTNode { static type = 'MusicResponsiveListItem'; @@ -24,8 +24,8 @@ export default class MusicResponsiveListItem extends YTNode { playlist_set_video_id: string; }; - endpoint: NavigationEndpoint | null; - item_type: 'album' | 'playlist' | 'artist' | 'library_artist' | 'video' | 'song' | 'endpoint' | 'unknown' | undefined; + endpoint?: NavigationEndpoint; + item_type: 'album' | 'playlist' | 'artist' | 'library_artist' | 'non_music_track' | 'video' | 'song' | 'endpoint' | 'unknown' | undefined; index?: Text; thumbnail?: MusicThumbnail | null; badges; @@ -82,9 +82,21 @@ export default class MusicResponsiveListItem extends YTNode { playlist_set_video_id: data?.playlistItemData?.playlistSetVideoId || null }; - this.endpoint = data.navigationEndpoint ? new NavigationEndpoint(data.navigationEndpoint) : null; + if (Reflect.has(data, 'navigationEndpoint')) { + this.endpoint = new NavigationEndpoint(data.navigationEndpoint); + } - const page_type = this.endpoint?.payload?.browseEndpointContextSupportedConfigs?.browseEndpointContextMusicConfig?.pageType; + let page_type = this.endpoint?.payload?.browseEndpointContextSupportedConfigs?.browseEndpointContextMusicConfig?.pageType; + + if (!page_type) { + const is_non_music_track = this.flex_columns.find( + (col) => col.title.endpoint?.payload?.browseEndpointContextSupportedConfigs?.browseEndpointContextMusicConfig?.pageType === 'MUSIC_PAGE_TYPE_NON_MUSIC_AUDIO_TRACK_PAGE' + ); + + if (is_non_music_track) { + page_type = 'MUSIC_PAGE_TYPE_NON_MUSIC_AUDIO_TRACK_PAGE'; + } + } switch (page_type) { case 'MUSIC_PAGE_TYPE_ALBUM': @@ -104,27 +116,41 @@ export default class MusicResponsiveListItem extends YTNode { this.item_type = 'library_artist'; this.#parseLibraryArtist(); break; + case 'MUSIC_PAGE_TYPE_NON_MUSIC_AUDIO_TRACK_PAGE': + this.item_type = 'non_music_track'; + this.#parseNonMusicTrack(); + break; default: if (this.flex_columns[1]) { this.#parseVideoOrSong(); } else { this.#parseOther(); } - break; } - if (data.index) { + if (Reflect.has(data, 'index')) { this.index = new Text(data.index); } - this.thumbnail = Parser.parseItem(data.thumbnail, MusicThumbnail); - this.badges = Parser.parseArray(data.badges); - this.menu = Parser.parseItem(data.menu, Menu); - this.overlay = Parser.parseItem(data.overlay, MusicItemThumbnailOverlay); + if (Reflect.has(data, 'thumbnail')) { + this.thumbnail = Parser.parseItem(data.thumbnail, MusicThumbnail); + } + + if (Reflect.has(data, 'badges')) { + this.badges = Parser.parseArray(data.badges); + } + + if (Reflect.has(data, 'menu')) { + this.menu = Parser.parseItem(data.menu, Menu); + } + + if (Reflect.has(data, 'overlay')) { + this.overlay = Parser.parseItem(data.overlay, MusicItemThumbnailOverlay); + } } #parseOther() { - this.title = this.flex_columns.first().key('title').instanceof(Text).toString(); + this.title = this.flex_columns.first().title.toString(); if (this.endpoint) { this.item_type = 'endpoint'; @@ -134,7 +160,7 @@ export default class MusicResponsiveListItem extends YTNode { } #parseVideoOrSong() { - const is_video = this.flex_columns.at(1)?.key('title').instanceof(Text).runs?.some((run) => run.text.match(/(.*?) views/)); + const is_video = this.flex_columns.at(1)?.title.runs?.some((run) => run.text.match(/(.*?) views/)); if (is_video) { this.item_type = 'video'; this.#parseVideo(); @@ -146,10 +172,10 @@ export default class MusicResponsiveListItem extends YTNode { #parseSong() { this.id = this.#playlist_item_data.video_id || this.endpoint?.payload?.videoId; - this.title = this.flex_columns.first().key('title').instanceof(Text).toString(); + this.title = this.flex_columns.first().title.toString(); - const duration_text = this.flex_columns.at(1)?.key('title').instanceof(Text).runs?.find( - (run) => (/^\d+$/).test(run.text.replace(/:/g, '')))?.text || this.fixed_columns.first()?.key('title').instanceof(Text)?.toString(); + const duration_text = this.flex_columns.at(1)?.title.runs?.find( + (run) => (/^\d+$/).test(run.text.replace(/:/g, '')))?.text || this.fixed_columns.first()?.title?.toString(); if (duration_text) { this.duration = { @@ -159,12 +185,12 @@ export default class MusicResponsiveListItem extends YTNode { } const album_run = - this.flex_columns.at(1)?.key('title').instanceof(Text).runs?.find( + this.flex_columns.at(1)?.title.runs?.find( (run) => (isTextRun(run) && run.endpoint) && run.endpoint.payload.browseId.startsWith('MPR') ) || - this.flex_columns.at(2)?.key('title').instanceof(Text).runs?.find( + this.flex_columns.at(2)?.title.runs?.find( (run) => (isTextRun(run) && run.endpoint) && run.endpoint.payload.browseId.startsWith('MPR') @@ -178,7 +204,7 @@ export default class MusicResponsiveListItem extends YTNode { }; } - const artist_runs = this.flex_columns.at(1)?.key('title').instanceof(Text).runs?.filter( + const artist_runs = this.flex_columns.at(1)?.title.runs?.filter( (run) => (isTextRun(run) && run.endpoint) && run.endpoint.payload.browseId.startsWith('UC') ); @@ -193,10 +219,10 @@ export default class MusicResponsiveListItem extends YTNode { #parseVideo() { this.id = this.#playlist_item_data.video_id; - this.title = this.flex_columns.first().key('title').instanceof(Text).toString(); - this.views = this.flex_columns.at(1)?.key('title').instanceof(Text).runs?.find((run) => run.text.match(/(.*?) views/))?.toString(); + this.title = this.flex_columns.first().title.toString(); + this.views = this.flex_columns.at(1)?.title.runs?.find((run) => run.text.match(/(.*?) views/))?.toString(); - const author_runs = this.flex_columns.at(1)?.key('title').instanceof(Text).runs?.filter( + const author_runs = this.flex_columns.at(1)?.title.runs?.filter( (run) => (isTextRun(run) && run.endpoint) && run.endpoint.payload.browseId.startsWith('UC') @@ -212,8 +238,8 @@ export default class MusicResponsiveListItem extends YTNode { }); } - const duration_text = this.flex_columns[1].key('title').instanceof(Text).runs?.find( - (run) => (/^\d+$/).test(run.text.replace(/:/g, '')))?.text || this.fixed_columns.first()?.key('title').instanceof(Text).runs?.find((run) => (/^\d+$/).test(run.text.replace(/:/g, '')))?.text; + const duration_text = this.flex_columns[1].title.runs?.find( + (run) => (/^\d+$/).test(run.text.replace(/:/g, '')))?.text || this.fixed_columns.first()?.title.runs?.find((run) => (/^\d+$/).test(run.text.replace(/:/g, '')))?.text; if (duration_text) { this.duration = { @@ -225,22 +251,27 @@ export default class MusicResponsiveListItem extends YTNode { #parseArtist() { this.id = this.endpoint?.payload?.browseId; - this.name = this.flex_columns.first().key('title').instanceof(Text).toString(); - this.subtitle = this.flex_columns.at(1)?.key('title').instanceof(Text); + this.name = this.flex_columns.first().title.toString(); + this.subtitle = this.flex_columns.at(1)?.title; this.subscribers = this.subtitle?.runs?.find((run) => (/^(\d*\.)?\d+[M|K]? subscribers?$/i).test(run.text))?.text || ''; } #parseLibraryArtist() { - this.name = this.flex_columns.first().key('title').instanceof(Text).toString(); - this.subtitle = this.flex_columns.at(1)?.key('title').instanceof(Text); + this.name = this.flex_columns.first().title.toString(); + this.subtitle = this.flex_columns.at(1)?.title; this.song_count = this.subtitle?.runs?.find((run) => (/^\d+(,\d+)? songs?$/i).test(run.text))?.text || ''; } + #parseNonMusicTrack() { + this.id = this.#playlist_item_data.video_id || this.endpoint?.payload?.videoId; + this.title = this.flex_columns.first().title.toString(); + } + #parseAlbum() { this.id = this.endpoint?.payload?.browseId; this.title = this.flex_columns.first().title.toString(); - const author_run = this.flex_columns.at(1)?.key('title').instanceof(Text).runs?.find( + const author_run = this.flex_columns.at(1)?.title.runs?.find( (run) => (isTextRun(run) && run.endpoint) && run.endpoint.payload.browseId.startsWith('UC') @@ -254,7 +285,7 @@ export default class MusicResponsiveListItem extends YTNode { }; } - this.year = this.flex_columns.at(1)?.key('title').instanceof(Text).runs?.find( + this.year = this.flex_columns.at(1)?.title.runs?.find( (run) => (/^[12][0-9]{3}$/).test(run.text) )?.text; } @@ -263,12 +294,12 @@ export default class MusicResponsiveListItem extends YTNode { this.id = this.endpoint?.payload?.browseId; this.title = this.flex_columns.first().title.toString(); - const item_count_run = this.flex_columns.at(1)?.key('title') - .instanceof(Text).runs?.find((run) => run.text.match(/\d+ (song|songs)/)); + const item_count_run = this.flex_columns.at(1)?.title + .runs?.find((run) => run.text.match(/\d+ (song|songs)/)); this.item_count = item_count_run ? item_count_run.text : undefined; - const author_run = this.flex_columns.at(1)?.key('title').instanceof(Text).runs?.find( + const author_run = this.flex_columns.at(1)?.title.runs?.find( (run) => (isTextRun(run) && run.endpoint) && run.endpoint.payload.browseId.startsWith('UC') diff --git a/deno/src/parser/classes/PlayerStoryboardSpec.ts b/deno/src/parser/classes/PlayerStoryboardSpec.ts index 13144876..d5cd20ed 100644 --- a/deno/src/parser/classes/PlayerStoryboardSpec.ts +++ b/deno/src/parser/classes/PlayerStoryboardSpec.ts @@ -1,19 +1,21 @@ import { YTNode } from '../helpers.ts'; import type { RawNode } from '../index.ts'; +export interface StoryboardData { + template_url: string; + thumbnail_width: number; + thumbnail_height: number; + thumbnail_count: number; + interval: number; + columns: number; + rows: number; + storyboard_count: number; +} + export default class PlayerStoryboardSpec extends YTNode { static type = 'PlayerStoryboardSpec'; - boards: { - template_url: string; - thumbnail_width: number; - thumbnail_height: number; - thumbnail_count: number; - interval: number; - columns: number; - rows: number; - storyboard_count: number; - }[]; + boards: StoryboardData[]; constructor(data: RawNode) { super(); diff --git a/deno/src/parser/classes/SharedPost.ts b/deno/src/parser/classes/SharedPost.ts index 30f2aa27..70dbdfe6 100644 --- a/deno/src/parser/classes/SharedPost.ts +++ b/deno/src/parser/classes/SharedPost.ts @@ -1,6 +1,6 @@ import { YTNode } from '../helpers.ts'; import type { RawNode } from '../index.ts'; -import Parser from '../parser.ts'; +import * as Parser from '../parser.ts'; import BackstagePost from './BackstagePost.ts'; import Button from './Button.ts'; import Menu from './menus/Menu.ts'; diff --git a/deno/src/parser/classes/actions/AppendContinuationItemsAction.ts b/deno/src/parser/classes/actions/AppendContinuationItemsAction.ts index 3cdf3227..3b4eb119 100644 --- a/deno/src/parser/classes/actions/AppendContinuationItemsAction.ts +++ b/deno/src/parser/classes/actions/AppendContinuationItemsAction.ts @@ -1,16 +1,17 @@ import Parser from '../../index.ts'; import type { RawNode } from '../../index.ts'; -import { type SuperParsedResult, YTNode } from '../../helpers.ts'; +import type { ObservedArray } from '../../helpers.ts'; +import { YTNode } from '../../helpers.ts'; export default class AppendContinuationItemsAction extends YTNode { static type = 'AppendContinuationItemsAction'; - items: SuperParsedResult; + contents: ObservedArray | null; target: string; constructor(data: RawNode) { super(); - this.items = Parser.parse(data.continuationItems); + this.contents = Parser.parseArray(data.continuationItems); this.target = data.target; } } \ No newline at end of file diff --git a/deno/src/parser/classes/comments/Comment.ts b/deno/src/parser/classes/comments/Comment.ts index 8a68ded5..779a9a65 100644 --- a/deno/src/parser/classes/comments/Comment.ts +++ b/deno/src/parser/classes/comments/Comment.ts @@ -11,7 +11,7 @@ import CommentReplyDialog from './CommentReplyDialog.ts'; import PdgCommentChip from './PdgCommentChip.ts'; import SponsorCommentBadge from './SponsorCommentBadge.ts'; -import Proto from '../../../proto/index.ts'; +import * as Proto from '../../../proto/index.ts'; import { InnertubeError } from '../../../utils/Utils.ts'; import { YTNode } from '../../helpers.ts'; diff --git a/deno/src/parser/continuations.ts b/deno/src/parser/continuations.ts new file mode 100644 index 00000000..fbdabbf8 --- /dev/null +++ b/deno/src/parser/continuations.ts @@ -0,0 +1,211 @@ +import type { ObservedArray} from './helpers.ts'; +import { YTNode, observe } from './helpers.ts'; +import type { RawNode } from './index.ts'; +import { Thumbnail } from './misc.ts'; +import { NavigationEndpoint, LiveChatItemList, LiveChatHeader, LiveChatParticipantsList, Message } from './nodes.ts'; +import * as Parser from './parser.ts'; + +export class ItemSectionContinuation extends YTNode { + static readonly type = 'itemSectionContinuation'; + + contents: ObservedArray | null; + continuation?: string; + + constructor(data: RawNode) { + super(); + this.contents = Parser.parseArray(data.contents); + if (Array.isArray(data.continuations)) { + this.continuation = data.continuations?.at(0)?.nextContinuationData?.continuation; + } + } +} + +export class NavigateAction extends YTNode { + static readonly type = 'navigateAction'; + + endpoint: NavigationEndpoint; + + constructor(data: RawNode) { + super(); + this.endpoint = new NavigationEndpoint(data.endpoint); + } +} + +export class ShowMiniplayerCommand extends YTNode { + static readonly type = 'showMiniplayerCommand'; + + miniplayer_command: NavigationEndpoint; + show_premium_branding: boolean; + + constructor(data: RawNode) { + super(); + this.miniplayer_command = new NavigationEndpoint(data.miniplayerCommand); + this.show_premium_branding = data.showPremiumBranding; + } +} + +export { default as AppendContinuationItemsAction } from './classes/actions/AppendContinuationItemsAction.ts'; + +export class ReloadContinuationItemsCommand extends YTNode { + static readonly type = 'reloadContinuationItemsCommand'; + + target_id: string; + contents: ObservedArray | null; + slot?: string; + + constructor(data: RawNode) { + super(); + this.target_id = data.targetId; + this.contents = Parser.parse(data.continuationItems, true); + this.slot = data?.slot; + } +} + +export class SectionListContinuation extends YTNode { + static readonly type = 'sectionListContinuation'; + + continuation: string; + contents: ObservedArray | null; + + constructor(data: RawNode) { + super(); + this.contents = Parser.parse(data.contents, true); + this.continuation = + data.continuations?.[0]?.nextContinuationData?.continuation || + data.continuations?.[0]?.reloadContinuationData?.continuation || null; + } +} + +export class MusicPlaylistShelfContinuation extends YTNode { + static readonly type = 'musicPlaylistShelfContinuation'; + + continuation: string; + contents: ObservedArray | null; + + constructor(data: RawNode) { + super(); + this.contents = Parser.parse(data.contents, true); + this.continuation = data.continuations?.[0].nextContinuationData.continuation || null; + } +} + +export class MusicShelfContinuation extends YTNode { + static readonly type = 'musicShelfContinuation'; + + continuation: string; + contents: ObservedArray | null; + + constructor(data: RawNode) { + super(); + this.contents = Parser.parseArray(data.contents); + this.continuation = + data.continuations?.[0].nextContinuationData?.continuation || + data.continuations?.[0].reloadContinuationData?.continuation || null; + } +} + +export class GridContinuation extends YTNode { + static readonly type = 'gridContinuation'; + + continuation: string; + items: ObservedArray | null; + + constructor(data: RawNode) { + super(); + this.items = Parser.parse(data.items, true); + this.continuation = data.continuations?.[0].nextContinuationData.continuation || null; + } + + get contents() { + return this.items; + } +} + +export class PlaylistPanelContinuation extends YTNode { + static readonly type = 'playlistPanelContinuation'; + + continuation: string; + contents: ObservedArray | null; + + constructor(data: RawNode) { + super(); + this.contents = Parser.parseArray(data.contents); + this.continuation = data.continuations?.[0]?.nextContinuationData?.continuation || + data.continuations?.[0]?.nextRadioContinuationData?.continuation || null; + } +} + +export class Continuation extends YTNode { + static readonly type = 'continuation'; + + continuation_type: string; + timeout_ms?: number; + time_until_last_message_ms?: number; + token: string; + + constructor(data: RawNode) { + super(); + this.continuation_type = data.type; + this.timeout_ms = data.continuation?.timeoutMs; + this.time_until_last_message_ms = data.continuation?.timeUntilLastMessageMsec; + this.token = data.continuation?.continuation; + } +} + +export class LiveChatContinuation extends YTNode { + static readonly type = 'liveChatContinuation'; + + actions: ObservedArray; + action_panel: YTNode | null; + item_list: LiveChatItemList | null; + header: LiveChatHeader | null; + participants_list: LiveChatParticipantsList | null; + popout_message: Message | null; + emojis: { + emoji_id: string; + shortcuts: string[]; + search_terms: string[]; + image: Thumbnail[]; + }[]; + continuation: Continuation; + viewer_name: string; + + constructor(data: RawNode) { + super(); + this.actions = Parser.parse(data.actions?.map((action: any) => { + delete action.clickTrackingParams; + return action; + }), true) || observe([]); + + this.action_panel = Parser.parseItem(data.actionPanel); + this.item_list = Parser.parseItem(data.itemList, LiveChatItemList); + this.header = Parser.parseItem(data.header, LiveChatHeader); + this.participants_list = Parser.parseItem(data.participantsList, LiveChatParticipantsList); + this.popout_message = Parser.parseItem(data.popoutMessage, Message); + + this.emojis = data.emojis?.map((emoji: any) => ({ + emoji_id: emoji.emojiId, + shortcuts: emoji.shortcuts, + search_terms: emoji.searchTerms, + image: Thumbnail.fromResponse(emoji.image), + is_custom_emoji: emoji.isCustomEmoji + })) || []; + + let continuation, type; + + if (data.continuations?.[0].timedContinuationData) { + type = 'timed'; + continuation = data.continuations?.[0].timedContinuationData; + } else if (data.continuations?.[0].invalidationContinuationData) { + type = 'invalidation'; + continuation = data.continuations?.[0].invalidationContinuationData; + } else if (data.continuations?.[0].liveChatReplayContinuationData) { + type = 'replay'; + continuation = data.continuations?.[0].liveChatReplayContinuationData; + } + + this.continuation = new Continuation({ continuation, type }); + + this.viewer_name = data.viewerName; + } +} diff --git a/deno/src/parser/generator.ts b/deno/src/parser/generator.ts index deb81b8b..8ceec39c 100644 --- a/deno/src/parser/generator.ts +++ b/deno/src/parser/generator.ts @@ -1,12 +1,12 @@ /* eslint-disable no-cond-assign */ -import { InnertubeError, Platform } from '../utils/Utils.ts'; +import { InnertubeError } from '../utils/Utils.ts'; import Author from './classes/misc/Author.ts'; import Text from './classes/misc/Text.ts'; import Thumbnail from './classes/misc/Thumbnail.ts'; import NavigationEndpoint from './classes/NavigationEndpoint.ts'; import type { YTNodeConstructor } from './helpers.ts'; import { YTNode } from './helpers.ts'; -import Parser from './parser.ts'; +import * as Parser from './parser.ts'; export type MiscInferenceType = { type: 'misc', @@ -53,551 +53,36 @@ export type InferenceType = { export type KeyInfo = (readonly [string, InferenceType])[]; -export class YTNodeGenerator { - static #ignored_keys = new Set([ - 'trackingParams', 'accessibility', 'accessibilityData' - ]); - static #renderers_examples: Record = {}; - static #camelToSnake(str: string) { - return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`); - } - static #logNewClass(classname: string, key_info: KeyInfo) { - console.warn( - new InnertubeError(`${classname} not found!\nThis is a bug, want to help us fix it? Follow the instructions at ${Platform.shim.info.repo_url}/blob/main/docs/updating-the-parser.md or report it at ${Platform.shim.info.bugs_url}!\nIntrospected and JIT generated this class in the meantime:\n${this.generateTypescriptClass(classname, key_info)}`) - ); - } - static #logChangedKeys(classname: string, key_info: KeyInfo, changed_keys: KeyInfo) { - console.warn(`${classname} changed!\nThe following keys where altered: ${changed_keys.map(([ key ]) => this.#camelToSnake(key)).join(', ')}\nThe class has changed to:\n${this.generateTypescriptClass(classname, key_info)}`); - } - /** - * Is this key ignored by the parser? - * @param key - The key to check - * @returns Whether or not the key is ignored - */ - static isIgnoredKey(key: string | symbol) { - return typeof key === 'string' && this.#ignored_keys.has(key); - } - /** - * Merges two sets of key info, resolving any conflicts - * @param key_info - The current key info - * @param new_key_info - The new key info - * @returns The merged key info - */ - static mergeKeyInfo(key_info: KeyInfo, new_key_info: KeyInfo) { - const changed_keys = new Map(); - const current_keys = new Set(key_info.map(([ key ]) => key)); - const new_keys = new Set(new_key_info.map(([ key ]) => key)); +const IGNORED_KEYS = new Set([ + 'trackingParams', 'accessibility', 'accessibilityData' +]); - const added_keys = new_key_info.filter(([ key ]) => !current_keys.has(key)); - const removed_keys = key_info.filter(([ key ]) => !new_keys.has(key)); +const RENDERER_EXAMPLES: Record = {}; - const common_keys = key_info.filter(([ key ]) => new_keys.has(key)); +export function camelToSnake(str: string) { + return str.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`); +} - const new_key_map = new Map(new_key_info); - - for (const [ key, type ] of common_keys) { - const new_type = new_key_map.get(key); - if (!new_type) continue; - if (type.type !== new_type.type) { - // We've got a type mismatch, this is unknown, we do not resolve unions - changed_keys.set(key, { - type: 'unknown', - optional: true - }); - continue; - } - // We've got the same type, so we can now resolve the changes - switch (type.type) { - case 'object': - { - if (new_type.type !== 'object') continue; - const { resolved_key_info } = this.mergeKeyInfo(type.keys, new_type.keys); - const resolved_key: InferenceType = { - type: 'object', - keys: resolved_key_info, - optional: type.optional || new_type.optional - }; - const did_change = JSON.stringify(resolved_key) !== JSON.stringify(type); - if (did_change) changed_keys.set(key, resolved_key); - } - break; - case 'renderer': - { - if (new_type.type !== 'renderer') continue; - const union_map = { - ...type.renderers, - ...new_type.renderers - }; - const either_optional = type.optional || new_type.optional; - const resolved_key: InferenceType = { - type: 'renderer', - renderers: union_map, - optional: either_optional - }; - const did_change = JSON.stringify({ - ...resolved_key, - renderers: Object.keys(resolved_key.renderers) - }) !== JSON.stringify({ - ...type, - renderers: Object.keys(type.renderers) - }); - if (did_change) changed_keys.set(key, resolved_key); - } - break; - case 'renderer_list': - { - if (new_type.type !== 'renderer_list') continue; - const union_map = { - ...type.renderers, - ...new_type.renderers - }; - const either_optional = type.optional || new_type.optional; - const resolved_key: InferenceType = { - type: 'renderer_list', - renderers: union_map, - optional: either_optional - }; - const did_change = JSON.stringify({ - ...resolved_key, - renderers: Object.keys(resolved_key.renderers) - }) !== JSON.stringify({ - ...type, - renderers: Object.keys(type.renderers) - }); - if (did_change) changed_keys.set(key, resolved_key); - } - break; - case 'misc': - { - if (new_type.type !== 'misc') continue; - if (type.misc_type !== new_type.misc_type) { - // We've got a type mismatch, this is unknown, we do not resolve unions - changed_keys.set(key, { - type: 'unknown', - optional: true - }); - } - switch (type.misc_type) { - case 'Author': - { - if (new_type.misc_type !== 'Author') break; - const had_optional_param = type.params[1] || new_type.params[1]; - const either_optional = type.optional || new_type.optional; - const resolved_key: MiscInferenceType = { - type: 'misc', - misc_type: 'Author', - optional: either_optional, - params: [ new_type.params[0], had_optional_param ] - }; - const did_change = JSON.stringify(resolved_key) !== JSON.stringify(type); - if (did_change) changed_keys.set(key, resolved_key); - } - break; - // Other cases can not change - } - } - break; - case 'primative': - { - if (new_type.type !== 'primative') continue; - const resolved_key: InferenceType = { - type: 'primative', - typeof: Array.from(new Set([ ...new_type.typeof, ...type.typeof ])), - optional: type.optional || new_type.optional - }; - const did_change = JSON.stringify(resolved_key) !== JSON.stringify(type); - if (did_change) changed_keys.set(key, resolved_key); - } - break; - } - } - - for (const [ key, type ] of added_keys) { - changed_keys.set(key, { - ...type, - optional: true - }); - } - - for (const [ key, type ] of removed_keys) { - changed_keys.set(key, { - ...type, - optional: true - }); - } - - const unchanged_keys = key_info.filter(([ key ]) => !changed_keys.has(key)); - - const resolved_key_info_map = new Map([ ...unchanged_keys, ...changed_keys ]); - const resolved_key_info = [ ...resolved_key_info_map.entries() ]; - - return { - resolved_key_info, - changed_keys: [ ...changed_keys.entries() ] - }; - } - /** - * Given a classname and its resolved key info, create a new class - * @param classname - The name of the class - * @param key_info - The resolved key info - * @returns Class based on the key info extending YTNode - */ - static createRuntimeClass(classname: string, key_info: KeyInfo): YTNodeConstructor { - this.#logNewClass(classname, key_info); - const node = class extends YTNode { - static type = classname; - static #key_info = new Map(); - static set key_info(key_info: KeyInfo) { - this.#key_info = new Map(key_info); - } - static get key_info() { - return [ ...this.#key_info.entries() ]; - } - constructor(data: any) { - super(); - const { - key_info, - unimplemented_dependencies - } = YTNodeGenerator.introspect(data); - - const { - resolved_key_info, - changed_keys - } = YTNodeGenerator.mergeKeyInfo(node.key_info, key_info); - - const did_change = changed_keys.length > 0; - - if (did_change) { - node.key_info = resolved_key_info; - YTNodeGenerator.#logChangedKeys(classname, node.key_info, changed_keys); - } - - for (const [ name, data ] of unimplemented_dependencies) - YTNodeGenerator.generateRuntimeClass(name, data); - - for (const [ key, value ] of key_info) { - let snake_key = YTNodeGenerator.#camelToSnake(key); - if (value.type === 'misc' && value.misc_type === 'NavigationEndpoint') - snake_key = 'endpoint'; - Reflect.set(this, snake_key, YTNodeGenerator.parse(key, value, data)); - } - } - }; - node.key_info = key_info; - Object.defineProperty(node, 'name', { value: classname, writable: false }); - return node; - } - /** - * Introspect an example of a class in order to determine its key info and dependencies - * @param classdata - The example of the class - * @returns The key info and any unimplemented dependencies - */ - static introspect(classdata: string) { - const key_info = this.#introspect(classdata); - const dependencies = new Map(); - for (const [ , value ] of key_info) { - if (value.type === 'renderer' || value.type === 'renderer_list') - for (const renderer of value.renderers) { - const example = this.#renderers_examples[renderer]; - if (example) - dependencies.set(renderer, example); - } - } - const unimplemented_dependencies = Array.from(dependencies).filter(([ classname ]) => !Parser.hasParser(classname)); - - return { - key_info, - unimplemented_dependencies - }; - } - /** - * Given example data for a class, introspect, implement dependencies, and create a new class - * @param classname - The name of the class - * @param classdata - The example of the class - * @returns Class based on the example classdata extending YTNode - */ - static generateRuntimeClass(classname: string, classdata: any) { - const { - key_info, - unimplemented_dependencies - } = this.introspect(classdata); - - const JITNode = this.createRuntimeClass(classname, key_info); - Parser.addRuntimeParser(classname, JITNode); - - for (const [ name, data ] of unimplemented_dependencies) - this.generateRuntimeClass(name, data); - - return JITNode; - } - /** - * Generate a typescript class based on the key info - * @param classname - The name of the class - * @param key_info - The key info, as returned by {@link YTNodeGenerator.introspect} - * @returns Typescript class file - */ - static generateTypescriptClass(classname: string, key_info: KeyInfo) { - const props: string[] = []; - const constructor_lines = [ - 'super();' - ]; - for (const [ key, value ] of key_info) { - let snake_key = this.#camelToSnake(key); - if (value.type === 'misc' && value.misc_type === 'NavigationEndpoint') - snake_key = 'endpoint'; - props.push(`${snake_key}${value.optional ? '?' : ''}: ${this.toTypeDeclaration(value)};`); - constructor_lines.push(`this.${snake_key} = ${this.toParser(key, value)};`); - } - return `class ${classname} extends YTNode {\n static type = '${classname}';\n\n ${props.join('\n ')}\n\n constructor(data: RawNode) {\n ${constructor_lines.join('\n ')}\n }\n}\n`; - } - /** - * For a given inference type, get the typescript type declaration - * @param inference_type - The inference type to get the declaration for - * @param indentation - The indentation level (used for objects) - * @returns Typescript type declaration - */ - static toTypeDeclaration(inference_type: InferenceType, indentation = 0): string { - switch (inference_type.type) { - case 'renderer': - { - return `${inference_type.renderers.map((type) => `YTNodes.${type}`).join(' | ')} | null`; - } - case 'renderer_list': - { - return `ObservedArray<${inference_type.renderers.map((type) => `YTNodes.${type}`).join(' | ')}> | null`; - } - case 'object': - { - return `{\n${inference_type.keys.map(([ key, value ]) => `${' '.repeat((indentation + 2) * 2)}${this.#camelToSnake(key)}${value.optional ? '?' : ''}: ${this.toTypeDeclaration(value, indentation + 1)}`).join(',\n')}\n${' '.repeat((indentation + 1) * 2)}}`; - } - case 'misc': - switch (inference_type.misc_type) { - case 'Thumbnail': - return 'Thumbnail[]'; - default: - return inference_type.misc_type; - } - case 'primative': - return inference_type.typeof.join(' | '); - case 'unknown': - return '/* TODO: determine correct type */ unknown'; - } - } - /** - * Generate statements to parse a given inference type - * @param key - The key to parse - * @param inference_type - The inference type to parse - * @param key_path - The path to the key (excluding the key itself) - * @param indentation - The indentation level (used for objects) - * @returns Statement to parse the given key - */ - static toParser(key: string, inference_type: InferenceType, key_path: string[] = [ 'data' ], indentation = 1) { - let parser = 'undefined'; - switch (inference_type.type) { - case 'renderer': - { - parser = `Parser.parseItem(${key_path.join('.')}.${key}, [ ${inference_type.renderers.map((type) => `YTNodes.${type}`).join(', ')} ])`; - } - break; - case 'renderer_list': - { - parser = `Parser.parse(${key_path.join('.')}.${key}, true, [ ${inference_type.renderers.map((type) => `YTNodes.${type}`).join(', ')} ])`; - } - break; - case 'object': - { - const new_keypath = [ ...key_path, key ]; - parser = `{\n${inference_type.keys.map(([ key, value ]) => `${' '.repeat((indentation + 2) * 2)}${this.#camelToSnake(key)}: ${this.toParser(key, value, new_keypath, indentation + 1)}`).join(',\n')}\n${' '.repeat((indentation + 1) * 2)}}`; - } - break; - case 'misc': - switch (inference_type.misc_type) { - case 'Thumbnail': - parser = `Thumbnail.fromResponse(${key_path.join('.')}.${key})`; - break; - case 'Author': - { - const author_parser = `new Author(${key_path.join('.')}.${inference_type.params[0]}, ${inference_type.params[1] ? `${key_path.join('.')}.${inference_type.params[1]}` : 'undefined'})`; - if (inference_type.optional) - return `Reflect.has(${key_path.join('.')}, '${inference_type.params[0]}') ? ${author_parser} : undefined`; - return author_parser; - } - default: - parser = `new ${inference_type.misc_type}(${key_path.join('.')}.${key})`; - break; - } - if (parser === 'undefined') - throw new Error('Unreachable code reached! Switch missing case!'); - break; - case 'primative': - case 'unknown': - parser = `${key_path.join('.')}.${key}`; - break; - } - if (inference_type.optional) - return `Reflect.has(${key_path.join('.')}, '${key}') ? ${parser} : undefined`; - return parser; - } - static #accessDataFromKeyPath(root: any, key_path: string[]) { - let data = root; - for (const key of key_path) - data = data[key]; - return data; - } - static #hasDataFromKeyPath(root: any, key_path: string[]) { - let data = root; - for (const key of key_path) - if (!Reflect.has(data, key)) - return false; - else - data = data[key]; - return true; - } - /** - * Parse a value from a given key path using the given inference type - * @param key - The key to parse - * @param inference_type - The inference type to parse - * @param data - The data to parse from - * @param key_path - The path to the key (excluding the key itself) - * @returns The parsed value - */ - static parse(key: string, inference_type: InferenceType, data: any, key_path: string[] = [ 'data' ]) { - const should_optional = !inference_type.optional || this.#hasDataFromKeyPath({ data }, [ ...key_path, key ]); - switch (inference_type.type) { - case 'renderer': - { - return should_optional ? Parser.parseItem(this.#accessDataFromKeyPath({ data }, [ ...key_path, key ]), inference_type.renderers.map((type) => Parser.getParserByName(type))) : undefined; - } - case 'renderer_list': - { - return should_optional ? Parser.parse(this.#accessDataFromKeyPath({ data }, [ ...key_path, key ]), true, inference_type.renderers.map((type) => Parser.getParserByName(type))) : undefined; - } - case 'object': - { - const obj: any = {}; - const new_key_path = [ ...key_path, key ]; - for (const [ key, value ] of inference_type.keys) { - obj[key] = should_optional ? this.parse(key, value, data, new_key_path) : undefined; - } - return obj; - } - case 'misc': - switch (inference_type.misc_type) { - case 'NavigationEndpoint': - return should_optional ? new NavigationEndpoint(this.#accessDataFromKeyPath({ data }, [ ...key_path, key ])) : undefined; - case 'Text': - return should_optional ? new Text(this.#accessDataFromKeyPath({ data }, [ ...key_path, key ])) : undefined; - case 'Thumbnail': - return should_optional ? Thumbnail.fromResponse(this.#accessDataFromKeyPath({ data }, [ ...key_path, key ])) : undefined; - case 'Author': - { - const author_should_optional = !inference_type.optional || this.#hasDataFromKeyPath({ data }, [ ...key_path, inference_type.params[0] ]); - return author_should_optional ? new Author( - this.#accessDataFromKeyPath({ data }, [ ...key_path, inference_type.params[0] ]), - inference_type.params[1] ? - this.#accessDataFromKeyPath({ data }, [ ...key_path, inference_type.params[1] ]) : undefined - ) : undefined; - } - } - throw new Error('Unreachable code reached! Switch missing case!'); - case 'primative': - case 'unknown': - return this.#accessDataFromKeyPath({ data }, [ ...key_path, key ]); - } - } - static #passOne(classdata: any) { - const keys = Reflect.ownKeys(classdata).filter((key) => !this.isIgnoredKey(key)).filter((key) => typeof key === 'string') as string[]; - const key_info = keys.map((key) => { - const value = classdata[key]; - const inferred_type = this.inferType(key as string, value); - return [ - key, - inferred_type - ] as const; - }); - return key_info; - } - static #passTwo(key_info: KeyInfo) { - // The second pass will detect Author - const channel_nav = key_info.filter(([ , value ]) => { - if (value.type !== 'misc') return false; - if (!(value.misc_type === 'NavigationEndpoint' || value.misc_type === 'Text')) return false; - return value.endpoint?.metadata.page_type === 'WEB_PAGE_TYPE_CHANNEL'; - }); - - // Whichever one has the longest text is the most probable match - const most_probable_match = channel_nav.sort(([ , a ], [ , b ]) => { - if (a.type !== 'misc' || b.type !== 'misc') return 0; - if (a.misc_type !== 'Text' || b.misc_type !== 'Text') return 0; - return b.text.length - a.text.length; - }); - - const excluded_keys = new Set(); - - const cannonical_channel_nav = most_probable_match[0]; - - let author: MiscInferenceType | undefined; - // We've found an author - if (cannonical_channel_nav) { - excluded_keys.add(cannonical_channel_nav[0]); - // Now to locate its metadata - // We'll first get all the keys in the classdata - const keys = key_info.map(([ key ]) => key); - // Check for anything ending in 'Badges' equals 'badges' - const badges = keys.filter((key) => key.endsWith('Badges') || key === 'badges'); - // The likely candidate is the one with some prefix (owner, author) - const likely_badges = badges.filter((key) => key.startsWith('owner') || key.startsWith('author')); - // If we have a likely candidate, we'll use that - const cannonical_badges = likely_badges[0] ?? badges[0]; - // Now we have the author and its badges - // Verify that its actually badges - const badge_key_info = key_info.find(([ key ]) => key === cannonical_badges); - const is_badges = badge_key_info ? - badge_key_info[1].type === 'renderer_list' && Reflect.has(badge_key_info[1].renderers, 'MetadataBadge') : - false; - - if (is_badges && cannonical_badges) excluded_keys.add(cannonical_badges); - // TODO: next we check for the author's thumbnail - author = { - type: 'misc', - misc_type: 'Author', - optional: false, - params: [ - cannonical_channel_nav[0], - is_badges ? cannonical_badges : undefined - ] - }; - } - - if (author) { - key_info.push([ 'author', author ]); - } - - return key_info.filter(([ key ]) => !excluded_keys.has(key)); - } - static #introspect(classdata: any) { - const key_info = this.#passOne(classdata); - return this.#passTwo(key_info); - } - /** - * Infer the type of a key given its value - * @param key - The key to infer the type of - * @param value - The value of the key - * @returns The inferred type - */ - static inferType(key: string, value: any): InferenceType { - let return_value: string | Record | boolean | MiscInferenceType = false; - if (return_value = this.isRenderer(value)) { - this.#renderers_examples[return_value] = value[Reflect.ownKeys(value)[0]]; +/** + * Infer the type of a key given its value + * @param key - The key to infer the type of + * @param value - The value of the key + * @returns The inferred type + */ +export function inferType(key: string, value: unknown): InferenceType { + let return_value: string | Record | boolean | MiscInferenceType = false; + if (typeof value === 'object' && value != null) { + if (return_value = isRenderer(value)) { + RENDERER_EXAMPLES[return_value] = Reflect.get(value, Reflect.ownKeys(value)[0]); return { type: 'renderer', renderers: [ return_value ], optional: false }; } - if (return_value = this.isRendererList(value)) { + if (return_value = isRendererList(value)) { for (const [ key, value ] of Object.entries(return_value)) { - this.#renderers_examples[key] = value; + RENDERER_EXAMPLES[key] = value; } return { type: 'renderer_list', @@ -605,89 +90,631 @@ export class YTNodeGenerator { optional: false }; } - if (return_value = this.isMiscType(key, value)) { + if (return_value = isMiscType(key, value)) { return return_value as MiscInferenceType; } - const primative_type = typeof value; - if (primative_type === 'object') - return { - type: 'object', - keys: Object.entries(value).map(([ key, value ]) => [ key, this.inferType(key, value) ]), - optional: false - }; + } + const primative_type = typeof value; + if (primative_type === 'object') return { - type: 'primative', - typeof: [ primative_type ], + type: 'object', + keys: Object.entries(value as object).map(([ key, value ]) => [ key, inferType(key, value) ]), + optional: false + }; + return { + type: 'primative', + typeof: [ primative_type ], + optional: false + }; +} + +/** + * Checks if the given value is an array of renderers + * @param value - The value to check + * @returns If it is a renderer list, return an object with keys being the classnames, and values being an example of that class. + * Otherwise, return false. + */ +export function isRendererList(value: unknown) { + const arr = Array.isArray(value); + const is_list = arr && value.every((item) => isRenderer(item)); + return ( + is_list ? + Object.fromEntries(value.map((item) => { + const key = Reflect.ownKeys(item)[0].toString(); + return [ Parser.sanitizeClassName(key), item[key] ]; + })) : + false + ); +} + +/** + * Check if the given value is a misc type. + * @param key - The key of the value + * @param value - The value to check + * @returns If it is a misc type, return the InferenceType. Otherwise, return false. + */ +export function isMiscType(key: string, value: unknown): MiscInferenceType | false { + // NavigationEndpoint + if ((key.endsWith('Endpoint') || key.endsWith('Command') || key === 'endpoint') && typeof value === 'object' && value !== null) { + return { + type: 'misc', + endpoint: new NavigationEndpoint(value), + optional: false, + misc_type: 'NavigationEndpoint' + }; + } + // Text + if (typeof value === 'object' && value !== null && (Reflect.has(value, 'simpleText') || Reflect.has(value, 'runs'))) { + const textNode = new Text(value); + return { + type: 'misc', + misc_type: 'Text', + optional: false, + endpoint: textNode.endpoint, + text: textNode.toString() + }; + } + // Thumbnail + if (typeof value === 'object' && value !== null && Reflect.has(value, 'thumbnails') && Array.isArray(Reflect.get(value, 'thumbnails'))) { + return { + type: 'misc', + misc_type: 'Thumbnail', optional: false }; } - /** - * Checks if the given value is an array of renderers - * @param value - The value to check - * @returns If it is a renderer list, return an object with keys being the classnames, and values being an example of that class. - * Otherwise, return false. - */ - static isRendererList(value: any) { - const arr = Array.isArray(value); - const is_list = arr && value.every((item) => this.isRenderer(item)); - return ( - is_list ? - Object.fromEntries(value.map((item) => { - const key = Reflect.ownKeys(item)[0].toString(); - return [ Parser.sanitizeClassName(key), item[key] ]; - })) : - false - ); + return false; +} + +/** + * Check if the given value is a renderer + * @param value - The value to check + * @returns If it is a renderer, return the class name. Otherwise, return false. + */ +export function isRenderer(value: unknown) { + const is_object = typeof value === 'object'; + if (!is_object) return false; + const keys = Reflect.ownKeys(value as object); + if (keys.length === 1 && keys[0].toString().includes('Renderer')) { + return Parser.sanitizeClassName(keys[0].toString()); } - /** - * Check if the given value is a misc type. - * @param key - The key of the value - * @param value - The value to check - * @returns If it is a misc type, return the InferenceType. Otherwise, return false. - */ - static isMiscType(key: string, value: any): MiscInferenceType | false { - // NavigationEndpoint - if ((key.endsWith('Endpoint') || key.endsWith('Command') || key === 'endpoint') && typeof value === 'object') { - return { - type: 'misc', - endpoint: new NavigationEndpoint(value), - optional: false, - misc_type: 'NavigationEndpoint' - }; - } - // Text - if (typeof value === 'object' && (Reflect.has(value, 'simpleText') || Reflect.has(value, 'runs'))) { - const textNode = new Text(value); - return { - type: 'misc', - misc_type: 'Text', - optional: false, - endpoint: textNode.endpoint, - text: textNode.toString() - }; - } - // Thumbnail - if (typeof value === 'object' && Reflect.has(value, 'thumbnails') && Array.isArray(value.thumbnails)) { - return { - type: 'misc', - misc_type: 'Thumbnail', - optional: false - }; - } - return false; + return false; +} + +function introspectKeysFirstPass(classdata: unknown): KeyInfo { + if (typeof classdata !== 'object' || classdata === null) { + throw new InnertubeError('Generator: Cannot introspect non-object', { + classdata + }); } - /** - * Check if the given value is a renderer - * @param value - The value to check - * @returns If it is a renderer, return the class name. Otherwise, return false. - */ - static isRenderer(value: any) { - const is_object = typeof value === 'object'; - if (!is_object) return false; - const keys = Reflect.ownKeys(value); - if (keys.length === 1 && keys[0].toString().includes('Renderer')) { - return Parser.sanitizeClassName(keys[0].toString()); - } - return false; + + const keys = Reflect.ownKeys(classdata) + .filter((key) => !isIgnoredKey(key)) + .filter((key): key is string => typeof key === 'string'); + + return keys.map((key) => { + const value = Reflect.get(classdata, key) as unknown; + const inferred_type = inferType(key, value); + return [ key, inferred_type ] as const; + }); +} + +function introspectKeysSecondPass(key_info: KeyInfo) { + // The second pass will detect Author + const channel_nav = key_info.filter(([ , value ]) => { + if (value.type !== 'misc') return false; + if (!(value.misc_type === 'NavigationEndpoint' || value.misc_type === 'Text')) return false; + return value.endpoint?.metadata.page_type === 'WEB_PAGE_TYPE_CHANNEL'; + }); + + // Whichever one has the longest text is the most probable match + const most_probable_match = channel_nav.sort(([ , a ], [ , b ]) => { + if (a.type !== 'misc' || b.type !== 'misc') return 0; + if (a.misc_type !== 'Text' || b.misc_type !== 'Text') return 0; + return b.text.length - a.text.length; + }); + + const excluded_keys = new Set(); + + const cannonical_channel_nav = most_probable_match[0]; + + let author: MiscInferenceType | undefined; + // We've found an author + if (cannonical_channel_nav) { + excluded_keys.add(cannonical_channel_nav[0]); + // Now to locate its metadata + // We'll first get all the keys in the classdata + const keys = key_info.map(([ key ]) => key); + // Check for anything ending in 'Badges' equals 'badges' + const badges = keys.filter((key) => key.endsWith('Badges') || key === 'badges'); + // The likely candidate is the one with some prefix (owner, author) + const likely_badges = badges.filter((key) => key.startsWith('owner') || key.startsWith('author')); + // If we have a likely candidate, we'll use that + const cannonical_badges = likely_badges[0] ?? badges[0]; + // Now we have the author and its badges + // Verify that its actually badges + const badge_key_info = key_info.find(([ key ]) => key === cannonical_badges); + const is_badges = badge_key_info ? + badge_key_info[1].type === 'renderer_list' && Reflect.has(badge_key_info[1].renderers, 'MetadataBadge') : + false; + + if (is_badges && cannonical_badges) excluded_keys.add(cannonical_badges); + // TODO: next we check for the author's thumbnail + author = { + type: 'misc', + misc_type: 'Author', + optional: false, + params: [ + cannonical_channel_nav[0], + is_badges ? cannonical_badges : undefined + ] + }; } -} \ No newline at end of file + + if (author) { + key_info.push([ 'author', author ]); + } + + return key_info.filter(([ key ]) => !excluded_keys.has(key)); +} + +function introspect2(classdata: unknown) { + const key_info = introspectKeysFirstPass(classdata); + return introspectKeysSecondPass(key_info); +} + +/** + * Introspect an example of a class in order to determine its key info and dependencies + * @param classdata - The example of the class + * @returns The key info and any unimplemented dependencies + */ +export function introspect(classdata: unknown) { + const key_info = introspect2(classdata); + const dependencies = new Map(); + for (const [ , value ] of key_info) { + if (value.type === 'renderer' || value.type === 'renderer_list') + for (const renderer of value.renderers) { + const example = RENDERER_EXAMPLES[renderer]; + if (example) + dependencies.set(renderer, example); + } + } + const unimplemented_dependencies = Array.from(dependencies).filter(([ classname ]) => !Parser.hasParser(classname)); + + return { + key_info, + unimplemented_dependencies + }; +} + +/** + * Is this key ignored by the parser? + * @param key - The key to check + * @returns Whether or not the key is ignored + */ +export function isIgnoredKey(key: string | symbol) { + return typeof key === 'string' && IGNORED_KEYS.has(key); +} + +/** + * Given a classname and its resolved key info, create a new class + * @param classname - The name of the class + * @param key_info - The resolved key info + * @returns Class based on the key info extending YTNode + */ +export function createRuntimeClass(classname: string, key_info: KeyInfo, logger: Parser.ParserErrorHandler): YTNodeConstructor { + logger({ + error_type: 'class_not_found', + classname, + key_info + }); + + const node = class extends YTNode { + static type = classname; + static #key_info = new Map(); + static set key_info(key_info: KeyInfo) { + this.#key_info = new Map(key_info); + } + static get key_info() { + return [ ...this.#key_info.entries() ]; + } + constructor(data: unknown) { + super(); + const { + key_info, + unimplemented_dependencies + } = introspect(data); + + const { + resolved_key_info, + changed_keys + } = mergeKeyInfo(node.key_info, key_info); + + const did_change = changed_keys.length > 0; + + if (did_change) { + node.key_info = resolved_key_info; + logger({ + error_type: 'class_changed', + classname, + key_info: node.key_info, + changed_keys + }); + } + + for (const [ name, data ] of unimplemented_dependencies) + generateRuntimeClass(name, data, logger); + + for (const [ key, value ] of key_info) { + let snake_key = camelToSnake(key); + if (value.type === 'misc' && value.misc_type === 'NavigationEndpoint') + snake_key = 'endpoint'; + Reflect.set(this, snake_key, parse(key, value, data)); + } + } + }; + node.key_info = key_info; + Object.defineProperty(node, 'name', { value: classname, writable: false }); + return node; +} + +/** + * Given example data for a class, introspect, implement dependencies, and create a new class + * @param classname - The name of the class + * @param classdata - The example of the class + * @returns Class based on the example classdata extending YTNode + */ +export function generateRuntimeClass(classname: string, classdata: unknown, logger: Parser.ParserErrorHandler) { + const { + key_info, + unimplemented_dependencies + } = introspect(classdata); + + const JITNode = createRuntimeClass(classname, key_info, logger); + Parser.addRuntimeParser(classname, JITNode); + + for (const [ name, data ] of unimplemented_dependencies) + generateRuntimeClass(name, data, logger); + + return JITNode; +} + +/** + * Generate a typescript class based on the key info + * @param classname - The name of the class + * @param key_info - The key info, as returned by {@link introspect} + * @returns Typescript class file + */ +export function generateTypescriptClass(classname: string, key_info: KeyInfo) { + const props: string[] = []; + const constructor_lines = [ + 'super();' + ]; + for (const [ key, value ] of key_info) { + let snake_key = camelToSnake(key); + if (value.type === 'misc' && value.misc_type === 'NavigationEndpoint') + snake_key = 'endpoint'; + props.push(`${snake_key}${value.optional ? '?' : ''}: ${toTypeDeclaration(value)};`); + constructor_lines.push(`this.${snake_key} = ${toParser(key, value)};`); + } + return `class ${classname} extends YTNode {\n static type = '${classname}';\n\n ${props.join('\n ')}\n\n constructor(data: RawNode) {\n ${constructor_lines.join('\n ')}\n }\n}\n`; +} + +/** + * For a given inference type, get the typescript type declaration + * @param inference_type - The inference type to get the declaration for + * @param indentation - The indentation level (used for objects) + * @returns Typescript type declaration + */ +export function toTypeDeclaration(inference_type: InferenceType, indentation = 0): string { + switch (inference_type.type) { + case 'renderer': + { + return `${inference_type.renderers.map((type) => `YTNodes.${type}`).join(' | ')} | null`; + } + case 'renderer_list': + { + return `ObservedArray<${inference_type.renderers.map((type) => `YTNodes.${type}`).join(' | ')}> | null`; + } + case 'object': + { + return `{\n${inference_type.keys.map(([ key, value ]) => `${' '.repeat((indentation + 2) * 2)}${camelToSnake(key)}${value.optional ? '?' : ''}: ${toTypeDeclaration(value, indentation + 1)}`).join(',\n')}\n${' '.repeat((indentation + 1) * 2)}}`; + } + case 'misc': + switch (inference_type.misc_type) { + case 'Thumbnail': + return 'Thumbnail[]'; + default: + return inference_type.misc_type; + } + case 'primative': + return inference_type.typeof.join(' | '); + case 'unknown': + return '/* TODO: determine correct type */ unknown'; + } +} + +/** + * Generate statements to parse a given inference type + * @param key - The key to parse + * @param inference_type - The inference type to parse + * @param key_path - The path to the key (excluding the key itself) + * @param indentation - The indentation level (used for objects) + * @returns Statement to parse the given key + */ +export function toParser(key: string, inference_type: InferenceType, key_path: string[] = [ 'data' ], indentation = 1) { + let parser = 'undefined'; + switch (inference_type.type) { + case 'renderer': + { + parser = `Parser.parseItem(${key_path.join('.')}.${key}, [ ${inference_type.renderers.map((type) => `YTNodes.${type}`).join(', ')} ])`; + } + break; + case 'renderer_list': + { + parser = `Parser.parse(${key_path.join('.')}.${key}, true, [ ${inference_type.renderers.map((type) => `YTNodes.${type}`).join(', ')} ])`; + } + break; + case 'object': + { + const new_keypath = [ ...key_path, key ]; + parser = `{\n${inference_type.keys.map(([ key, value ]) => `${' '.repeat((indentation + 2) * 2)}${camelToSnake(key)}: ${toParser(key, value, new_keypath, indentation + 1)}`).join(',\n')}\n${' '.repeat((indentation + 1) * 2)}}`; + } + break; + case 'misc': + switch (inference_type.misc_type) { + case 'Thumbnail': + parser = `Thumbnail.fromResponse(${key_path.join('.')}.${key})`; + break; + case 'Author': + { + const author_parser = `new Author(${key_path.join('.')}.${inference_type.params[0]}, ${inference_type.params[1] ? `${key_path.join('.')}.${inference_type.params[1]}` : 'undefined'})`; + if (inference_type.optional) + return `Reflect.has(${key_path.join('.')}, '${inference_type.params[0]}') ? ${author_parser} : undefined`; + return author_parser; + } + default: + parser = `new ${inference_type.misc_type}(${key_path.join('.')}.${key})`; + break; + } + if (parser === 'undefined') + throw new Error('Unreachable code reached! Switch missing case!'); + break; + case 'primative': + case 'unknown': + parser = `${key_path.join('.')}.${key}`; + break; + } + if (inference_type.optional) + return `Reflect.has(${key_path.join('.')}, '${key}') ? ${parser} : undefined`; + return parser; +} + +function accessDataFromKeyPath(root: any, key_path: string[]) { + let data = root; + for (const key of key_path) + data = data[key]; + return data; +} + +function hasDataFromKeyPath(root: any, key_path: string[]) { + let data = root; + for (const key of key_path) + if (!Reflect.has(data, key)) + return false; + else + data = data[key]; + return true; +} + +/** + * Parse a value from a given key path using the given inference type + * @param key - The key to parse + * @param inference_type - The inference type to parse + * @param data - The data to parse from + * @param key_path - The path to the key (excluding the key itself) + * @returns The parsed value + */ +export function parse(key: string, inference_type: InferenceType, data: unknown, key_path: string[] = [ 'data' ]) { + const should_optional = !inference_type.optional || hasDataFromKeyPath({ data }, [ ...key_path, key ]); + switch (inference_type.type) { + case 'renderer': + { + return should_optional ? Parser.parseItem(accessDataFromKeyPath({ data }, [ ...key_path, key ]), inference_type.renderers.map((type) => Parser.getParserByName(type))) : undefined; + } + case 'renderer_list': + { + return should_optional ? Parser.parse(accessDataFromKeyPath({ data }, [ ...key_path, key ]), true, inference_type.renderers.map((type) => Parser.getParserByName(type))) : undefined; + } + case 'object': + { + const obj: any = {}; + const new_key_path = [ ...key_path, key ]; + for (const [ key, value ] of inference_type.keys) { + obj[key] = should_optional ? parse(key, value, data, new_key_path) : undefined; + } + return obj; + } + case 'misc': + switch (inference_type.misc_type) { + case 'NavigationEndpoint': + return should_optional ? new NavigationEndpoint(accessDataFromKeyPath({ data }, [ ...key_path, key ])) : undefined; + case 'Text': + return should_optional ? new Text(accessDataFromKeyPath({ data }, [ ...key_path, key ])) : undefined; + case 'Thumbnail': + return should_optional ? Thumbnail.fromResponse(accessDataFromKeyPath({ data }, [ ...key_path, key ])) : undefined; + case 'Author': + { + const author_should_optional = !inference_type.optional || hasDataFromKeyPath({ data }, [ ...key_path, inference_type.params[0] ]); + return author_should_optional ? new Author( + accessDataFromKeyPath({ data }, [ ...key_path, inference_type.params[0] ]), + inference_type.params[1] ? + accessDataFromKeyPath({ data }, [ ...key_path, inference_type.params[1] ]) : undefined + ) : undefined; + } + } + throw new Error('Unreachable code reached! Switch missing case!'); + case 'primative': + case 'unknown': + return accessDataFromKeyPath({ data }, [ ...key_path, key ]); + } +} + +/** + * Merges two sets of key info, resolving any conflicts + * @param key_info - The current key info + * @param new_key_info - The new key info + * @returns The merged key info + */ +export function mergeKeyInfo(key_info: KeyInfo, new_key_info: KeyInfo) { + const changed_keys = new Map(); + const current_keys = new Set(key_info.map(([ key ]) => key)); + const new_keys = new Set(new_key_info.map(([ key ]) => key)); + + const added_keys = new_key_info.filter(([ key ]) => !current_keys.has(key)); + const removed_keys = key_info.filter(([ key ]) => !new_keys.has(key)); + + const common_keys = key_info.filter(([ key ]) => new_keys.has(key)); + + const new_key_map = new Map(new_key_info); + + for (const [ key, type ] of common_keys) { + const new_type = new_key_map.get(key); + if (!new_type) continue; + if (type.type !== new_type.type) { + // We've got a type mismatch, this is unknown, we do not resolve unions + changed_keys.set(key, { + type: 'unknown', + optional: true + }); + continue; + } + // We've got the same type, so we can now resolve the changes + switch (type.type) { + case 'object': + { + if (new_type.type !== 'object') continue; + const { resolved_key_info } = mergeKeyInfo(type.keys, new_type.keys); + const resolved_key: InferenceType = { + type: 'object', + keys: resolved_key_info, + optional: type.optional || new_type.optional + }; + const did_change = JSON.stringify(resolved_key) !== JSON.stringify(type); + if (did_change) changed_keys.set(key, resolved_key); + } + break; + case 'renderer': + { + if (new_type.type !== 'renderer') continue; + const union_map = { + ...type.renderers, + ...new_type.renderers + }; + const either_optional = type.optional || new_type.optional; + const resolved_key: InferenceType = { + type: 'renderer', + renderers: union_map, + optional: either_optional + }; + const did_change = JSON.stringify({ + ...resolved_key, + renderers: Object.keys(resolved_key.renderers) + }) !== JSON.stringify({ + ...type, + renderers: Object.keys(type.renderers) + }); + if (did_change) changed_keys.set(key, resolved_key); + } + break; + case 'renderer_list': + { + if (new_type.type !== 'renderer_list') continue; + const union_map = { + ...type.renderers, + ...new_type.renderers + }; + const either_optional = type.optional || new_type.optional; + const resolved_key: InferenceType = { + type: 'renderer_list', + renderers: union_map, + optional: either_optional + }; + const did_change = JSON.stringify({ + ...resolved_key, + renderers: Object.keys(resolved_key.renderers) + }) !== JSON.stringify({ + ...type, + renderers: Object.keys(type.renderers) + }); + if (did_change) changed_keys.set(key, resolved_key); + } + break; + case 'misc': + { + if (new_type.type !== 'misc') continue; + if (type.misc_type !== new_type.misc_type) { + // We've got a type mismatch, this is unknown, we do not resolve unions + changed_keys.set(key, { + type: 'unknown', + optional: true + }); + } + switch (type.misc_type) { + case 'Author': + { + if (new_type.misc_type !== 'Author') break; + const had_optional_param = type.params[1] || new_type.params[1]; + const either_optional = type.optional || new_type.optional; + const resolved_key: MiscInferenceType = { + type: 'misc', + misc_type: 'Author', + optional: either_optional, + params: [ new_type.params[0], had_optional_param ] + }; + const did_change = JSON.stringify(resolved_key) !== JSON.stringify(type); + if (did_change) changed_keys.set(key, resolved_key); + } + break; + // Other cases can not change + } + } + break; + case 'primative': + { + if (new_type.type !== 'primative') continue; + const resolved_key: InferenceType = { + type: 'primative', + typeof: Array.from(new Set([ ...new_type.typeof, ...type.typeof ])), + optional: type.optional || new_type.optional + }; + const did_change = JSON.stringify(resolved_key) !== JSON.stringify(type); + if (did_change) changed_keys.set(key, resolved_key); + } + break; + } + } + + for (const [ key, type ] of added_keys) { + changed_keys.set(key, { + ...type, + optional: true + }); + } + + for (const [ key, type ] of removed_keys) { + changed_keys.set(key, { + ...type, + optional: true + }); + } + + const unchanged_keys = key_info.filter(([ key ]) => !changed_keys.has(key)); + + const resolved_key_info_map = new Map([ ...unchanged_keys, ...changed_keys ]); + const resolved_key_info = [ ...resolved_key_info_map.entries() ]; + + return { + resolved_key_info, + changed_keys: [ ...changed_keys.entries() ] + }; +} diff --git a/deno/src/parser/index.ts b/deno/src/parser/index.ts index 50733e07..0fcb2cb6 100644 --- a/deno/src/parser/index.ts +++ b/deno/src/parser/index.ts @@ -1,5 +1,5 @@ -export { default as Parser } from './parser.ts'; -export * from './parser.ts'; +export * as Parser from './parser.ts'; +export * from './continuations.ts'; export * from './types/index.ts'; export * as Misc from './misc.ts'; export * as YTNodes from './nodes.ts'; @@ -8,5 +8,5 @@ export * as YTMusic from './ytmusic/index.ts'; export * as YTKids from './ytkids/index.ts'; export * as Helpers from './helpers.ts'; export * as Generator from './generator.ts'; -import Parser from './parser.ts'; +import * as Parser from './parser.ts'; export default Parser; \ No newline at end of file diff --git a/deno/src/parser/nodes.ts b/deno/src/parser/nodes.ts index b0992caf..26f7fdce 100644 --- a/deno/src/parser/nodes.ts +++ b/deno/src/parser/nodes.ts @@ -223,6 +223,7 @@ export { default as MusicImmersiveHeader } from './classes/MusicImmersiveHeader. export { default as MusicInlineBadge } from './classes/MusicInlineBadge.ts'; export { default as MusicItemThumbnailOverlay } from './classes/MusicItemThumbnailOverlay.ts'; export { default as MusicLargeCardItemCarousel } from './classes/MusicLargeCardItemCarousel.ts'; +export { default as MusicMultiRowListItem } from './classes/MusicMultiRowListItem.ts'; export { default as MusicNavigationButton } from './classes/MusicNavigationButton.ts'; export { default as MusicPlayButton } from './classes/MusicPlayButton.ts'; export { default as MusicPlaylistShelf } from './classes/MusicPlaylistShelf.ts'; diff --git a/deno/src/parser/parser.ts b/deno/src/parser/parser.ts index d2f9dd82..d879bea4 100644 --- a/deno/src/parser/parser.ts +++ b/deno/src/parser/parser.ts @@ -5,10 +5,6 @@ import PlayerAnnotationsExpanded from './classes/PlayerAnnotationsExpanded.ts'; import PlayerCaptionsTracklist from './classes/PlayerCaptionsTracklist.ts'; import PlayerLiveStoryboardSpec from './classes/PlayerLiveStoryboardSpec.ts'; import PlayerStoryboardSpec from './classes/PlayerStoryboardSpec.ts'; -import Message from './classes/Message.ts'; -import LiveChatParticipantsList from './classes/LiveChatParticipantsList.ts'; -import LiveChatHeader from './classes/LiveChatHeader.ts'; -import LiveChatItemList from './classes/LiveChatItemList.ts'; import Alert from './classes/Alert.ts'; import type { IParsedResponse, IRawResponse, RawData, RawNode } from './types/index.ts'; @@ -17,749 +13,621 @@ import MusicMultiSelectMenuItem from './classes/menus/MusicMultiSelectMenuItem.t import Format from './classes/misc/Format.ts'; import VideoDetails from './classes/misc/VideoDetails.ts'; import NavigationEndpoint from './classes/NavigationEndpoint.ts'; -import Thumbnail from './classes/misc/Thumbnail.ts'; import { InnertubeError, ParsingError, Platform } from '../utils/Utils.ts'; -import type { ObservedArray, YTNodeConstructor } from './helpers.ts'; -import { Memo, observe, SuperParsedResult, YTNode } from './helpers.ts'; +import type { ObservedArray, YTNodeConstructor, YTNode } from './helpers.ts'; +import { Memo, observe, SuperParsedResult } from './helpers.ts'; import * as YTNodes from './nodes.ts'; -import { YTNodeGenerator } from './generator.ts'; +import type { KeyInfo } from './generator.ts'; +import { camelToSnake, generateRuntimeClass, generateTypescriptClass } from './generator.ts'; +import { Continuation, ItemSectionContinuation, SectionListContinuation, LiveChatContinuation, MusicPlaylistShelfContinuation, MusicShelfContinuation, GridContinuation, PlaylistPanelContinuation, NavigateAction, ShowMiniplayerCommand, ReloadContinuationItemsCommand } from './continuations.ts'; + +export type ParserError = { + classname: string, +} & ({ + error_type: 'typecheck', + classdata: RawNode, + expected: string | string[] +} | { + error_type: 'parse', + classdata: RawNode, + error: unknown +} | { + error_type: 'mutation_data_missing' +} | { + error_type: 'mutation_data_invalid', + total: number, + failed: number, + titles: string[] +} | { + error_type: 'class_not_found', + key_info: KeyInfo, +} | { + error_type: 'class_changed', + key_info: KeyInfo, + changed_keys: KeyInfo +}); -export type ParserError = { classname: string, classdata: any, err: any }; export type ParserErrorHandler = (error: ParserError) => void; -export default class Parser { - static #errorHandler: ParserErrorHandler = Parser.#printError; - static #memo: Memo | null = null; +const IGNORED_LIST = new Set([ + 'AdSlot', + 'DisplayAd', + 'SearchPyv', + 'MealbarPromo', + 'PrimetimePromo', + 'BackgroundPromo', + 'PromotedSparklesWeb', + 'RunAttestationCommand', + 'CompactPromotedVideo', + 'BrandVideoShelf', + 'BrandVideoSingleton', + 'StatementBanner', + 'GuideSigninPromo', + 'AdsEngagementPanelContent' +]); - static setParserErrorHandler(handler: ParserErrorHandler) { - this.#errorHandler = handler; - } +const RUNTIME_NODES = new Map(Object.entries(YTNodes)); - static #clearMemo() { - Parser.#memo = null; - } +const DYNAMIC_NODES = new Map(); - static #createMemo() { - Parser.#memo = new Memo(); - } +let MEMO: Memo | null = null; - static #addToMemo(classname: string, result: YTNode) { - if (!Parser.#memo) - return; - - const list = Parser.#memo.get(classname); - if (!list) - return Parser.#memo.set(classname, [ result ]); - - list.push(result); - } - - static #getMemo() { - if (!Parser.#memo) - throw new Error('Parser#getMemo() called before Parser#createMemo()'); - return Parser.#memo; - } - - /** - * Parses given InnerTube response. - * @param data - Raw data. - */ - static parseResponse(data: IRawResponse): T { - const parsed_data = {} as T; - - this.#createMemo(); - const contents = this.parse(data.contents); - const contents_memo = this.#getMemo(); - if (contents) { - parsed_data.contents = contents; - parsed_data.contents_memo = contents_memo; - } - this.#clearMemo(); - - this.#createMemo(); - const on_response_received_actions = data.onResponseReceivedActions ? this.parseRR(data.onResponseReceivedActions) : null; - const on_response_received_actions_memo = this.#getMemo(); - if (on_response_received_actions) { - parsed_data.on_response_received_actions = on_response_received_actions; - parsed_data.on_response_received_actions_memo = on_response_received_actions_memo; - } - this.#clearMemo(); - - this.#createMemo(); - const on_response_received_endpoints = data.onResponseReceivedEndpoints ? this.parseRR(data.onResponseReceivedEndpoints) : null; - const on_response_received_endpoints_memo = this.#getMemo(); - if (on_response_received_endpoints) { - parsed_data.on_response_received_endpoints = on_response_received_endpoints; - parsed_data.on_response_received_endpoints_memo = on_response_received_endpoints_memo; - } - this.#clearMemo(); - - this.#createMemo(); - const on_response_received_commands = data.onResponseReceivedCommands ? this.parseRR(data.onResponseReceivedCommands) : null; - const on_response_received_commands_memo = this.#getMemo(); - if (on_response_received_commands) { - parsed_data.on_response_received_commands = on_response_received_commands; - parsed_data.on_response_received_commands_memo = on_response_received_commands_memo; - } - this.#clearMemo(); - - this.#createMemo(); - const continuation_contents = data.continuationContents ? this.parseLC(data.continuationContents) : null; - const continuation_contents_memo = this.#getMemo(); - if (continuation_contents) { - parsed_data.continuation_contents = continuation_contents; - parsed_data.continuation_contents_memo = continuation_contents_memo; - } - this.#clearMemo(); - - this.#createMemo(); - const actions = data.actions ? this.parseActions(data.actions) : null; - const actions_memo = this.#getMemo(); - if (actions) { - parsed_data.actions = actions; - parsed_data.actions_memo = actions_memo; - } - this.#clearMemo(); - - this.#createMemo(); - const live_chat_item_context_menu_supported_renderers = data.liveChatItemContextMenuSupportedRenderers ? this.parseItem(data.liveChatItemContextMenuSupportedRenderers) : null; - const live_chat_item_context_menu_supported_renderers_memo = this.#getMemo(); - if (live_chat_item_context_menu_supported_renderers) { - parsed_data.live_chat_item_context_menu_supported_renderers = live_chat_item_context_menu_supported_renderers; - parsed_data.live_chat_item_context_menu_supported_renderers_memo = live_chat_item_context_menu_supported_renderers_memo; - } - this.#clearMemo(); - - this.#createMemo(); - const header = data.header ? this.parse(data.header) : null; - const header_memo = this.#getMemo(); - if (header) { - parsed_data.header = header; - parsed_data.header_memo = header_memo; - } - this.#clearMemo(); - - this.#createMemo(); - const sidebar = data.sidebar ? this.parseItem(data.sidebar) : null; - const sidebar_memo = this.#getMemo(); - if (sidebar) { - parsed_data.sidebar = sidebar; - parsed_data.sidebar_memo = sidebar_memo; - } - this.#clearMemo(); - - this.applyMutations(contents_memo, data.frameworkUpdates?.entityBatchUpdate?.mutations); - - const continuation = data.continuation ? this.parseC(data.continuation) : null; - if (continuation) { - parsed_data.continuation = continuation; - } - - const metadata = this.parse(data.metadata); - if (metadata) { - parsed_data.metadata = metadata; - } - - const microformat = this.parseItem(data.microformat); - if (microformat) { - parsed_data.microformat = microformat; - } - - const overlay = this.parseItem(data.overlay); - if (overlay) { - parsed_data.overlay = overlay; - } - - const alerts = this.parseArray(data.alerts, Alert); - if (alerts.length) { - parsed_data.alerts = alerts; - } - - const refinements = data.refinements; - if (refinements) { - parsed_data.refinements = refinements; - } - - const estimated_results = data.estimatedResults ? parseInt(data.estimatedResults) : null; - if (estimated_results) { - parsed_data.estimated_results = estimated_results; - } - - const player_overlays = this.parse(data.playerOverlays); - if (player_overlays) { - parsed_data.player_overlays = player_overlays; - } - - const playback_tracking = data.playbackTracking ? { - videostats_watchtime_url: data.playbackTracking.videostatsWatchtimeUrl.baseUrl, - videostats_playback_url: data.playbackTracking.videostatsPlaybackUrl.baseUrl - } : null; - - if (playback_tracking) { - parsed_data.playback_tracking = playback_tracking; - } - - const playability_status = data.playabilityStatus ? { - status: data.playabilityStatus.status, - reason: data.playabilityStatus.reason || '', - embeddable: !!data.playabilityStatus.playableInEmbed || false, - audio_only_playablility: this.parseItem(data.playabilityStatus.audioOnlyPlayability, AudioOnlyPlayability), - error_screen: this.parseItem(data.playabilityStatus.errorScreen) - } : null; - - if (playability_status) { - parsed_data.playability_status = playability_status; - } - - const streaming_data = data.streamingData ? { - expires: new Date(Date.now() + parseInt(data.streamingData.expiresInSeconds) * 1000), - formats: Parser.parseFormats(data.streamingData.formats), - adaptive_formats: Parser.parseFormats(data.streamingData.adaptiveFormats), - dash_manifest_url: data.streamingData.dashManifestUrl || null, - hls_manifest_url: data.streamingData.hlsManifestUrl || null - } : undefined; - - if (streaming_data) { - parsed_data.streaming_data = streaming_data; - } - - const current_video_endpoint = data.currentVideoEndpoint ? new NavigationEndpoint(data.currentVideoEndpoint) : null; - if (current_video_endpoint) { - parsed_data.current_video_endpoint = current_video_endpoint; - } - - const endpoint = data.endpoint ? new NavigationEndpoint(data.endpoint) : null; - if (endpoint) { - parsed_data.endpoint = endpoint; - } - - const captions = this.parseItem(data.captions, PlayerCaptionsTracklist); - if (captions) { - parsed_data.captions = captions; - } - - const video_details = data.videoDetails ? new VideoDetails(data.videoDetails) : null; - if (video_details) { - parsed_data.video_details = video_details; - } - - const annotations = this.parseArray(data.annotations, PlayerAnnotationsExpanded); - if (annotations.length) { - parsed_data.annotations = annotations; - } - - const storyboards = this.parseItem(data.storyboards, [ PlayerStoryboardSpec, PlayerLiveStoryboardSpec ]); - if (storyboards) { - parsed_data.storyboards = storyboards; - } - - const endscreen = this.parseItem(data.endscreen, Endscreen); - if (endscreen) { - parsed_data.endscreen = endscreen; - } - - const cards = this.parseItem(data.cards, CardCollection); - if (cards) { - parsed_data.cards = cards; - } - - const engagement_panels = data.engagementPanels?.map((e) => { - const item = this.parseItem(e, YTNodes.EngagementPanelSectionList) as YTNodes.EngagementPanelSectionList; - return item; - }); - if (engagement_panels) { - parsed_data.engagement_panels = engagement_panels; - } - this.#createMemo(); - const items = this.parse(data.items); - if (items) { - parsed_data.items = items; - parsed_data.items_memo = this.#getMemo(); - } - this.#clearMemo(); - - return parsed_data; - } - - /** - * Parses a single item. - * @param data - The data to parse. - * @param validTypes - YTNode types that are allowed to be parsed. - */ - static parseItem[]>(data: RawNode | undefined, validTypes: K): InstanceType | null; - static parseItem(data: RawNode | undefined, validTypes: YTNodeConstructor): T | null; - static parseItem(data?: RawNode) : YTNode; - static parseItem(data?: RawNode, validTypes?: YTNodeConstructor | YTNodeConstructor[]) { - if (!data) return null; - - const keys = Object.keys(data); - - if (!keys.length) - return null; - - const classname = this.sanitizeClassName(keys[0]); - - if (!this.shouldIgnore(classname)) { - try { - const has_target_class = this.hasParser(classname); - - const TargetClass = has_target_class ? this.getParserByName(classname) : YTNodeGenerator.generateRuntimeClass(classname, data[keys[0]]); - - if (validTypes) { - if (Array.isArray(validTypes)) { - if (!validTypes.some((type) => type.type === TargetClass.type)) - throw new ParsingError(`Type mismatch, got ${classname} but expected one of ${validTypes.map((type) => type.type).join(', ')}`); - } else if (TargetClass.type !== validTypes.type) - throw new ParsingError(`Type mismatch, got ${classname} but expected ${validTypes.type}`); - } - - const result = new TargetClass(data[keys[0]]); - this.#addToMemo(classname, result); - - return result; - } catch (err) { - this.#errorHandler({ classname, classdata: data[keys[0]], err }); - return null; +let ERROR_HANDLER: ParserErrorHandler = ({ classname, ...context }: ParserError) => { + switch (context.error_type) { + case 'parse': + if (context.error instanceof Error) { + console.warn( + new InnertubeError( + `Something went wrong at ${classname}!\n` + + `This is a bug, please report it at ${Platform.shim.info.bugs_url}`, { + stack: context.error.stack + } + ) + ); } - } - - return null; - } - - /** - * Parses an array of items. - * @param data - The data to parse. - * @param validTypes - YTNode types that are allowed to be parsed. - */ - static parseArray[]>(data: RawNode[] | undefined, validTypes: K): ObservedArray>; - static parseArray(data: RawNode[] | undefined, validType: YTNodeConstructor): ObservedArray; - static parseArray(data: RawNode[] | undefined): ObservedArray; - static parseArray(data?: RawNode[], validTypes?: YTNodeConstructor | YTNodeConstructor[]) { - if (Array.isArray(data)) { - const results: YTNode[] = []; - - for (const item of data) { - const result = this.parseItem(item, validTypes as YTNodeConstructor); - if (result) { - results.push(result); - } - } - - return observe(results); - } else if (!data) { - return observe([] as YTNode[]); - } - throw new ParsingError('Expected array but got a single item'); - } - - /** - * Parses an item or an array of items. - * @param data - The data to parse. - * @param requireArray - Whether the data should be parsed as an array. - * @param validTypes - YTNode types that are allowed to be parsed. - */ - static parse[]>(data: RawData, requireArray: true, validTypes?: K): ObservedArray> | null; - static parse(data?: RawData, requireArray?: false | undefined, validTypes?: YTNodeConstructor | YTNodeConstructor[]): SuperParsedResult; - static parse(data?: RawData, requireArray?: boolean, validTypes?: YTNodeConstructor | YTNodeConstructor[]) { - if (!data) return null; - - if (Array.isArray(data)) { - const results: T[] = []; - - for (const item of data) { - const result = this.parseItem(item, validTypes as YTNodeConstructor); - if (result) { - results.push(result); - } - } - - const res = observe(results); - - return requireArray ? res : new SuperParsedResult(observe(results)); - } else if (requireArray) { - throw new ParsingError('Expected array but got a single item'); - } - - return new SuperParsedResult(this.parseItem(data, validTypes as YTNodeConstructor)); - } - - static parseC(data: RawNode) { - if (data.timedContinuationData) - return new Continuation({ continuation: data.timedContinuationData, type: 'timed' }); - return null; - } - - static parseLC(data: RawNode) { - if (data.itemSectionContinuation) - return new ItemSectionContinuation(data.itemSectionContinuation); - if (data.sectionListContinuation) - return new SectionListContinuation(data.sectionListContinuation); - if (data.liveChatContinuation) - return new LiveChatContinuation(data.liveChatContinuation); - if (data.musicPlaylistShelfContinuation) - return new MusicPlaylistShelfContinuation(data.musicPlaylistShelfContinuation); - if (data.musicShelfContinuation) - return new MusicShelfContinuation(data.musicShelfContinuation); - if (data.gridContinuation) - return new GridContinuation(data.gridContinuation); - if (data.playlistPanelContinuation) - return new PlaylistPanelContinuation(data.playlistPanelContinuation); - - return null; - } - - static parseRR(actions: RawNode[]) { - return observe(actions.map((action: any) => { - if (action.navigateAction) - return new NavigateAction(action.navigateAction); - if (action.showMiniplayerCommand) - return new ShowMiniplayerCommand(action.showMiniplayerCommand); - if (action.reloadContinuationItemsCommand) - return new ReloadContinuationItemsCommand(action.reloadContinuationItemsCommand); - if (action.appendContinuationItemsAction) - return new AppendContinuationItemsAction(action.appendContinuationItemsAction); - }).filter((item) => item) as (ReloadContinuationItemsCommand | AppendContinuationItemsAction)[]); - } - - static parseActions(data: RawData) { - if (Array.isArray(data)) { - return Parser.parse(data.map((action) => { - delete action.clickTrackingParams; - return action; - })); - } - return new SuperParsedResult(this.parseItem(data)); - } - - static parseFormats(formats: RawNode[]): Format[] { - return formats?.map((format) => new Format(format)) || []; - } - - static applyMutations(memo: Memo, mutations: RawNode[]) { - // Apply mutations to MusicMultiSelectMenuItems - const music_multi_select_menu_items = memo.getType(MusicMultiSelectMenuItem); - - if (music_multi_select_menu_items.length > 0 && !mutations) { + break; + case 'typecheck': + console.warn( + new ParsingError( + `Type mismatch, got ${classname} expected ${Array.isArray(context.expected) ? context.expected.join(' | ') : context.expected}.`, + context.classdata + ) + ); + break; + case 'mutation_data_missing': console.warn( new InnertubeError( 'Mutation data required for processing MusicMultiSelectMenuItems, but none found.\n' + `This is a bug, please report it at ${Platform.shim.info.bugs_url}` ) ); - } else { - const missing_or_invalid_mutations = []; - - for (const menu_item of music_multi_select_menu_items) { - const mutation = mutations - .find((mutation) => mutation.payload?.musicFormBooleanChoice?.id === menu_item.form_item_entity_key); - - const choice = mutation?.payload.musicFormBooleanChoice; - - if (choice?.selected !== undefined && choice?.opaqueToken) { - menu_item.selected = choice.selected; - } else { - missing_or_invalid_mutations.push(`'${menu_item.title}'`); - } - } - if (missing_or_invalid_mutations.length > 0) { - console.warn( - new InnertubeError( - `Mutation data missing or invalid for ${missing_or_invalid_mutations.length} out of ${music_multi_select_menu_items.length} MusicMultiSelectMenuItems. ` + - `The titles of the failed items are: ${missing_or_invalid_mutations.join(', ')}.\n` + - `This is a bug, please report it at ${Platform.shim.info.bugs_url}` - ) - ); - } - } - } - - static #printError({ classname, classdata, err }: ParserError) { - if (err.code == 'MODULE_NOT_FOUND') { - return console.warn( + break; + case 'mutation_data_invalid': + console.warn( new InnertubeError( - `${classname} not found!\n` + - `This is a bug, want to help us fix it? Follow the instructions at ${Platform.shim.info.repo_url.split('#')[0]}/blob/main/docs/updating-the-parser.md or report it at ${Platform.shim.info.bugs_url}!`, classdata + `Mutation data missing or invalid for ${context.failed} out of ${context.total} MusicMultiSelectMenuItems. ` + + `The titles of the failed items are: ${context.titles.join(', ')}.\n` + + `This is a bug, please report it at ${Platform.shim.info.bugs_url}` ) ); - } - - console.warn( - new InnertubeError( - `Something went wrong at ${classname}!\n` + - `This is a bug, please report it at ${Platform.shim.info.bugs_url}`, { stack: err.stack } - ) - ); + break; + case 'class_not_found': + console.warn( + new InnertubeError( + `${classname} not found!\n` + + `This is a bug, want to help us fix it? Follow the instructions at ${Platform.shim.info.repo_url}/blob/main/docs/updating-the-parser.md or report it at ${Platform.shim.info.bugs_url}!\n` + + `Introspected and JIT generated this class in the meantime:\n${generateTypescriptClass(classname, context.key_info)}` + ) + ); + break; + case 'class_changed': + console.warn( + `${classname} changed!\n` + + `The following keys where altered: ${context.changed_keys.map(([ key ]) => camelToSnake(key)).join(', ')}\n` + + `The class has changed to:\n${generateTypescriptClass(classname, context.key_info)}` + ); + break; + default: + console.warn( + 'Unreachable code reached at ParserErrorHandler' + ); + break; } +}; - static sanitizeClassName(input: string) { - return (input.charAt(0).toUpperCase() + input.slice(1)) - .replace(/Renderer|Model/g, '') - .replace(/Radio/g, 'Mix').trim(); - } - - static ignore_list = new Set([ - 'AdSlot', - 'DisplayAd', - 'SearchPyv', - 'MealbarPromo', - 'PrimetimePromo', - 'BackgroundPromo', - 'PromotedSparklesWeb', - 'RunAttestationCommand', - 'CompactPromotedVideo', - 'BrandVideoShelf', - 'BrandVideoSingleton', - 'StatementBanner', - 'GuideSigninPromo', - 'AdsEngagementPanelContent' - ]); - - static shouldIgnore(classname: string) { - return this.ignore_list.has(classname); - } - - static #rt_nodes = new Map(Object.entries(YTNodes)); - static #dynamic_nodes = new Map(); - - static getParserByName(classname: string) { - const ParserConstructor = this.#rt_nodes.get(classname); - - if (!ParserConstructor) { - const error = new Error(`Module not found: ${classname}`); - (error as any).code = 'MODULE_NOT_FOUND'; - throw error; - } - - return ParserConstructor; - } - - static hasParser(classname: string) { - return this.#rt_nodes.has(classname); - } - - static addRuntimeParser(classname: string, ParserConstructor: YTNodeConstructor) { - this.#rt_nodes.set(classname, ParserConstructor); - this.#dynamic_nodes.set(classname, ParserConstructor); - } - - static getDynamicParsers() { - return Object.fromEntries(this.#dynamic_nodes); - } +export function setParserErrorHandler(handler: ParserErrorHandler) { + ERROR_HANDLER = handler; } -// Continuation +function _clearMemo() { + MEMO = null; +} -export class ItemSectionContinuation extends YTNode { - static readonly type = 'itemSectionContinuation'; +function _createMemo() { + MEMO = new Memo(); +} - contents: ObservedArray | null; - continuation?: string; +function _addToMemo(classname: string, result: YTNode) { + if (!MEMO) + return; - constructor(data: RawNode) { - super(); - this.contents = Parser.parseArray(data.contents); - if (Array.isArray(data.continuations)) { - this.continuation = data.continuations?.at(0)?.nextContinuationData?.continuation; + const list = MEMO.get(classname); + if (!list) + return MEMO.set(classname, [ result ]); + + list.push(result); +} + +function _getMemo() { + if (!MEMO) + throw new Error('Parser#getMemo() called before Parser#createMemo()'); + return MEMO; +} + +export function shouldIgnore(classname: string) { + return IGNORED_LIST.has(classname); +} + +export function sanitizeClassName(input: string) { + return (input.charAt(0).toUpperCase() + input.slice(1)) + .replace(/Renderer|Model/g, '') + .replace(/Radio/g, 'Mix').trim(); +} + +export function getParserByName(classname: string) { + const ParserConstructor = RUNTIME_NODES.get(classname); + + if (!ParserConstructor) { + const error = new Error(`Module not found: ${classname}`); + (error as any).code = 'MODULE_NOT_FOUND'; + throw error; + } + + return ParserConstructor; +} + +export function hasParser(classname: string) { + return RUNTIME_NODES.has(classname); +} + +export function addRuntimeParser(classname: string, ParserConstructor: YTNodeConstructor) { + RUNTIME_NODES.set(classname, ParserConstructor); + DYNAMIC_NODES.set(classname, ParserConstructor); +} + +export function getDynamicParsers() { + return Object.fromEntries(DYNAMIC_NODES); +} + +/** + * Parses given InnerTube response. + * @param data - Raw data. + */ +export function parseResponse(data: IRawResponse): T { + const parsed_data = {} as T; + + _createMemo(); + const contents = parse(data.contents); + const contents_memo = _getMemo(); + if (contents) { + parsed_data.contents = contents; + parsed_data.contents_memo = contents_memo; + } + _clearMemo(); + + _createMemo(); + const on_response_received_actions = data.onResponseReceivedActions ? parseRR(data.onResponseReceivedActions) : null; + const on_response_received_actions_memo = _getMemo(); + if (on_response_received_actions) { + parsed_data.on_response_received_actions = on_response_received_actions; + parsed_data.on_response_received_actions_memo = on_response_received_actions_memo; + } + _clearMemo(); + + _createMemo(); + const on_response_received_endpoints = data.onResponseReceivedEndpoints ? parseRR(data.onResponseReceivedEndpoints) : null; + const on_response_received_endpoints_memo = _getMemo(); + if (on_response_received_endpoints) { + parsed_data.on_response_received_endpoints = on_response_received_endpoints; + parsed_data.on_response_received_endpoints_memo = on_response_received_endpoints_memo; + } + _clearMemo(); + + _createMemo(); + const on_response_received_commands = data.onResponseReceivedCommands ? parseRR(data.onResponseReceivedCommands) : null; + const on_response_received_commands_memo = _getMemo(); + if (on_response_received_commands) { + parsed_data.on_response_received_commands = on_response_received_commands; + parsed_data.on_response_received_commands_memo = on_response_received_commands_memo; + } + _clearMemo(); + + _createMemo(); + const continuation_contents = data.continuationContents ? parseLC(data.continuationContents) : null; + const continuation_contents_memo = _getMemo(); + if (continuation_contents) { + parsed_data.continuation_contents = continuation_contents; + parsed_data.continuation_contents_memo = continuation_contents_memo; + } + _clearMemo(); + + _createMemo(); + const actions = data.actions ? parseActions(data.actions) : null; + const actions_memo = _getMemo(); + if (actions) { + parsed_data.actions = actions; + parsed_data.actions_memo = actions_memo; + } + _clearMemo(); + + _createMemo(); + const live_chat_item_context_menu_supported_renderers = data.liveChatItemContextMenuSupportedRenderers ? parseItem(data.liveChatItemContextMenuSupportedRenderers) : null; + const live_chat_item_context_menu_supported_renderers_memo = _getMemo(); + if (live_chat_item_context_menu_supported_renderers) { + parsed_data.live_chat_item_context_menu_supported_renderers = live_chat_item_context_menu_supported_renderers; + parsed_data.live_chat_item_context_menu_supported_renderers_memo = live_chat_item_context_menu_supported_renderers_memo; + } + _clearMemo(); + + _createMemo(); + const header = data.header ? parse(data.header) : null; + const header_memo = _getMemo(); + if (header) { + parsed_data.header = header; + parsed_data.header_memo = header_memo; + } + _clearMemo(); + + _createMemo(); + const sidebar = data.sidebar ? parseItem(data.sidebar) : null; + const sidebar_memo = _getMemo(); + if (sidebar) { + parsed_data.sidebar = sidebar; + parsed_data.sidebar_memo = sidebar_memo; + } + _clearMemo(); + + applyMutations(contents_memo, data.frameworkUpdates?.entityBatchUpdate?.mutations); + + const continuation = data.continuation ? parseC(data.continuation) : null; + if (continuation) { + parsed_data.continuation = continuation; + } + + const metadata = parse(data.metadata); + if (metadata) { + parsed_data.metadata = metadata; + } + + const microformat = parseItem(data.microformat); + if (microformat) { + parsed_data.microformat = microformat; + } + + const overlay = parseItem(data.overlay); + if (overlay) { + parsed_data.overlay = overlay; + } + + const alerts = parseArray(data.alerts, Alert); + if (alerts.length) { + parsed_data.alerts = alerts; + } + + const refinements = data.refinements; + if (refinements) { + parsed_data.refinements = refinements; + } + + const estimated_results = data.estimatedResults ? parseInt(data.estimatedResults) : null; + if (estimated_results) { + parsed_data.estimated_results = estimated_results; + } + + const player_overlays = parse(data.playerOverlays); + if (player_overlays) { + parsed_data.player_overlays = player_overlays; + } + + const playback_tracking = data.playbackTracking ? { + videostats_watchtime_url: data.playbackTracking.videostatsWatchtimeUrl.baseUrl, + videostats_playback_url: data.playbackTracking.videostatsPlaybackUrl.baseUrl + } : null; + + if (playback_tracking) { + parsed_data.playback_tracking = playback_tracking; + } + + const playability_status = data.playabilityStatus ? { + status: data.playabilityStatus.status, + reason: data.playabilityStatus.reason || '', + embeddable: !!data.playabilityStatus.playableInEmbed || false, + audio_only_playablility: parseItem(data.playabilityStatus.audioOnlyPlayability, AudioOnlyPlayability), + error_screen: parseItem(data.playabilityStatus.errorScreen) + } : null; + + if (playability_status) { + parsed_data.playability_status = playability_status; + } + + const streaming_data = data.streamingData ? { + expires: new Date(Date.now() + parseInt(data.streamingData.expiresInSeconds) * 1000), + formats: parseFormats(data.streamingData.formats), + adaptive_formats: parseFormats(data.streamingData.adaptiveFormats), + dash_manifest_url: data.streamingData.dashManifestUrl || null, + hls_manifest_url: data.streamingData.hlsManifestUrl || null + } : undefined; + + if (streaming_data) { + parsed_data.streaming_data = streaming_data; + } + + const current_video_endpoint = data.currentVideoEndpoint ? new NavigationEndpoint(data.currentVideoEndpoint) : null; + if (current_video_endpoint) { + parsed_data.current_video_endpoint = current_video_endpoint; + } + + const endpoint = data.endpoint ? new NavigationEndpoint(data.endpoint) : null; + if (endpoint) { + parsed_data.endpoint = endpoint; + } + + const captions = parseItem(data.captions, PlayerCaptionsTracklist); + if (captions) { + parsed_data.captions = captions; + } + + const video_details = data.videoDetails ? new VideoDetails(data.videoDetails) : null; + if (video_details) { + parsed_data.video_details = video_details; + } + + const annotations = parseArray(data.annotations, PlayerAnnotationsExpanded); + if (annotations.length) { + parsed_data.annotations = annotations; + } + + const storyboards = parseItem(data.storyboards, [ PlayerStoryboardSpec, PlayerLiveStoryboardSpec ]); + if (storyboards) { + parsed_data.storyboards = storyboards; + } + + const endscreen = parseItem(data.endscreen, Endscreen); + if (endscreen) { + parsed_data.endscreen = endscreen; + } + + const cards = parseItem(data.cards, CardCollection); + if (cards) { + parsed_data.cards = cards; + } + + const engagement_panels = data.engagementPanels?.map((e) => { + const item = parseItem(e, YTNodes.EngagementPanelSectionList) as YTNodes.EngagementPanelSectionList; + return item; + }); + if (engagement_panels) { + parsed_data.engagement_panels = engagement_panels; + } + _createMemo(); + const items = parse(data.items); + if (items) { + parsed_data.items = items; + parsed_data.items_memo = _getMemo(); + } + _clearMemo(); + + return parsed_data; +} + +/** + * Parses a single item. + * @param data - The data to parse. + * @param validTypes - YTNode types that are allowed to be parsed. + */ +export function parseItem[]>(data: RawNode | undefined, validTypes: K): InstanceType | null; +export function parseItem(data: RawNode | undefined, validTypes: YTNodeConstructor): T | null; +export function parseItem(data?: RawNode) : YTNode; +export function parseItem(data?: RawNode, validTypes?: YTNodeConstructor | YTNodeConstructor[]) { + if (!data) return null; + + const keys = Object.keys(data); + + if (!keys.length) + return null; + + const classname = sanitizeClassName(keys[0]); + + if (!shouldIgnore(classname)) { + try { + const has_target_class = hasParser(classname); + + const TargetClass = has_target_class ? + getParserByName(classname) : + generateRuntimeClass(classname, data[keys[0]], ERROR_HANDLER); + + if (validTypes) { + if (Array.isArray(validTypes)) { + if (!validTypes.some((type) => type.type === TargetClass.type)) { + ERROR_HANDLER({ + classdata: data[keys[0]], + classname, + error_type: 'typecheck', + expected: validTypes.map((type) => type.type) + }); + return null; + } + } else if (TargetClass.type !== validTypes.type) { + ERROR_HANDLER({ + classdata: data[keys[0]], + classname, + error_type: 'typecheck', + expected: validTypes.type + }); + return null; + } + } + + const result = new TargetClass(data[keys[0]]); + _addToMemo(classname, result); + + return result; + } catch (err) { + ERROR_HANDLER({ + classname, + classdata: data[keys[0]], + error: err, + error_type: 'parse' + }); + return null; } } + + return null; } -export class NavigateAction extends YTNode { - static readonly type = 'navigateAction'; +/** + * Parses an array of items. + * @param data - The data to parse. + * @param validTypes - YTNode types that are allowed to be parsed. + */ +export function parseArray[]>(data: RawNode[] | undefined, validTypes: K): ObservedArray>; +export function parseArray(data: RawNode[] | undefined, validType: YTNodeConstructor): ObservedArray; +export function parseArray(data: RawNode[] | undefined): ObservedArray; +export function parseArray(data?: RawNode[], validTypes?: YTNodeConstructor | YTNodeConstructor[]) { + if (Array.isArray(data)) { + const results: YTNode[] = []; - endpoint: NavigationEndpoint; + for (const item of data) { + const result = parseItem(item, validTypes as YTNodeConstructor); + if (result) { + results.push(result); + } + } - constructor(data: RawNode) { - super(); - this.endpoint = new NavigationEndpoint(data.endpoint); + return observe(results); + } else if (!data) { + return observe([] as YTNode[]); } + throw new ParsingError('Expected array but got a single item'); } -export class ShowMiniplayerCommand extends YTNode { - static readonly type = 'showMiniplayerCommand'; +/** + * Parses an item or an array of items. + * @param data - The data to parse. + * @param requireArray - Whether the data should be parsed as an array. + * @param validTypes - YTNode types that are allowed to be parsed. + */ +export function parse[]>(data: RawData, requireArray: true, validTypes?: K): ObservedArray> | null; +export function parse(data?: RawData, requireArray?: false | undefined, validTypes?: YTNodeConstructor | YTNodeConstructor[]): SuperParsedResult; +export function parse(data?: RawData, requireArray?: boolean, validTypes?: YTNodeConstructor | YTNodeConstructor[]) { + if (!data) return null; - miniplayer_command: NavigationEndpoint; - show_premium_branding: boolean; + if (Array.isArray(data)) { + const results: T[] = []; - constructor(data: RawNode) { - super(); - this.miniplayer_command = new NavigationEndpoint(data.miniplayerCommand); - this.show_premium_branding = data.showPremiumBranding; + for (const item of data) { + const result = parseItem(item, validTypes as YTNodeConstructor); + if (result) { + results.push(result); + } + } + + const res = observe(results); + + return requireArray ? res : new SuperParsedResult(res); + } else if (requireArray) { + throw new ParsingError('Expected array but got a single item'); } + + return new SuperParsedResult(parseItem(data, validTypes as YTNodeConstructor)); } -export class AppendContinuationItemsAction extends YTNode { - static readonly type = 'appendContinuationItemsAction'; - - contents: ObservedArray | null; - - constructor(data: RawNode) { - super(); - this.contents = Parser.parseArray(data.continuationItems); - } +export function parseC(data: RawNode) { + if (data.timedContinuationData) + return new Continuation({ continuation: data.timedContinuationData, type: 'timed' }); + return null; } -export class ReloadContinuationItemsCommand extends YTNode { - static readonly type = 'reloadContinuationItemsCommand'; +export function parseLC(data: RawNode) { + if (data.itemSectionContinuation) + return new ItemSectionContinuation(data.itemSectionContinuation); + if (data.sectionListContinuation) + return new SectionListContinuation(data.sectionListContinuation); + if (data.liveChatContinuation) + return new LiveChatContinuation(data.liveChatContinuation); + if (data.musicPlaylistShelfContinuation) + return new MusicPlaylistShelfContinuation(data.musicPlaylistShelfContinuation); + if (data.musicShelfContinuation) + return new MusicShelfContinuation(data.musicShelfContinuation); + if (data.gridContinuation) + return new GridContinuation(data.gridContinuation); + if (data.playlistPanelContinuation) + return new PlaylistPanelContinuation(data.playlistPanelContinuation); - target_id: string; - contents: ObservedArray | null; - slot?: string; - - constructor(data: RawNode) { - super(); - this.target_id = data.targetId; - this.contents = Parser.parse(data.continuationItems, true); - this.slot = data?.slot; - } + return null; } -export class SectionListContinuation extends YTNode { - static readonly type = 'sectionListContinuation'; - - continuation: string; - contents: ObservedArray | null; - - constructor(data: RawNode) { - super(); - this.contents = Parser.parse(data.contents, true); - this.continuation = - data.continuations?.[0]?.nextContinuationData?.continuation || - data.continuations?.[0]?.reloadContinuationData?.continuation || null; - } +export function parseRR(actions: RawNode[]) { + return observe(actions.map((action: any) => { + if (action.navigateAction) + return new NavigateAction(action.navigateAction); + if (action.showMiniplayerCommand) + return new ShowMiniplayerCommand(action.showMiniplayerCommand); + if (action.reloadContinuationItemsCommand) + return new ReloadContinuationItemsCommand(action.reloadContinuationItemsCommand); + if (action.appendContinuationItemsAction) + return new YTNodes.AppendContinuationItemsAction(action.appendContinuationItemsAction); + }).filter((item) => item) as (ReloadContinuationItemsCommand | YTNodes.AppendContinuationItemsAction)[]); } -export class MusicPlaylistShelfContinuation extends YTNode { - static readonly type = 'musicPlaylistShelfContinuation'; - - continuation: string; - contents: ObservedArray | null; - - constructor(data: RawNode) { - super(); - this.contents = Parser.parse(data.contents, true); - this.continuation = data.continuations?.[0].nextContinuationData.continuation || null; - } -} - -export class MusicShelfContinuation extends YTNode { - static readonly type = 'musicShelfContinuation'; - - continuation: string; - contents: ObservedArray | null; - - constructor(data: RawNode) { - super(); - this.contents = Parser.parseArray(data.contents); - this.continuation = - data.continuations?.[0].nextContinuationData?.continuation || - data.continuations?.[0].reloadContinuationData?.continuation || null; - } -} - -export class GridContinuation extends YTNode { - static readonly type = 'gridContinuation'; - - continuation: string; - items: ObservedArray | null; - - constructor(data: RawNode) { - super(); - this.items = Parser.parse(data.items, true); - this.continuation = data.continuations?.[0].nextContinuationData.continuation || null; - } - - get contents() { - return this.items; - } -} - -export class PlaylistPanelContinuation extends YTNode { - static readonly type = 'playlistPanelContinuation'; - - continuation: string; - contents: ObservedArray | null; - - constructor(data: RawNode) { - super(); - this.contents = Parser.parseArray(data.contents); - this.continuation = data.continuations?.[0]?.nextContinuationData?.continuation || - data.continuations?.[0]?.nextRadioContinuationData?.continuation || null; - } -} - -export class Continuation extends YTNode { - static readonly type = 'continuation'; - - continuation_type: string; - timeout_ms?: number; - time_until_last_message_ms?: number; - token: string; - - constructor(data: RawNode) { - super(); - this.continuation_type = data.type; - this.timeout_ms = data.continuation?.timeoutMs; - this.time_until_last_message_ms = data.continuation?.timeUntilLastMessageMsec; - this.token = data.continuation?.continuation; - } -} - -export class LiveChatContinuation extends YTNode { - static readonly type = 'liveChatContinuation'; - - actions: ObservedArray; - action_panel: YTNode | null; - item_list: LiveChatItemList | null; - header: LiveChatHeader | null; - participants_list: LiveChatParticipantsList | null; - popout_message: Message | null; - emojis: { - emoji_id: string; - shortcuts: string[]; - search_terms: string[]; - image: Thumbnail[]; - }[]; - continuation: Continuation; - viewer_name: string; - - constructor(data: RawNode) { - super(); - this.actions = Parser.parse(data.actions?.map((action: any) => { +export function parseActions(data: RawData) { + if (Array.isArray(data)) { + return parse(data.map((action) => { delete action.clickTrackingParams; return action; - }), true) || observe([]); + })); + } + return new SuperParsedResult(parseItem(data)); +} - this.action_panel = Parser.parseItem(data.actionPanel); - this.item_list = Parser.parseItem(data.itemList, LiveChatItemList); - this.header = Parser.parseItem(data.header, LiveChatHeader); - this.participants_list = Parser.parseItem(data.participantsList, LiveChatParticipantsList); - this.popout_message = Parser.parseItem(data.popoutMessage, Message); +export function parseFormats(formats: RawNode[]): Format[] { + return formats?.map((format) => new Format(format)) || []; +} - this.emojis = data.emojis?.map((emoji: any) => ({ - emoji_id: emoji.emojiId, - shortcuts: emoji.shortcuts, - search_terms: emoji.searchTerms, - image: Thumbnail.fromResponse(emoji.image), - is_custom_emoji: emoji.isCustomEmoji - })) || []; +export function applyMutations(memo: Memo, mutations: RawNode[]) { + // Apply mutations to MusicMultiSelectMenuItems + const music_multi_select_menu_items = memo.getType(MusicMultiSelectMenuItem); - let continuation, type; + if (music_multi_select_menu_items.length > 0 && !mutations) { + ERROR_HANDLER({ + error_type: 'mutation_data_missing', + classname: 'MusicMultiSelectMenuItem' + }); + } else { + const missing_or_invalid_mutations = []; - if (data.continuations?.[0].timedContinuationData) { - type = 'timed'; - continuation = data.continuations?.[0].timedContinuationData; - } else if (data.continuations?.[0].invalidationContinuationData) { - type = 'invalidation'; - continuation = data.continuations?.[0].invalidationContinuationData; - } else if (data.continuations?.[0].liveChatReplayContinuationData) { - type = 'replay'; - continuation = data.continuations?.[0].liveChatReplayContinuationData; + for (const menu_item of music_multi_select_menu_items) { + const mutation = mutations + .find((mutation) => mutation.payload?.musicFormBooleanChoice?.id === menu_item.form_item_entity_key); + + const choice = mutation?.payload.musicFormBooleanChoice; + + if (choice?.selected !== undefined && choice?.opaqueToken) { + menu_item.selected = choice.selected; + } else { + missing_or_invalid_mutations.push(`'${menu_item.title}'`); + } + } + if (missing_or_invalid_mutations.length > 0) { + ERROR_HANDLER({ + error_type: 'mutation_data_invalid', + classname: 'MusicMultiSelectMenuItem', + total: music_multi_select_menu_items.length, + failed: missing_or_invalid_mutations.length, + titles: missing_or_invalid_mutations + }); } - - this.continuation = new Continuation({ continuation, type }); - - this.viewer_name = data.viewerName; } } diff --git a/deno/src/parser/types/ParsedResponse.ts b/deno/src/parser/types/ParsedResponse.ts index fd85090d..da703f6b 100644 --- a/deno/src/parser/types/ParsedResponse.ts +++ b/deno/src/parser/types/ParsedResponse.ts @@ -1,7 +1,7 @@ import type { Memo, ObservedArray, SuperParsedResult, YTNode } from '../helpers.ts'; import type { - ReloadContinuationItemsCommand, AppendContinuationItemsAction, Continuation, GridContinuation, + ReloadContinuationItemsCommand, Continuation, GridContinuation, ItemSectionContinuation, LiveChatContinuation, MusicPlaylistShelfContinuation, MusicShelfContinuation, PlaylistPanelContinuation, SectionListContinuation } from '../index.ts'; @@ -18,6 +18,7 @@ import type Alert from '../classes/Alert.ts'; import type NavigationEndpoint from '../classes/NavigationEndpoint.ts'; import type PlayerAnnotationsExpanded from '../classes/PlayerAnnotationsExpanded.ts'; import type EngagementPanelSectionList from '../classes/EngagementPanelSectionList.ts'; +import type { AppendContinuationItemsAction } from '../nodes.ts'; export interface IParsedResponse { actions?: SuperParsedResult; actions_memo?: Memo; @@ -51,20 +52,8 @@ export interface IParsedResponse { videostats_watchtime_url: string; videostats_playback_url: string; }; - playability_status?: { - status: string; - error_screen: YTNode | null; - audio_only_playablility: AudioOnlyPlayability | null; - embeddable: boolean; - reason: string; - }; - streaming_data?: { - expires: Date; - formats: Format[]; - adaptive_formats: Format[]; - dash_manifest_url: string | null; - hls_manifest_url: string | null; - }; + playability_status?: IPlayabilityStatus; + streaming_data?: IStreamingData; current_video_endpoint?: NavigationEndpoint; endpoint?: NavigationEndpoint; captions?: PlayerCaptionsTracklist; @@ -77,26 +66,22 @@ export interface IParsedResponse { items?: SuperParsedResult; } +export interface IStreamingData { + expires: Date; + formats: Format[]; + adaptive_formats: Format[]; + dash_manifest_url: string | null; + hls_manifest_url: string | null; +} + export interface IPlayerResponse { captions?: PlayerCaptionsTracklist; cards?: CardCollection; endscreen?: Endscreen; microformat?: YTNode; annotations?: ObservedArray; - playability_status: { - status: string; - error_screen: YTNode | null; - audio_only_playablility: AudioOnlyPlayability | null; - embeddable: boolean; - reason: string; - }; - streaming_data?: { - expires: Date; - formats: Format[]; - adaptive_formats: Format[]; - dash_manifest_url: string | null; - hls_manifest_url: string | null; - }; + playability_status: IPlayabilityStatus; + streaming_data?: IStreamingData; playback_tracking?: { videostats_watchtime_url: string; videostats_playback_url: string; @@ -105,6 +90,14 @@ export interface IPlayerResponse { video_details?: VideoDetails; } +export interface IPlayabilityStatus { + status: string; + error_screen: YTNode | null; + audio_only_playablility: AudioOnlyPlayability | null; + embeddable: boolean; + reason: string; +} + export interface INextResponse { contents?: SuperParsedResult; contents_memo?: Memo; diff --git a/deno/src/parser/types/RawResponse.ts b/deno/src/parser/types/RawResponse.ts index 95182013..8dd05f2e 100644 --- a/deno/src/parser/types/RawResponse.ts +++ b/deno/src/parser/types/RawResponse.ts @@ -54,4 +54,5 @@ export interface IRawResponse { items?: RawNode[]; frameworkUpdates?: any; engagementPanels: RawNode[]; + [key: string]: any; } diff --git a/deno/src/parser/youtube/Channel.ts b/deno/src/parser/youtube/Channel.ts index c49e2e84..f0ab8c05 100644 --- a/deno/src/parser/youtube/Channel.ts +++ b/deno/src/parser/youtube/Channel.ts @@ -10,6 +10,7 @@ import ExpandableTab from '../classes/ExpandableTab.ts'; import SectionList from '../classes/SectionList.ts'; import Tab from '../classes/Tab.ts'; import PageHeader from '../classes/PageHeader.ts'; +import TwoColumnBrowseResults from '../classes/TwoColumnBrowseResults.ts'; import Feed from '../../core/mixins/Feed.ts'; import FilterableFeed from '../../core/mixins/FilterableFeed.ts'; @@ -53,7 +54,7 @@ export default class Channel extends TabbedFeed { this.subscribe_button = this.page.header_memo?.getType(SubscribeButton).first(); - this.current_tab = this.page.contents?.item().key('tabs').parsed().array().filterType(Tab, ExpandableTab).get({ selected: true }); + this.current_tab = this.page.contents?.item().as(TwoColumnBrowseResults).tabs.array().filterType(Tab, ExpandableTab).get({ selected: true }); } /** diff --git a/deno/src/parser/youtube/HashtagFeed.ts b/deno/src/parser/youtube/HashtagFeed.ts index 6bb6153c..d7f3319b 100644 --- a/deno/src/parser/youtube/HashtagFeed.ts +++ b/deno/src/parser/youtube/HashtagFeed.ts @@ -8,9 +8,10 @@ import type Actions from '../../core/Actions.ts'; import type { ApiResponse } from '../../core/Actions.ts'; import type ChipCloudChip from '../classes/ChipCloudChip.ts'; import type { IBrowseResponse } from '../index.ts'; +import { PageHeader } from '../nodes.ts'; export default class HashtagFeed extends FilterableFeed { - header?: HashtagHeader; + header?: HashtagHeader | PageHeader; contents: RichGrid; constructor(actions: Actions, response: IBrowseResponse | ApiResponse) { @@ -25,7 +26,7 @@ export default class HashtagFeed extends FilterableFeed { throw new InnertubeError('Content tab has no content', tab); if (this.page.header) { - this.header = this.page.header.item().as(HashtagHeader); + this.header = this.page.header.item().as(HashtagHeader, PageHeader); } this.contents = tab.content.as(RichGrid); diff --git a/deno/src/parser/youtube/LiveChat.ts b/deno/src/parser/youtube/LiveChat.ts index dc4cc0f2..5c4a9366 100644 --- a/deno/src/parser/youtube/LiveChat.ts +++ b/deno/src/parser/youtube/LiveChat.ts @@ -21,7 +21,7 @@ import type AddBannerToLiveChatCommand from '../classes/livechat/AddBannerToLive import type RemoveBannerForLiveChatCommand from '../classes/livechat/RemoveBannerForLiveChatCommand.ts'; import type ShowLiveChatTooltipCommand from '../classes/livechat/ShowLiveChatTooltipCommand.ts'; -import Proto from '../../proto/index.ts'; +import * as Proto from '../../proto/index.ts'; import { InnertubeError, Platform } from '../../utils/Utils.ts'; import type { ObservedArray, YTNode } from '../helpers.ts'; diff --git a/deno/src/parser/youtube/Playlist.ts b/deno/src/parser/youtube/Playlist.ts index eb2e57d2..1da3da2b 100644 --- a/deno/src/parser/youtube/Playlist.ts +++ b/deno/src/parser/youtube/Playlist.ts @@ -36,7 +36,7 @@ class Playlist extends Feed { this.info = { ...this.page.metadata?.item().as(PlaylistMetadata), ...{ - subtitle: header.subtitle, + subtitle: header ? header.subtitle : null, author: secondary_info?.owner?.as(VideoOwner).author ?? header?.author, thumbnails: primary_info?.thumbnail_renderer?.as(PlaylistVideoThumbnail, PlaylistCustomThumbnail).thumbnail as Thumbnail[], total_items: this.#getStat(0, primary_info), diff --git a/deno/src/parser/youtube/VideoInfo.ts b/deno/src/parser/youtube/VideoInfo.ts index c81bb3b4..343641d7 100644 --- a/deno/src/parser/youtube/VideoInfo.ts +++ b/deno/src/parser/youtube/VideoInfo.ts @@ -108,8 +108,7 @@ class VideoInfo extends MediaInfo { } else if (typeof this.captions?.default_audio_track_index !== 'undefined' && this.captions?.audio_tracks && this.captions.caption_tracks) { // For videos with a single audio track and captions, we can use the captions to figure out the language of the audio and combined formats const audioTrack = this.captions.audio_tracks[this.captions.default_audio_track_index]; - const defaultCaptionTrackIndex = audioTrack.default_caption_track_index; - const index = audioTrack.caption_track_indices[defaultCaptionTrackIndex ? defaultCaptionTrackIndex : 0]; + const index = audioTrack.default_caption_track_index || 0; const language_code = this.captions.caption_tracks[index].language_code; this.streaming_data.adaptive_formats.forEach((format) => { diff --git a/deno/src/parser/ytmusic/HomeFeed.ts b/deno/src/parser/ytmusic/HomeFeed.ts index 750ef6d5..817802d2 100644 --- a/deno/src/parser/ytmusic/HomeFeed.ts +++ b/deno/src/parser/ytmusic/HomeFeed.ts @@ -29,7 +29,7 @@ class HomeFeed { if (!tab) throw new InnertubeError('Could not find Home tab.'); - if (tab.key('content').isNull()) { + if (tab.content === null) { if (!this.#page.continuation_contents) throw new InnertubeError('Continuation did not have any content.'); diff --git a/deno/src/parser/ytmusic/Library.ts b/deno/src/parser/ytmusic/Library.ts index 2668613f..5fd2c1d3 100644 --- a/deno/src/parser/ytmusic/Library.ts +++ b/deno/src/parser/ytmusic/Library.ts @@ -164,8 +164,7 @@ class LibraryContinuation { this.contents = this.#page.continuation_contents.as(MusicShelfContinuation, GridContinuation); - this.#continuation = this.#page.continuation_contents?.key('continuation').isNull() - ? null : this.#page.continuation_contents?.key('continuation').string(); + this.#continuation = this.contents.continuation || null; } async getContinuation(): Promise { diff --git a/deno/src/parser/ytmusic/TrackInfo.ts b/deno/src/parser/ytmusic/TrackInfo.ts index 242383b1..7c8b2497 100644 --- a/deno/src/parser/ytmusic/TrackInfo.ts +++ b/deno/src/parser/ytmusic/TrackInfo.ts @@ -86,7 +86,7 @@ class TrackInfo extends MediaInfo { const page = await target_tab.endpoint.call(this.actions, { client: 'YTMUSIC', parse: true }); - if (page.contents?.item().key('type').string() === 'Message') + if (page.contents?.item().type === 'Message') return page.contents.item().as(Message); if (!page.contents) diff --git a/deno/src/platform/README.md b/deno/src/platform/README.md index 9f83fe06..20f4dba7 100644 --- a/deno/src/platform/README.md +++ b/deno/src/platform/README.md @@ -24,7 +24,6 @@ If you wish to bring YouTube.js to another platform, you will need to provide th - `sha1hash(data: string)`: Function that takes a string and returns a SHA-1 hash of it. - `uuidv4()`: Function that returns a UUIDv4 string. - `eval(code: string, env: Record)`: Function to evaluate untrusted javascript script and return the result. -- `DOMParser`: DOMParser implementation. Used for generating DASH manifests. - `fetch`: WHATWG Fetch API implementation. - `Headers`: Headers implementation. - `Request`: Request implementation. diff --git a/deno/src/platform/deno.ts b/deno/src/platform/deno.ts index 7f8d70a2..c47e78fd 100644 --- a/deno/src/platform/deno.ts +++ b/deno/src/platform/deno.ts @@ -1,7 +1,6 @@ // Deno Platform Support import type { ICache } from '../types/Cache.ts'; import { Platform } from '../utils/Utils.ts'; -import DOMParser from './polyfills/server-dom.ts'; import evaluate from './jsruntime/jinter.ts'; import sha1Hash from './polyfills/web-crypto.ts'; import package_json from '../../package.json' assert { type: 'json' }; @@ -95,10 +94,6 @@ Platform.load({ return crypto.randomUUID(); }, eval: evaluate, - DOMParser, - serializeDOM(document) { - return document.toString(); - }, fetch: globalThis.fetch, Request: globalThis.Request, Response: globalThis.Response, diff --git a/deno/src/platform/lib.ts b/deno/src/platform/lib.ts index 487e1a22..fecbcb45 100644 --- a/deno/src/platform/lib.ts +++ b/deno/src/platform/lib.ts @@ -3,7 +3,7 @@ import Innertube from '../Innertube.ts'; export * from '../core/index.ts'; export * from '../parser/index.ts'; export { default as Parser } from '../parser/index.ts'; -export { default as Proto } from '../proto/index.ts'; +export * as Proto from '../proto/index.ts'; export * as Types from '../types/index.ts'; export * from '../utils/index.ts'; diff --git a/deno/src/platform/node.ts b/deno/src/platform/node.ts index f2469af2..29b63d52 100644 --- a/deno/src/platform/node.ts +++ b/deno/src/platform/node.ts @@ -16,7 +16,6 @@ import path from 'path'; import os from 'os'; import fs from 'fs/promises'; import { readFileSync } from 'fs'; -import DOMParser from './polyfills/server-dom.ts'; import CustomEvent from './polyfills/node-custom-event.ts'; import { fileURLToPath } from 'url'; import evaluate from './jsruntime/jinter.ts'; @@ -114,11 +113,7 @@ Platform.load({ uuidv4() { return crypto.randomUUID(); }, - serializeDOM(document) { - return document.toString(); - }, eval: evaluate, - DOMParser, fetch: defaultFetch as unknown as FetchFunction, Request: Request as unknown as typeof globalThis.Request, Response: Response as unknown as typeof globalThis.Response, diff --git a/deno/src/platform/polyfills/server-dom.ts b/deno/src/platform/polyfills/server-dom.ts deleted file mode 100644 index 0d7e1beb..00000000 --- a/deno/src/platform/polyfills/server-dom.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { DOMParser as DOMParserImpl } from 'https://esm.sh/linkedom'; - -export default DOMParserImpl as typeof globalThis.DOMParser; \ No newline at end of file diff --git a/deno/src/platform/web.ts b/deno/src/platform/web.ts index 3d131d1f..a40c0b82 100644 --- a/deno/src/platform/web.ts +++ b/deno/src/platform/web.ts @@ -105,10 +105,6 @@ Platform.load({ }); }, eval: evaluate, - DOMParser: globalThis.DOMParser, - serializeDOM(document) { - return new XMLSerializer().serializeToString(document); - }, fetch: globalThis.fetch, Request: globalThis.Request, Response: globalThis.Response, diff --git a/deno/src/proto/index.ts b/deno/src/proto/index.ts index 0e944911..8c984463 100644 --- a/deno/src/proto/index.ts +++ b/deno/src/proto/index.ts @@ -15,321 +15,317 @@ import * as NotificationPreferences from './generated/messages/youtube/Notificat import * as InnertubePayload from './generated/messages/youtube/InnertubePayload.ts'; import * as Hashtag from './generated/messages/youtube/Hashtag.ts'; -class Proto { - static encodeVisitorData(id: string, timestamp: number): string { - const buf = VisitorData.encodeBinary({ id, timestamp }); - return encodeURIComponent(u8ToBase64(buf).replace(/\+/g, '-').replace(/\//g, '_')); - } - - static decodeVisitorData(visitor_data: string): VisitorData.Type { - const data = VisitorData.decodeBinary(base64ToU8(decodeURIComponent(visitor_data))); - return data; - } - - static encodeChannelAnalyticsParams(channel_id: string): string { - const buf = ChannelAnalytics.encodeBinary({ - params: { - channelId: channel_id - } - }); - return encodeURIComponent(u8ToBase64(buf)); - } - - static encodeSearchFilters(filters: { - upload_date?: 'all' | 'hour' | 'today' | 'week' | 'month' | 'year', - type?: 'all' | 'video' | 'channel' | 'playlist' | 'movie', - duration?: 'all' | 'short' | 'medium' | 'long', - sort_by?: 'relevance' | 'rating' | 'upload_date' | 'view_count', - features?: ('hd' | 'subtitles' | 'creative_commons' | '3d' | 'live' | 'purchased' | '4k' | '360' | 'location' | 'hdr' | 'vr180')[] - }): string { - const upload_date = { - all: undefined, - hour: 1, - today: 2, - week: 3, - month: 4, - year: 5 - }; - - const type = { - all: undefined, - video: 1, - channel: 2, - playlist: 3, - movie: 4 - }; - - const duration = { - all: undefined, - short: 1, - long: 2, - medium: 3 - }; - - const order = { - relevance: undefined, - rating: 1, - upload_date: 2, - view_count: 3 - }; - - const features = { - hd: 'featuresHd', - subtitles: 'featuresSubtitles', - creative_commons: 'featuresCreativeCommons', - '3d': 'features3D', - live: 'featuresLive', - purchased: 'featuresPurchased', - '4k': 'features4K', - '360': 'features360', - location: 'featuresLocation', - hdr: 'featuresHdr', - vr180: 'featuresVr180' - }; - - const data: SearchFilter.Type = {}; - - if (filters) - data.filters = {}; - else - data.noFilter = 0; - - if (data.filters) { - if (filters.upload_date) { - data.filters.uploadDate = upload_date[filters.upload_date]; - } - - if (filters.type) { - data.filters.type = type[filters.type]; - } - - if (filters.duration) { - data.filters.duration = duration[filters.duration]; - } - - if (filters.sort_by && filters.sort_by !== 'relevance') { - data.sortBy = order[filters.sort_by]; - } - - if (filters.features) { - for (const feature of filters.features) { - data.filters[features[feature] as keyof SearchFilter_Filters.Type] = 1; - } - } - } - - const buf = SearchFilter.encodeBinary(data); - return encodeURIComponent(u8ToBase64(buf)); - } - - static encodeMusicSearchFilters(filters: { - type?: 'all' | 'song' | 'video' | 'album' | 'playlist' | 'artist' - }): string { - const data: MusicSearchFilter.Type = { - filters: { - type: {} - } - }; - - // TODO: See protobuf definition (protoc doesn't allow zero index: optional int32 all = 0;) - if (filters.type && filters.type !== 'all' && data.filters?.type) - data.filters.type[filters.type] = 1; - - const buf = MusicSearchFilter.encodeBinary(data); - return encodeURIComponent(u8ToBase64(buf)); - } - - static encodeMessageParams(channel_id: string, video_id: string): string { - const buf = LiveMessageParams.encodeBinary({ - params: { - ids: { - channelId: channel_id, videoId: video_id - } - }, - number0: 1, number1: 4 - }); - - return btoa(encodeURIComponent(u8ToBase64(buf))); - } - - static encodeCommentsSectionParams(video_id: string, options: { - type?: number, - sort_by?: 'TOP_COMMENTS' | 'NEWEST_FIRST' - } = {}): string { - const sort_options = { - TOP_COMMENTS: 0, - NEWEST_FIRST: 1 - }; - - const buf = GetCommentsSectionParams.encodeBinary({ - ctx: { - videoId: video_id - }, - unkParam: 6, - params: { - opts: { - videoId: video_id, - sortBy: sort_options[options.sort_by || 'TOP_COMMENTS'], - type: options.type || 2 - }, - target: 'comments-section' - } - }); - - return encodeURIComponent(u8ToBase64(buf)); - } - - static encodeCommentParams(video_id: string): string { - const buf = CreateCommentParams.encodeBinary({ - videoId: video_id, - params: { - index: 0 - }, - number: 7 - }); - return encodeURIComponent(u8ToBase64(buf)); - } - - static encodeCommentActionParams(type: number, args: { - comment_id?: string, - video_id?: string, - text?: string, - target_language?: string - } = {}): string { - const data: PeformCommentActionParams.Type = { - type, - commentId: args.comment_id || ' ', - videoId: args.video_id || ' ', - channelId: ' ', - unkNum: 2 - }; - - if (args.hasOwnProperty('text')) { - if (typeof args.target_language !== 'string') - throw new Error('target_language must be a string'); - args.comment_id && (delete data.unkNum); - data.translateCommentParams = { - params: { - comment: { - text: args.text as string - } - }, - commentId: args.comment_id || ' ', - targetLanguage: args.target_language - }; - } - - const buf = PeformCommentActionParams.encodeBinary(data); - return encodeURIComponent(u8ToBase64(buf)); - } - - static encodeNotificationPref(channel_id: string, index: number): string { - const buf = NotificationPreferences.encodeBinary({ - channelId: channel_id, - prefId: { - index - }, - number0: 0, number1: 4 - }); - - return encodeURIComponent(u8ToBase64(buf)); - } - - static encodeVideoMetadataPayload(video_id: string, metadata: UpdateVideoMetadataOptions): Uint8Array { - const data: InnertubePayload.Type = { - context: { - client: { - unkparam: 14, - clientName: CLIENTS.ANDROID.NAME, - clientVersion: CLIENTS.YTSTUDIO_ANDROID.VERSION - } - }, - target: video_id - }; - - if (Reflect.has(metadata, 'title')) - data.title = { text: metadata.title || '' }; - - if (Reflect.has(metadata, 'description')) - data.description = { text: metadata.description || '' }; - - if (Reflect.has(metadata, 'license')) - data.license = { type: metadata.license || '' }; - - if (Reflect.has(metadata, 'tags')) - data.tags = { list: metadata.tags || [] }; - - if (Reflect.has(metadata, 'category')) - data.category = { id: metadata.category || 0 }; - - if (Reflect.has(metadata, 'privacy')) { - switch (metadata.privacy) { - case 'PUBLIC': - data.privacy = { type: 1 }; - break; - case 'UNLISTED': - data.privacy = { type: 2 }; - break; - case 'PRIVATE': - data.privacy = { type: 3 }; - break; - default: - throw new Error('Invalid visibility option'); - } - } - - if (Reflect.has(metadata, 'made_for_kids')) { - data.madeForKids = { - unkparam: 1, - choice: metadata.made_for_kids ? 1 : 2 - }; - } - - if (Reflect.has(metadata, 'age_restricted')) { - data.ageRestricted = { - unkparam: 1, - choice: metadata.age_restricted ? 1 : 2 - }; - } - - const buf = InnertubePayload.encodeBinary(data); - - return buf; - } - - static encodeCustomThumbnailPayload(video_id: string, bytes: Uint8Array): Uint8Array { - const data: InnertubePayload.Type = { - context: { - client: { - unkparam: 14, - clientName: CLIENTS.ANDROID.NAME, - clientVersion: CLIENTS.YTSTUDIO_ANDROID.VERSION - } - }, - target: video_id, - videoThumbnail: { - type: 3, - thumbnail: { - imageData: bytes - } - } - }; - - const buf = InnertubePayload.encodeBinary(data); - - return buf; - } - - static encodeHashtag(hashtag: string): string { - const buf = Hashtag.encodeBinary({ - params: { - hashtag, - type: 1 - } - }); - - return encodeURIComponent(u8ToBase64(buf)); - } +export function encodeVisitorData(id: string, timestamp: number): string { + const buf = VisitorData.encodeBinary({ id, timestamp }); + return encodeURIComponent(u8ToBase64(buf).replace(/\+/g, '-').replace(/\//g, '_')); } -export default Proto; \ No newline at end of file +export function decodeVisitorData(visitor_data: string): VisitorData.Type { + const data = VisitorData.decodeBinary(base64ToU8(decodeURIComponent(visitor_data))); + return data; +} + +export function encodeChannelAnalyticsParams(channel_id: string): string { + const buf = ChannelAnalytics.encodeBinary({ + params: { + channelId: channel_id + } + }); + return encodeURIComponent(u8ToBase64(buf)); +} + +export function encodeSearchFilters(filters: { + upload_date?: 'all' | 'hour' | 'today' | 'week' | 'month' | 'year', + type?: 'all' | 'video' | 'channel' | 'playlist' | 'movie', + duration?: 'all' | 'short' | 'medium' | 'long', + sort_by?: 'relevance' | 'rating' | 'upload_date' | 'view_count', + features?: ('hd' | 'subtitles' | 'creative_commons' | '3d' | 'live' | 'purchased' | '4k' | '360' | 'location' | 'hdr' | 'vr180')[] +}): string { + const upload_date = { + all: undefined, + hour: 1, + today: 2, + week: 3, + month: 4, + year: 5 + }; + + const type = { + all: undefined, + video: 1, + channel: 2, + playlist: 3, + movie: 4 + }; + + const duration = { + all: undefined, + short: 1, + long: 2, + medium: 3 + }; + + const order = { + relevance: undefined, + rating: 1, + upload_date: 2, + view_count: 3 + }; + + const features = { + hd: 'featuresHd', + subtitles: 'featuresSubtitles', + creative_commons: 'featuresCreativeCommons', + '3d': 'features3D', + live: 'featuresLive', + purchased: 'featuresPurchased', + '4k': 'features4K', + '360': 'features360', + location: 'featuresLocation', + hdr: 'featuresHdr', + vr180: 'featuresVr180' + }; + + const data: SearchFilter.Type = {}; + + if (filters) + data.filters = {}; + else + data.noFilter = 0; + + if (data.filters) { + if (filters.upload_date) { + data.filters.uploadDate = upload_date[filters.upload_date]; + } + + if (filters.type) { + data.filters.type = type[filters.type]; + } + + if (filters.duration) { + data.filters.duration = duration[filters.duration]; + } + + if (filters.sort_by && filters.sort_by !== 'relevance') { + data.sortBy = order[filters.sort_by]; + } + + if (filters.features) { + for (const feature of filters.features) { + data.filters[features[feature] as keyof SearchFilter_Filters.Type] = 1; + } + } + } + + const buf = SearchFilter.encodeBinary(data); + return encodeURIComponent(u8ToBase64(buf)); +} + +export function encodeMusicSearchFilters(filters: { + type?: 'all' | 'song' | 'video' | 'album' | 'playlist' | 'artist' +}): string { + const data: MusicSearchFilter.Type = { + filters: { + type: {} + } + }; + + // TODO: See protobuf definition (protoc doesn't allow zero index: optional int32 all = 0;) + if (filters.type && filters.type !== 'all' && data.filters?.type) + data.filters.type[filters.type] = 1; + + const buf = MusicSearchFilter.encodeBinary(data); + return encodeURIComponent(u8ToBase64(buf)); +} + +export function encodeMessageParams(channel_id: string, video_id: string): string { + const buf = LiveMessageParams.encodeBinary({ + params: { + ids: { + channelId: channel_id, videoId: video_id + } + }, + number0: 1, number1: 4 + }); + + return btoa(encodeURIComponent(u8ToBase64(buf))); +} + +export function encodeCommentsSectionParams(video_id: string, options: { + type?: number, + sort_by?: 'TOP_COMMENTS' | 'NEWEST_FIRST' +} = {}): string { + const sort_options = { + TOP_COMMENTS: 0, + NEWEST_FIRST: 1 + }; + + const buf = GetCommentsSectionParams.encodeBinary({ + ctx: { + videoId: video_id + }, + unkParam: 6, + params: { + opts: { + videoId: video_id, + sortBy: sort_options[options.sort_by || 'TOP_COMMENTS'], + type: options.type || 2 + }, + target: 'comments-section' + } + }); + + return encodeURIComponent(u8ToBase64(buf)); +} + +export function encodeCommentParams(video_id: string): string { + const buf = CreateCommentParams.encodeBinary({ + videoId: video_id, + params: { + index: 0 + }, + number: 7 + }); + return encodeURIComponent(u8ToBase64(buf)); +} + +export function encodeCommentActionParams(type: number, args: { + comment_id?: string, + video_id?: string, + text?: string, + target_language?: string +} = {}): string { + const data: PeformCommentActionParams.Type = { + type, + commentId: args.comment_id || ' ', + videoId: args.video_id || ' ', + channelId: ' ', + unkNum: 2 + }; + + if (args.hasOwnProperty('text')) { + if (typeof args.target_language !== 'string') + throw new Error('target_language must be a string'); + args.comment_id && (delete data.unkNum); + data.translateCommentParams = { + params: { + comment: { + text: args.text as string + } + }, + commentId: args.comment_id || ' ', + targetLanguage: args.target_language + }; + } + + const buf = PeformCommentActionParams.encodeBinary(data); + return encodeURIComponent(u8ToBase64(buf)); +} + +export function encodeNotificationPref(channel_id: string, index: number): string { + const buf = NotificationPreferences.encodeBinary({ + channelId: channel_id, + prefId: { + index + }, + number0: 0, number1: 4 + }); + + return encodeURIComponent(u8ToBase64(buf)); +} + +export function encodeVideoMetadataPayload(video_id: string, metadata: UpdateVideoMetadataOptions): Uint8Array { + const data: InnertubePayload.Type = { + context: { + client: { + unkparam: 14, + clientName: CLIENTS.ANDROID.NAME, + clientVersion: CLIENTS.YTSTUDIO_ANDROID.VERSION + } + }, + target: video_id + }; + + if (Reflect.has(metadata, 'title')) + data.title = { text: metadata.title || '' }; + + if (Reflect.has(metadata, 'description')) + data.description = { text: metadata.description || '' }; + + if (Reflect.has(metadata, 'license')) + data.license = { type: metadata.license || '' }; + + if (Reflect.has(metadata, 'tags')) + data.tags = { list: metadata.tags || [] }; + + if (Reflect.has(metadata, 'category')) + data.category = { id: metadata.category || 0 }; + + if (Reflect.has(metadata, 'privacy')) { + switch (metadata.privacy) { + case 'PUBLIC': + data.privacy = { type: 1 }; + break; + case 'UNLISTED': + data.privacy = { type: 2 }; + break; + case 'PRIVATE': + data.privacy = { type: 3 }; + break; + default: + throw new Error('Invalid visibility option'); + } + } + + if (Reflect.has(metadata, 'made_for_kids')) { + data.madeForKids = { + unkparam: 1, + choice: metadata.made_for_kids ? 1 : 2 + }; + } + + if (Reflect.has(metadata, 'age_restricted')) { + data.ageRestricted = { + unkparam: 1, + choice: metadata.age_restricted ? 1 : 2 + }; + } + + const buf = InnertubePayload.encodeBinary(data); + + return buf; +} + +export function encodeCustomThumbnailPayload(video_id: string, bytes: Uint8Array): Uint8Array { + const data: InnertubePayload.Type = { + context: { + client: { + unkparam: 14, + clientName: CLIENTS.ANDROID.NAME, + clientVersion: CLIENTS.YTSTUDIO_ANDROID.VERSION + } + }, + target: video_id, + videoThumbnail: { + type: 3, + thumbnail: { + imageData: bytes + } + } + }; + + const buf = InnertubePayload.encodeBinary(data); + + return buf; +} + +export function encodeHashtag(hashtag: string): string { + const buf = Hashtag.encodeBinary({ + params: { + hashtag, + type: 1 + } + }); + + return encodeURIComponent(u8ToBase64(buf)); +} \ No newline at end of file diff --git a/deno/src/types/FormatUtils.ts b/deno/src/types/FormatUtils.ts new file mode 100644 index 00000000..1cc49ee7 --- /dev/null +++ b/deno/src/types/FormatUtils.ts @@ -0,0 +1,37 @@ +import type { Format } from '../parser/misc.ts'; + +export type URLTransformer = (url: URL) => URL; +export type FormatFilter = (format: Format) => boolean; + +export interface FormatOptions { + /** + * Video quality; 360p, 720p, 1080p, etc... also accepts 'best' and 'bestefficiency'. + */ + quality?: string; + /** + * Download type, can be: video, audio or video+audio + */ + type?: 'video' | 'audio' | 'video+audio'; + /** + * Language code, defaults to 'original'. + */ + language?: string; + /** + * File format, use 'any' to download any format + */ + format?: string; + /** + * InnerTube client, can be ANDROID, WEB, YTMUSIC, YTMUSIC_ANDROID, YTSTUDIO_ANDROID or TV_EMBEDDED + */ + client?: 'WEB' | 'ANDROID' | 'YTMUSIC_ANDROID' | 'YTMUSIC' | 'YTSTUDIO_ANDROID' | 'TV_EMBEDDED'; +} + +export interface DownloadOptions extends FormatOptions { + /** + * Download range, indicates which bytes should be downloaded. + */ + range?: { + start: number; + end: number; + } +} \ No newline at end of file diff --git a/deno/src/types/PlatformShim.ts b/deno/src/types/PlatformShim.ts index dab630fd..dc2732d3 100644 --- a/deno/src/types/PlatformShim.ts +++ b/deno/src/types/PlatformShim.ts @@ -18,8 +18,6 @@ interface PlatformShim { sha1Hash(data: string): Promise; uuidv4(): string; eval(code: string, env: Record): unknown; - DOMParser: typeof globalThis.DOMParser; - serializeDOM: (document: Document) => string; fetch: FetchFunction; Request: typeof Request; Response: typeof Response; diff --git a/deno/src/types/index.ts b/deno/src/types/index.ts index 3ca30d27..ab6426f0 100644 --- a/deno/src/types/index.ts +++ b/deno/src/types/index.ts @@ -3,4 +3,5 @@ export type { default as PlatformShim } from './PlatformShim.ts'; export * from './Cache.ts'; export * from './PlatformShim.ts'; export * from './Clients.ts'; -export * from './Endpoints.ts'; \ No newline at end of file +export * from './Endpoints.ts'; +export * from './FormatUtils.ts'; \ No newline at end of file diff --git a/deno/src/utils/Constants.ts b/deno/src/utils/Constants.ts index 36ee278b..aae19d9c 100644 --- a/deno/src/utils/Constants.ts +++ b/deno/src/utils/Constants.ts @@ -33,6 +33,12 @@ export const OAUTH = Object.freeze({ }) }); export const CLIENTS = Object.freeze({ + iOS: { + NAME: 'iOS', + VERSION: '18.06.35', + USER_AGENT: 'com.google.ios.youtube/18.06.35 (iPhone; CPU iPhone OS 14_4 like Mac OS X; en_US)', + DEVICE_MODEL: 'iPhone10,6' + }, WEB: { NAME: 'WEB', VERSION: '2.20230622.06.00', diff --git a/deno/src/utils/DashManifest.js b/deno/src/utils/DashManifest.js new file mode 100644 index 00000000..14989928 --- /dev/null +++ b/deno/src/utils/DashManifest.js @@ -0,0 +1,148 @@ +var __defProp = Object.defineProperty; +var __name = (target, value) => __defProp(target, "name", { value, configurable: true }); +import * as DashUtils from "./DashUtils.ts"; +import { getStreamingInfo } from "./StreamingInfo.ts"; +import { InnertubeError } from "./Utils.ts"; +async function OTFSegmentInfo({ info }) { + if (!info.is_oft) + return null; + const template = await info.getSegmentTemplate(); + return /* @__PURE__ */ DashUtils.createElement("segment-template", { + startNumber: "1", + timescale: "1000", + initialization: template.init_url, + media: template.media_url + }, /* @__PURE__ */ DashUtils.createElement("segment-timeline", null, template.timeline.map((segment_duration) => /* @__PURE__ */ DashUtils.createElement("s", { + d: segment_duration.duration, + r: segment_duration.repeat_count + })))); +} +__name(OTFSegmentInfo, "OTFSegmentInfo"); +function SegmentInfo({ info }) { + if (info.is_oft) { + return /* @__PURE__ */ DashUtils.createElement(OTFSegmentInfo, { + info + }); + } + return /* @__PURE__ */ DashUtils.createElement(DashUtils.Fragment, null, /* @__PURE__ */ DashUtils.createElement("base-url", null, info.base_url), /* @__PURE__ */ DashUtils.createElement("segment-base", { + indexRange: `${info.index_range.start}-${info.index_range.end}` + }, /* @__PURE__ */ DashUtils.createElement("initialization", { + range: `${info.init_range.start}-${info.init_range.end}` + }))); +} +__name(SegmentInfo, "SegmentInfo"); +function DashManifest({ + streamingData, + transformURL, + rejectFormat, + cpn, + player, + actions, + storyboards +}) { + const { + duration, + audio_sets, + video_sets, + image_sets + } = getStreamingInfo(streamingData, transformURL, rejectFormat, cpn, player, actions, storyboards); + return /* @__PURE__ */ DashUtils.createElement("mpd", { + xmlns: "urn:mpeg:dash:schema:mpd:2011", + minBufferTime: "PT1.500S", + profiles: "urn:mpeg:dash:profile:isoff-main:2011", + type: "static", + mediaPresentationDuration: `PT${duration}S`, + "xmlns:xsi": "http://www.w3.org/2001/XMLSchema-instance", + "xsi:schemaLocation": "urn:mpeg:dash:schema:mpd:2011 http://standards.iso.org/ittf/PubliclyAvailableStandards/MPEG-DASH_schema_files/DASH-MPD.xsd" + }, /* @__PURE__ */ DashUtils.createElement("period", null, audio_sets.map((set, index) => /* @__PURE__ */ DashUtils.createElement("adaptation-set", { + id: `audio_${index}`, + mimeType: set.mime_type, + startWithSAP: "1", + subsegmentAlignment: "true", + lang: set.language, + codecs: set.codecs, + audioSamplingRate: set.audio_sample_rate + }, set.track_role && /* @__PURE__ */ DashUtils.createElement("role", { + schemeIdUri: "urn:mpeg:dash:role:2011", + value: set.track_role + }), set.track_name && /* @__PURE__ */ DashUtils.createElement("label", { + id: `audio_${index}` + }, set.track_name), set.channels && /* @__PURE__ */ DashUtils.createElement("audio-channel-configuration", { + schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011", + value: set.channels + }), set.representations.map((rep) => /* @__PURE__ */ DashUtils.createElement("representation", { + id: rep.uid, + bandwidth: rep.bitrate, + codecs: rep.codecs, + audioSamplingRate: rep.audio_sample_rate + }, rep.channels && /* @__PURE__ */ DashUtils.createElement("audio-channel-configuration", { + schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011", + value: rep.channels + }), /* @__PURE__ */ DashUtils.createElement(SegmentInfo, { + info: rep.segment_info + }))))), video_sets.map((set, index) => /* @__PURE__ */ DashUtils.createElement("adaptation-set", { + id: `video_${index}`, + mimeType: set.mime_type, + startWithSAP: "1", + subsegmentAlignment: "true", + codecs: set.codecs, + maxPlayoutRate: "1", + frameRate: set.fps + }, set.color_info.primaries && /* @__PURE__ */ DashUtils.createElement("essential-property", { + schemeIdUri: "urn:mpeg:mpegB:cicp:ColourPrimaries", + value: set.color_info.primaries + }), set.color_info.transfer_characteristics && /* @__PURE__ */ DashUtils.createElement("essential-property", { + schemeIdUri: "urn:mpeg:mpegB:cicp:TransferCharacteristics", + value: set.color_info.transfer_characteristics + }), set.color_info.matrix_coefficients && /* @__PURE__ */ DashUtils.createElement("essential-property", { + schemeIdUri: "urn:mpeg:mpegB:cicp:MatrixCoefficients", + value: set.color_info.matrix_coefficients + }), set.representations.map((rep) => /* @__PURE__ */ DashUtils.createElement("representation", { + id: rep.uid, + bandwidth: rep.bitrate, + width: rep.width, + height: rep.height, + codecs: rep.codecs, + frameRate: rep.fps + }, /* @__PURE__ */ DashUtils.createElement(SegmentInfo, { + info: rep.segment_info + }))))), image_sets.map(async (set, index) => { + return /* @__PURE__ */ DashUtils.createElement("adaptation-set", { + id: `thumbs_${index}`, + mimeType: await set.getMimeType(), + contentType: "image" + }, set.representations.map(async (rep) => /* @__PURE__ */ DashUtils.createElement("representation", { + id: `thumbnails_${rep.thumbnail_width}x${rep.thumbnail_height}`, + bandwidth: await rep.getBitrate(), + width: rep.sheet_width, + height: rep.sheet_height + }, /* @__PURE__ */ DashUtils.createElement("essential-property", { + schemeIdUri: "http://dashif.org/thumbnail_tile", + value: `${rep.columns}x${rep.rows}` + }), /* @__PURE__ */ DashUtils.createElement("segment-template", { + media: rep.template_url, + duration: rep.template_duration, + startNumber: "0" + })))); + }))); +} +__name(DashManifest, "DashManifest"); +function toDash(streaming_data, url_transformer = (url) => url, format_filter, cpn, player, actions, storyboards) { + if (!streaming_data) + throw new InnertubeError("Streaming data not available"); + return DashUtils.renderToString( + /* @__PURE__ */ DashUtils.createElement(DashManifest, { + streamingData: streaming_data, + transformURL: url_transformer, + rejectFormat: format_filter, + cpn, + player, + actions, + storyboards + }) + ); +} +__name(toDash, "toDash"); +export { + toDash +}; diff --git a/deno/src/utils/DashManifest.tsx b/deno/src/utils/DashManifest.tsx new file mode 100644 index 00000000..7cddd700 --- /dev/null +++ b/deno/src/utils/DashManifest.tsx @@ -0,0 +1,249 @@ +/* eslint-disable tsdoc/syntax */ +/** @jsxFactory DashUtils.createElement */ +/** @jsxFragmentFactory DashUtils.Fragment */ +import type Actions from '../core/Actions.ts'; +import type Player from '../core/Player.ts'; +import type { IStreamingData } from '../parser/index.ts'; +import type { PlayerStoryboardSpec } from '../parser/nodes.ts'; +import * as DashUtils from './DashUtils.ts'; +import type { SegmentInfo as FSegmentInfo } from './StreamingInfo.ts'; +import { getStreamingInfo } from './StreamingInfo.ts'; +import type { FormatFilter, URLTransformer } from '../types/FormatUtils.ts'; +import { InnertubeError } from './Utils.ts'; + +interface DashManifestProps { + streamingData: IStreamingData; + transformURL?: URLTransformer; + rejectFormat?: FormatFilter; + cpn?: string; + player?: Player; + actions?: Actions; + storyboards?: PlayerStoryboardSpec; +} + +async function OTFSegmentInfo({ info }: { info: FSegmentInfo }) { + if (!info.is_oft) return null; + + const template = await info.getSegmentTemplate(); + + return + + { + template.timeline.map((segment_duration) => ( + + )) + } + + ; +} + +function SegmentInfo({ info }: { info: FSegmentInfo }) { + if (info.is_oft) { + return ; + } + return <> + + {info.base_url} + + + + + ; +} + +function DashManifest({ + streamingData, + transformURL, + rejectFormat, + cpn, + player, + actions, + storyboards +}: DashManifestProps) { + const { + duration, + audio_sets, + video_sets, + image_sets + } = getStreamingInfo(streamingData, transformURL, rejectFormat, cpn, player, actions, storyboards); + + // XXX: DASH spec: https://standards.iso.org/ittf/PubliclyAvailableStandards/c083314_ISO_IEC%2023009-1_2022(en).zip + + return + + { + audio_sets.map((set, index) => ( + + { + set.track_role && + + } + { + set.track_name && + + } + { + set.channels && + + } + { + set.representations.map((rep) => ( + + { + rep.channels && + + } + + + )) + } + + )) + } + { + video_sets.map((set, index) => ( + + { + set.color_info.primaries && + + } + { + set.color_info.transfer_characteristics && + + } + { + set.color_info.matrix_coefficients && + + } + { + set.representations.map((rep) => ( + + + + )) + } + + )) + } + { + image_sets.map(async (set, index) => { + return + { + set.representations.map(async (rep) => ( + + + + + )) + } + ; + }) + } + + ; +} + +export function toDash( + streaming_data?: IStreamingData, + url_transformer: URLTransformer = (url) => url, + format_filter?: FormatFilter, + cpn?: string, + player?: Player, + actions?: Actions, + storyboards?: PlayerStoryboardSpec +) { + if (!streaming_data) + throw new InnertubeError('Streaming data not available'); + + return DashUtils.renderToString( + + ); +} diff --git a/deno/src/utils/DashUtils.ts b/deno/src/utils/DashUtils.ts new file mode 100644 index 00000000..869b2ba0 --- /dev/null +++ b/deno/src/utils/DashUtils.ts @@ -0,0 +1,104 @@ +/* eslint-disable @typescript-eslint/no-namespace */ +declare global { + namespace JSX { + interface IntrinsicElements { + [key: string]: DashProps; + } + } +} + +export type DashChild = (DashNode | (DashNode | Promise) | Promise); +export interface DashProps { + [key: string]: unknown, + children?: DashChild[] +} + +export interface DashNode { + type: string, + props: DashProps, +} + +const XML_CHARACTER_MAP = { + '&': '&', + '"': '"', + '\'': ''', + '<': '<', + '>': '>' +} as const; + +function escapeXMLString(str: string) { + return str.replace(/([&"<>'])/g, (_, item: keyof typeof XML_CHARACTER_MAP) => { + return XML_CHARACTER_MAP[item]; + }); +} + +function normalizeTag(tag: string) { + if (tag === 'mpd') return 'MPD'; + if (tag === 'base-url') return 'BaseURL'; + + const sections = tag.split('-'); + return sections.map((section) => section.charAt(0).toUpperCase() + section.slice(1)).join(''); +} + +export function createElement( + tagNameOrFunction: string | ((props: DashProps) => DashNode | Promise), + props: { [key: string] : unknown } | null | undefined, + ...children: DashChild[] +): DashNode | Promise { + const normalizedChildren = children.flat().map((child) => typeof child === 'string' ? createTextElement(child) : child); + + if (typeof tagNameOrFunction === 'function') { + return tagNameOrFunction({ ...props, children: normalizedChildren }); + } + + return { + type: normalizeTag(tagNameOrFunction), + props: { + ...props, + children: normalizedChildren + } + }; +} + +export function createTextElement(text: string): DashNode { + return { + type: 'TEXT_ELEMENT', + props: { nodeValue: text } + }; +} + +export async function renderElementToString(element: DashNode): Promise { + if (element.type === 'TEXT_ELEMENT') + return escapeXMLString(typeof element.props.nodeValue === 'string' ? element.props.nodeValue : ''); + + let dom = `<${element.type}`; + + if (element.props) { + const properties = Object.keys(element.props) + .filter((key) => ![ 'children', 'nodeValue' ].includes(key) && element.props[key] !== undefined) + .map((name) => `${name}="${escapeXMLString(`${element.props[name]}`)}"`); + + if (properties.length > 0) + dom += ` ${properties.join(' ')}`; + } + + if (element.props.children) { + const children = await Promise.all((await Promise.all(element.props.children.flat())).flat().filter((child) => !!child).map((child) => renderElementToString(child))); + if (children.length > 0) { + dom += `>${children.join('')}`; + return dom; + } + } + + return `${dom}/>`; +} + +export async function renderToString(root: DashNode | Promise) { + const dom = await renderElementToString(await root); + + return `${dom}`; +} + +export function Fragment(props: DashProps) { + return props.children; +} diff --git a/deno/src/utils/FormatUtils.ts b/deno/src/utils/FormatUtils.ts index 3ea0c6f8..c83e2d13 100644 --- a/deno/src/utils/FormatUtils.ts +++ b/deno/src/utils/FormatUtils.ts @@ -2,860 +2,193 @@ import type Player from '../core/Player.ts'; import type Actions from '../core/Actions.ts'; import type Format from '../parser/classes/misc/Format.ts'; -import type AudioOnlyPlayability from '../parser/classes/AudioOnlyPlayability.ts'; -import type PlayerStoryboardSpec from '../parser/classes/PlayerStoryboardSpec.ts'; -import type { YTNode } from '../parser/helpers.ts'; import * as Constants from './Constants.ts'; -import { getStringBetweenStrings, InnertubeError, Platform, streamToIterable } from './Utils.ts'; - -export type URLTransformer = (url: URL) => URL; -export type FormatFilter = (format: Format) => boolean; - -export interface FormatOptions { - /** - * Video quality; 360p, 720p, 1080p, etc... also accepts 'best' and 'bestefficiency'. - */ - quality?: string; - /** - * Download type, can be: video, audio or video+audio - */ - type?: 'video' | 'audio' | 'video+audio'; - /** - * Language code, defaults to 'original'. - */ - language?: string; - /** - * File format, use 'any' to download any format - */ - format?: string; - /** - * InnerTube client, can be ANDROID, WEB, YTMUSIC, YTMUSIC_ANDROID, YTSTUDIO_ANDROID or TV_EMBEDDED - */ - client?: 'WEB' | 'ANDROID' | 'YTMUSIC_ANDROID' | 'YTMUSIC' | 'YTSTUDIO_ANDROID' | 'TV_EMBEDDED'; -} - -export interface DownloadOptions extends FormatOptions { - /** - * Download range, indicates which bytes should be downloaded. - */ - range?: { - start: number; - end: number; - } -} - -class FormatUtils { - static async download(options: DownloadOptions, actions: Actions, playability_status?: { - status: string; - error_screen: YTNode | null; - audio_only_playablility: AudioOnlyPlayability | null; - embeddable: boolean; - reason: any; - }, streaming_data?: { - expires: Date; - formats: Format[]; - adaptive_formats: Format[]; - dash_manifest_url: string | null; - hls_manifest_url: string | null; - }, player?: Player, cpn?: string): Promise> { - if (playability_status?.status === 'UNPLAYABLE') - throw new InnertubeError('Video is unplayable', { error_type: 'UNPLAYABLE' }); - if (playability_status?.status === 'LOGIN_REQUIRED') - throw new InnertubeError('Video is login required', { error_type: 'LOGIN_REQUIRED' }); - if (!streaming_data) - throw new InnertubeError('Streaming data not available.', { error_type: 'NO_STREAMING_DATA' }); - - const opts: DownloadOptions = { - quality: '360p', - type: 'video+audio', - format: 'mp4', - range: undefined, - ...options - }; - - const format = this.chooseFormat(opts, streaming_data); - const format_url = format.decipher(player); - - // If we're not downloading the video in chunks, we just use fetch once. - if (opts.type === 'video+audio' && !options.range) { - const response = await actions.session.http.fetch_function(`${format_url}&cpn=${cpn}`, { - method: 'GET', - headers: Constants.STREAM_HEADERS, - redirect: 'follow' - }); - - // Throw if the response is not 2xx - if (!response.ok) - throw new InnertubeError('The server responded with a non 2xx status code', { error_type: 'FETCH_FAILED', response }); - - const body = response.body; - - if (!body) - throw new InnertubeError('Could not get ReadableStream from fetch Response.', { error_type: 'FETCH_FAILED', response }); - - return body; - } - - // We need to download in chunks. - - const chunk_size = 1048576 * 10; // 10MB - - let chunk_start = (options.range ? options.range.start : 0); - let chunk_end = (options.range ? options.range.end : chunk_size); - let must_end = false; - - let cancel: AbortController; - - const readable_stream = new Platform.shim.ReadableStream({ - // eslint-disable-next-line @typescript-eslint/no-empty-function - start() { }, - pull: async (controller) => { - if (must_end) { - controller.close(); - return; - } - - if ((chunk_end >= (format.content_length ? format.content_length : 0)) || options.range) { - must_end = true; - } - - return new Promise(async (resolve, reject) => { - try { - cancel = new AbortController(); - - const response = await actions.session.http.fetch_function(`${format_url}&cpn=${cpn}&range=${chunk_start}-${chunk_end || ''}`, { - method: 'GET', - headers: { - ...Constants.STREAM_HEADERS - // XXX: use YouTube's range parameter instead of a Range header. - // Range: `bytes=${chunk_start}-${chunk_end}` - }, - signal: cancel.signal - }); - - const body = response.body; - - if (!body) - throw new InnertubeError('Could not get ReadableStream from fetch Response.', { video: this, error_type: 'FETCH_FAILED', response }); - - for await (const chunk of streamToIterable(body)) { - controller.enqueue(chunk); - } - - chunk_start = chunk_end + 1; - chunk_end += chunk_size; - - resolve(); - - } catch (e: any) { - reject(e); - } - }); - }, - async cancel(reason) { - cancel.abort(reason); - } - }, { - highWaterMark: 1, // TODO: better value? - size(chunk) { - return chunk.byteLength; - } - }); - - return readable_stream; - } - - /** - * Selects the format that best matches the given options. - * @param options - Options - * @param streaming_data - Streaming data - */ - static chooseFormat(options: FormatOptions, streaming_data?: { - expires: Date; - formats: Format[]; - adaptive_formats: Format[]; - dash_manifest_url: string | null; - hls_manifest_url: string | null; - }): Format { - if (!streaming_data) - throw new InnertubeError('Streaming data not available'); - - const formats = [ - ...(streaming_data.formats || []), - ...(streaming_data.adaptive_formats || []) - ]; - - const requires_audio = options.type ? options.type.includes('audio') : true; - const requires_video = options.type ? options.type.includes('video') : true; - const language = options.language || 'original'; - const quality = options.quality || 'best'; - - let best_width = -1; - - const is_best = [ 'best', 'bestefficiency' ].includes(quality); - const use_most_efficient = quality !== 'best'; - - let candidates = formats.filter((format) => { - if (requires_audio && !format.has_audio) - return false; - if (requires_video && !format.has_video) - return false; - if (options.format !== 'any' && !format.mime_type.includes(options.format || 'mp4')) - return false; - if (!is_best && format.quality_label !== quality) - return false; - if (best_width < format.width) - best_width = format.width; - return true; - }); - - if (!candidates.length) - throw new InnertubeError('No matching formats found', { options }); - - if (is_best && requires_video) - candidates = candidates.filter((format) => format.width === best_width); - - if (requires_audio && !requires_video) { - const audio_only = candidates.filter((format) => { - if (language !== 'original') { - return !format.has_video && format.language === language; - } - return !format.has_video && format.is_original; - - }); - if (audio_only.length > 0) { - candidates = audio_only; - } - } - - if (use_most_efficient) { - // Sort by bitrate (lower is better) - candidates.sort((a, b) => a.bitrate - b.bitrate); - } else { - // Sort by bitrate (higher is better) - candidates.sort((a, b) => b.bitrate - a.bitrate); - } - - return candidates[0]; - } - - static async toDash(streaming_data?: { - expires: Date; - formats: Format[]; - adaptive_formats: Format[]; - dash_manifest_url: string | null; - hls_manifest_url: string | null; - }, url_transformer: URLTransformer = (url) => url, format_filter?: FormatFilter, cpn?: string, player?: Player, actions?: Actions, storyboards?: PlayerStoryboardSpec): Promise { - if (!streaming_data) - throw new InnertubeError('Streaming data not available'); - - let adaptive_formats; - - if (format_filter) { - adaptive_formats = streaming_data.adaptive_formats.filter((fmt: Format) => !(format_filter(fmt))); - } else { - adaptive_formats = streaming_data.adaptive_formats; - } - - if (!adaptive_formats.length) - throw new InnertubeError('No adaptive formats found'); - - const length = adaptive_formats[0].approx_duration_ms / 1000; - - // DASH spec: https://standards.iso.org/ittf/PubliclyAvailableStandards/c083314_ISO_IEC%2023009-1_2022(en).zip - - const document = new Platform.shim.DOMParser().parseFromString('', 'application/xml'); - const mpd = document.querySelector('MPD') as HTMLElement; - const period = document.createElement('Period'); - - mpd.replaceWith(this.#el(document, 'MPD', { - xmlns: 'urn:mpeg:dash:schema:mpd:2011', - minBufferTime: 'PT1.500S', - profiles: 'urn:mpeg:dash:profile:isoff-main:2011', - type: 'static', - mediaPresentationDuration: `PT${length}S`, - 'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance', - 'xsi:schemaLocation': 'urn:mpeg:dash:schema:mpd:2011 http://standards.iso.org/ittf/PubliclyAvailableStandards/MPEG-DASH_schema_files/DASH-MPD.xsd' - }, [ - period - ])); - - await this.#generateAdaptationSet(document, period, adaptive_formats, url_transformer, cpn, player, actions, storyboards); - - return Platform.shim.serializeDOM(document); - } - - static #el(document: XMLDocument, tag: string, attrs: Record, children: Node[] = []) { - const el = document.createElement(tag); - for (const [ key, value ] of Object.entries(attrs)) { - value && el.setAttribute(key, value); - } - for (const child of children) { - if (typeof child === 'undefined') continue; - el.appendChild(child); - } - return el; - } - - static async #generateAdaptationSet( - document: XMLDocument, - period: Element, - formats: Format[], - url_transformer: URLTransformer, - cpn?: string, - player?: Player, - actions?: Actions, - storyboards?: PlayerStoryboardSpec - ) { - const mime_types: string[] = []; - const mime_objects: Format[][] = [ [] ]; - - formats.forEach((video_format) => { - if ((!video_format.index_range || !video_format.init_range) && !video_format.is_type_otf) { - return; - } - const mime_type = video_format.mime_type; - const mime_type_index = mime_types.indexOf(mime_type); - if (mime_type_index > -1) { - mime_objects[mime_type_index].push(video_format); - } else { - mime_types.push(mime_type); - mime_objects.push([]); - mime_objects[mime_types.length - 1].push(video_format); - } - }); - - let set_id = 0; - for (let i = 0; i < mime_types.length; i++) { - // When the video has multiple different audio tracks we want to include the extra information in the manifest - if (mime_objects[i][0].has_audio && mime_objects[i][0].audio_track) { - const track_ids: string[] = []; - const track_objects: Format[][] = [ [] ]; - - mime_objects[i].forEach((format) => { - const id_index = track_ids.indexOf(format.audio_track?.id as string); - if (id_index > -1) { - track_objects[id_index].push(format); - } else { - track_ids.push(format.audio_track?.id as string); - track_objects.push([]); - track_objects[track_ids.length - 1].push(format); - } - }); - - // The lang attribute has to go on the AdaptationSet element and the Role element goes inside the AdaptationSet too, so we need a separate adaptation set for each language and role - for (let j = 0; j < track_ids.length; j++) { - const first_format = track_objects[j][0]; - - const children = []; - - let role; - if (first_format.audio_track?.audio_is_default) { - role = 'main'; - } else if (first_format.is_dubbed) { - role = 'dub'; - } else if (first_format.is_descriptive) { - role = 'description'; - } else { - role = 'alternate'; - } - - children.push( - this.#el(document, 'Role', { - schemeIdUri: 'urn:mpeg:dash:role:2011', - value: role - }), - this.#el(document, 'Label', { - id: set_id.toString() - }, [ - document.createTextNode(first_format.audio_track?.display_name as string) - ]) - ); - - const set = this.#el(document, 'AdaptationSet', { - id: `${set_id++}`, - mimeType: mime_types[i].split(';')[0], - startWithSAP: '1', - subsegmentAlignment: 'true', - lang: first_format.language as string, - // Non-standard attribute used by shaka instead of the standard Label element - label: first_format.audio_track?.display_name as string - }, children); - - const hoisted: string[] = []; - - this.#hoistCodecsIfPossible(set, track_objects[j], hoisted); - this.#hoistNumberAttributeIfPossible(set, track_objects[j], 'audioSamplingRate', 'audio_sample_rate', hoisted); - - this.#hoistAudioChannelsIfPossible(document, set, track_objects[j], hoisted); - - period.appendChild(set); - - for (const format of track_objects[j]) { - await this.#generateRepresentationAudio(document, set, format, url_transformer, hoisted, cpn, player, actions); - } - } - } else { - const set = this.#el(document, 'AdaptationSet', { - id: `${set_id++}`, - mimeType: mime_types[i].split(';')[0], - startWithSAP: '1', - subsegmentAlignment: 'true' - }); - - const color_info = mime_objects[i][0].color_info; - if (typeof color_info !== 'undefined') { - // Section 5.5 Video source metadata signalling https://dashif.org/docs/IOP-Guidelines/DASH-IF-IOP-Part7-v5.0.0.pdf - // Section 8 Video code points https://www.itu.int/rec/T-REC-H.273-202107-I/en - // The player.js file was also helpful - - if (color_info.primaries) { - let primaries = ''; - - switch (color_info.primaries) { - case 'BT709': - primaries = '1'; - break; - case 'BT2020': - primaries = '9'; - break; - } - - if (primaries !== '') { - set.appendChild(this.#el(document, 'EssentialProperty', { - schemeIdUri: 'urn:mpeg:mpegB:cicp:ColourPrimaries', - value: primaries - })); - } - } - - if (color_info.transfer_characteristics) { - let transfer_characteristics = ''; - - switch (color_info.transfer_characteristics) { - case 'BT709': - transfer_characteristics = '1'; - break; - case 'BT2020_10': - transfer_characteristics = '14'; - break; - case 'SMPTEST2084': - transfer_characteristics = '16'; - break; - case 'ARIB_STD_B67': - transfer_characteristics = '18'; - break; - } - - if (transfer_characteristics !== '') { - set.appendChild(this.#el(document, 'EssentialProperty', { - schemeIdUri: 'urn:mpeg:mpegB:cicp:TransferCharacteristics', - value: transfer_characteristics - })); - } - } - - - if (color_info.matrix_coefficients) { - let matrix_coefficients = ''; - - // This list is incomplete, as the player.js doesn't currently have any code for matrix coefficients, - // So it doesn't have a list like with the other two, so this is just based on what we've seen in responses - switch (color_info.matrix_coefficients) { - case 'BT709': - matrix_coefficients = '1'; - break; - case 'BT2020_NCL': - matrix_coefficients = '14'; - break; - default: { - const format = mime_objects[i][0]; - const url = new URL(format.url as string); - - const anonymisedFormat = JSON.parse(JSON.stringify(format)); - anonymisedFormat.url = 'REDACTED'; - anonymisedFormat.signature_cipher = 'REDACTED'; - anonymisedFormat.cipher = 'REDACTED'; - - console.warn(`YouTube.js toDash(): Unknown matrix coefficients "${color_info.matrix_coefficients}", the DASH manifest is still usuable without this.\n` - + `Please report it at ${Platform.shim.info.bugs_url} so we can add support for it.\n` - + `Innertube client: ${url.searchParams.get('c')}\nformat:`, anonymisedFormat); - } - } - - if (matrix_coefficients !== '') { - set.appendChild(this.#el(document, 'EssentialProperty', { - schemeIdUri: 'urn:mpeg:mpegB:cicp:MatrixCoefficients', - value: matrix_coefficients - })); - } - } - } - - const hoisted: string[] = []; - - this.#hoistCodecsIfPossible(set, mime_objects[i], hoisted); - - if (mime_objects[i][0].has_audio) { - this.#hoistNumberAttributeIfPossible(set, mime_objects[i], 'audioSamplingRate', 'audio_sample_rate', hoisted); - - this.#hoistAudioChannelsIfPossible(document, set, mime_objects[i], hoisted); - - const language = mime_objects[i][0].language; - if (language) { - set.setAttribute('lang', language); - } - } else { - set.setAttribute('maxPlayoutRate', '1'); - - this.#hoistNumberAttributeIfPossible(set, mime_objects[i], 'frameRate', 'fps', hoisted); - } - - period.appendChild(set); - - for (const format of mime_objects[i]) { - if (format.has_video) { - await this.#generateRepresentationVideo(document, set, format, url_transformer, hoisted, cpn, player, actions); - } else { - await this.#generateRepresentationAudio(document, set, format, url_transformer, hoisted, cpn, player, actions); - } - } - } - } - - // We need to make requests to get the image sizes, so we'll skip the storyboards if we don't have an Actions instance - if (storyboards && actions) { - const mime_types: string[] = []; - const mime_objects: { - template_url: string; - thumbnail_width: number; - thumbnail_height: number; - thumbnail_count: number; - interval: number; - columns: number; - rows: number; - storyboard_count: number; - }[][] = [ [] ]; - - for (const storyboard of storyboards.boards) { - const extension = new URL(storyboard.template_url).pathname.split('.').at(-1); - - let mime_type = ''; - - switch (extension) { - case 'jpg': - mime_type = 'image/jpeg'; - break; - case 'png': - mime_type = 'image/png'; - break; - case 'webp': - mime_type = 'image/webp'; - break; - } - - const mime_type_index = mime_types.indexOf(mime_type); - if (mime_type_index > -1) { - mime_objects[mime_type_index].push(storyboard); - } else { - mime_types.push(mime_type); - mime_objects.push([]); - mime_objects[mime_types.length - 1].push(storyboard); - } - } - - const duration = formats[0].approx_duration_ms / 1000; - - for (let i = 0; i < mime_types.length; i++) { - const set = this.#el(document, 'AdaptationSet', { - id: `${set_id++}`, - mimeType: mime_types[i], - contentType: 'image' - }); - - period.appendChild(set); - - for (const storyboard of mime_objects[i]) { - await this.#generateRepresentationImage(document, set, storyboard, duration, url_transformer, actions); - } - } - } - } - - static #hoistCodecsIfPossible(set: Element, formats: Format[], hoisted: string[]) { - if (formats.length > 1 && new Set(formats.map((format) => getStringBetweenStrings(format.mime_type, 'codecs="', '"'))).size === 1) { - set.setAttribute('codecs', getStringBetweenStrings(formats[0].mime_type, 'codecs="', '"') as string); - hoisted.push('codecs'); - } - } - - static #hoistNumberAttributeIfPossible(set: Element, formats: Format[], attribute: 'audioSamplingRate' | 'frameRate', property: 'audio_sample_rate' | 'fps', hoisted: string[]) { - if (formats.length > 1 && new Set(formats.map((format) => format.fps)).size === 1) { - set.setAttribute(attribute, formats[0][property]?.toString() as string); - hoisted.push(attribute); - } - } - - static #hoistAudioChannelsIfPossible(document: XMLDocument, set: Element, formats: Format[], hoisted: string[]) { - if (formats.length > 1 && new Set(formats.map((format) => format.audio_channels?.toString() || '2')).size === 1) { - set.appendChild( - this.#el(document, 'AudioChannelConfiguration', { - schemeIdUri: 'urn:mpeg:dash:23003:3:audio_channel_configuration:2011', - value: formats[0].audio_channels?.toString() || '2' - }) - ); - hoisted.push('AudioChannelConfiguration'); - } - } - - static async #generateRepresentationVideo(document: XMLDocument, set: Element, format: Format, url_transformer: URLTransformer, hoisted: string[], cpn?: string, player?: Player, actions?: Actions) { - const url = new URL(format.decipher(player)); - url.searchParams.set('cpn', cpn || ''); - - const representation = this.#el(document, 'Representation', { - id: format.itag?.toString(), - bandwidth: format.bitrate?.toString(), - width: format.width?.toString(), - height: format.height?.toString() - }); - - if (!hoisted.includes('codecs')) { - const codecs = getStringBetweenStrings(format.mime_type, 'codecs="', '"'); - representation.setAttribute('codecs', codecs as string); - } - - if (!hoisted.includes('frameRate')) { - representation.setAttribute('frameRate', format.fps?.toString() as string); - } - - set.appendChild(representation); - - await this.#generateSegmentInformation(document, representation, format, url_transformer(url)?.toString(), actions); - } - - static async #generateRepresentationAudio(document: XMLDocument, set: Element, format: Format, url_transformer: URLTransformer, hoisted: string[], cpn?: string, player?: Player, actions?: Actions) { - const url = new URL(format.decipher(player)); - url.searchParams.set('cpn', cpn || ''); - - let id; - if (format.audio_track) { - id = `${format.itag?.toString()}-${format.audio_track.id}`; - } else { - id = format.itag?.toString(); - } - - const representation = this.#el(document, 'Representation', { - id, - bandwidth: format.bitrate?.toString() - }); - - if (!hoisted.includes('codecs')) { - const codecs = getStringBetweenStrings(format.mime_type, 'codecs="', '"'); - representation.setAttribute('codecs', codecs as string); - } - - if (!hoisted.includes('audioSamplingRate')) { - representation.setAttribute('audioSamplingRate', format.audio_sample_rate?.toString() as string); - } - - if (!hoisted.includes('AudioChannelConfiguration')) { - representation.appendChild( - this.#el(document, 'AudioChannelConfiguration', { - schemeIdUri: 'urn:mpeg:dash:23003:3:audio_channel_configuration:2011', - value: format.audio_channels?.toString() || '2' - }) - ); - } - - set.appendChild(representation); - - await this.#generateSegmentInformation(document, representation, format, url_transformer(url)?.toString(), actions); - } - - static async #generateRepresentationImage(document: XMLDocument, set: Element, storyboard: { - template_url: string; - thumbnail_width: number; - thumbnail_height: number; - thumbnail_count: number; - interval: number; - columns: number; - rows: number; - storyboard_count: number; - }, duration: number, url_transformer: URLTransformer, actions: Actions) { - const url = storyboard.template_url; - - const response_promises: Promise[] = []; - - // Set a limit so we don't take forever for long videos - const requestLimit = storyboard.storyboard_count > 10 ? 10 : storyboard.storyboard_count; - for (let i = 0; i < requestLimit; i++) { - const response_promise = actions.session.http.fetch_function(new URL(url.replace('$M', i.toString())), { - method: 'HEAD', - headers: Constants.STREAM_HEADERS - }); - - response_promises.push(response_promise); - } - - // Run the requests in parallel to avoid causing too much delay - const responses = await Promise.all(response_promises); - - const content_lengths = []; - - for (const response of responses) { - content_lengths.push(parseInt(response.headers.get('Content-Length') || '0', 10)); - - const content_type = response.headers.get('Content-Type'); - - // Sometimes youtube returns webp instead of jpg despite the file extension being jpg - // So we need to update the mime type to reflect the actual mime type of the response - - if (content_type && content_type.length > 0) { - if (set.getAttribute('mimeType') !== content_type) { - set.setAttribute('mimeType', content_type); - } - } - } - - // This is a rough estimate, so it probably won't reflect that actual peak bitrate - // Hopefully it's close enough, because figuring out the actual peak bitrate would require downloading and analysing all storyboard tiles - const bandwidth = Math.ceil((Math.max(...content_lengths) / (storyboard.rows * storyboard.columns)) * 8); - - const representation = this.#el(document, 'Representation', { - id: `thumbnails_${storyboard.thumbnail_width}x${storyboard.thumbnail_height}`, - bandwidth: bandwidth.toString(), - width: (storyboard.thumbnail_width * storyboard.columns).toString(), - height: (storyboard.thumbnail_height * storyboard.rows).toString() - }, [ - this.#el(document, 'EssentialProperty', { - schemeIdUri: 'http://dashif.org/thumbnail_tile', - value: `${storyboard.columns}x${storyboard.rows}` - }), - this.#el(document, 'SegmentTemplate', { - media: url_transformer(new URL(url.replace('$M', '$Number$'))).toString(), - duration: (duration / storyboard.storyboard_count).toString(), - startNumber: '0' - }) - ]); - - set.appendChild(representation); - } - - static async #generateSegmentInformation(document: XMLDocument, representation: Element, format: Format, url: string, actions?: Actions) { - if (format.is_type_otf) { - if (!actions) { - throw new InnertubeError('Unable to get segment durations for this OTF stream without an Actions instance', { format }); - } - - const { resolved_url, segment_durations } = await this.#getOTFSegmentInformation(url, actions); - const segment_elements = []; - - for (const segment_duration of segment_durations) { - let attributes; - - if (typeof segment_duration.repeat_count === 'undefined') { - attributes = { - d: segment_duration.duration.toString() - }; - } else { - attributes = { - d: segment_duration.duration.toString(), - r: segment_duration.repeat_count.toString() - }; - } - segment_elements.push(this.#el(document, 'S', attributes)); - } - - representation.appendChild( - this.#el(document, 'SegmentTemplate', { - startNumber: '1', - timescale: '1000', - initialization: `${resolved_url}&sq=0`, - media: `${resolved_url}&sq=$Number$` - }, [ - this.#el(document, 'SegmentTimeline', {}, segment_elements) - ]) - ); - } else { - if (!format.index_range || !format.init_range) - throw new InnertubeError('Index and init ranges not available', { format }); - - representation.appendChild( - this.#el(document, 'BaseURL', {}, [ - document.createTextNode(url) - ]) - ); - representation.appendChild( - this.#el(document, 'SegmentBase', { - indexRange: `${format.index_range.start}-${format.index_range.end}` - }, [ - this.#el(document, 'Initialization', { - range: `${format.init_range.start}-${format.init_range.end}` - }) - ]) - ); - } - } - - static async #getOTFSegmentInformation(url: string, actions: Actions): Promise<{ - resolved_url: string, - segment_durations: { - duration: number, - repeat_count?: number - }[] - }> { - // Fetch the first segment as it contains the segment durations which we need to generate the manifest - const response = await actions.session.http.fetch_function(`${url}&rn=0&sq=0`, { +import { InnertubeError, Platform, streamToIterable } from './Utils.ts'; +import type { IPlayabilityStatus, IStreamingData } from '../parser/index.ts'; +import type { DownloadOptions, FormatOptions } from '../types/FormatUtils.ts'; + +export async function download( + options: DownloadOptions, + actions: Actions, + playability_status?: IPlayabilityStatus, + streaming_data?: IStreamingData, + player?: Player, + cpn?: string +): Promise> { + if (playability_status?.status === 'UNPLAYABLE') + throw new InnertubeError('Video is unplayable', { error_type: 'UNPLAYABLE' }); + if (playability_status?.status === 'LOGIN_REQUIRED') + throw new InnertubeError('Video is login required', { error_type: 'LOGIN_REQUIRED' }); + if (!streaming_data) + throw new InnertubeError('Streaming data not available.', { error_type: 'NO_STREAMING_DATA' }); + + const opts: DownloadOptions = { + quality: '360p', + type: 'video+audio', + format: 'mp4', + range: undefined, + ...options + }; + + const format = chooseFormat(opts, streaming_data); + const format_url = format.decipher(player); + + // If we're not downloading the video in chunks, we just use fetch once. + if (opts.type === 'video+audio' && !options.range) { + const response = await actions.session.http.fetch_function(`${format_url}&cpn=${cpn}`, { method: 'GET', headers: Constants.STREAM_HEADERS, redirect: 'follow' }); - // Example OTF video: https://www.youtube.com/watch?v=DJ8GQUNUXGM + // Throw if the response is not 2xx + if (!response.ok) + throw new InnertubeError('The server responded with a non 2xx status code', { error_type: 'FETCH_FAILED', response }); - // There might have been redirects, if there were we want to write the resolved URL to the manifest - // So that the player doesn't have to follow the redirects every time it requests a segment - const resolved_url = response.url.replace('&rn=0', '').replace('&sq=0', ''); + const body = response.body; - // In this function we only need the segment durations and how often the durations are repeated - // The segment count could be useful for other stuff though - // The response body contains a lot of junk but the useful stuff looks like this: - // Segment-Count: 922\r\n' + - // 'Segment-Durations-Ms: 5120(r=920),3600,\r\n' - const response_text = await response.text(); + if (!body) + throw new InnertubeError('Could not get ReadableStream from fetch Response.', { error_type: 'FETCH_FAILED', response }); - const segment_duration_strings = getStringBetweenStrings(response_text, 'Segment-Durations-Ms:', '\r\n')?.split(','); - - if (!segment_duration_strings) { - throw new InnertubeError('Failed to extract the segment durations from this OTF stream', { url }); - } - - const segment_durations = []; - for (const segment_duration_string of segment_duration_strings) { - const trimmed_segment_duration = segment_duration_string.trim(); - if (trimmed_segment_duration.length === 0) { - continue; - } - - let repeat_count; - - const repeat_count_string = getStringBetweenStrings(trimmed_segment_duration, '(r=', ')'); - if (repeat_count_string) { - repeat_count = parseInt(repeat_count_string); - } - - segment_durations.push({ - duration: parseInt(trimmed_segment_duration), - repeat_count - }); - } - - return { - resolved_url, - segment_durations - }; + return body; } + + // We need to download in chunks. + + const chunk_size = 1048576 * 10; // 10MB + + let chunk_start = (options.range ? options.range.start : 0); + let chunk_end = (options.range ? options.range.end : chunk_size); + let must_end = false; + + let cancel: AbortController; + + const readable_stream = new Platform.shim.ReadableStream({ + // eslint-disable-next-line @typescript-eslint/no-empty-function + start() { }, + pull: async (controller) => { + if (must_end) { + controller.close(); + return; + } + + if ((chunk_end >= (format.content_length ? format.content_length : 0)) || options.range) { + must_end = true; + } + + return new Promise(async (resolve, reject) => { + try { + cancel = new AbortController(); + + const response = await actions.session.http.fetch_function(`${format_url}&cpn=${cpn}&range=${chunk_start}-${chunk_end || ''}`, { + method: 'GET', + headers: { + ...Constants.STREAM_HEADERS + // XXX: use YouTube's range parameter instead of a Range header. + // Range: `bytes=${chunk_start}-${chunk_end}` + }, + signal: cancel.signal + }); + + const body = response.body; + + if (!body) + throw new InnertubeError('Could not get ReadableStream from fetch Response.', { error_type: 'FETCH_FAILED', response }); + + for await (const chunk of streamToIterable(body)) { + controller.enqueue(chunk); + } + + chunk_start = chunk_end + 1; + chunk_end += chunk_size; + + resolve(); + + } catch (e: any) { + reject(e); + } + }); + }, + async cancel(reason) { + cancel.abort(reason); + } + }, { + highWaterMark: 1, // TODO: better value? + size(chunk) { + return chunk.byteLength; + } + }); + + return readable_stream; } -export default FormatUtils; \ No newline at end of file +/** + * Selects the format that best matches the given options. + * @param options - Options + * @param streaming_data - Streaming data + */ +export function chooseFormat(options: FormatOptions, streaming_data?: IStreamingData): Format { + if (!streaming_data) + throw new InnertubeError('Streaming data not available'); + + const formats = [ + ...(streaming_data.formats || []), + ...(streaming_data.adaptive_formats || []) + ]; + + const requires_audio = options.type ? options.type.includes('audio') : true; + const requires_video = options.type ? options.type.includes('video') : true; + const language = options.language || 'original'; + const quality = options.quality || 'best'; + + let best_width = -1; + + const is_best = [ 'best', 'bestefficiency' ].includes(quality); + const use_most_efficient = quality !== 'best'; + + let candidates = formats.filter((format) => { + if (requires_audio && !format.has_audio) + return false; + if (requires_video && !format.has_video) + return false; + if (options.format !== 'any' && !format.mime_type.includes(options.format || 'mp4')) + return false; + if (!is_best && format.quality_label !== quality) + return false; + if (best_width < format.width) + best_width = format.width; + return true; + }); + + if (!candidates.length) + throw new InnertubeError('No matching formats found', { options }); + + if (is_best && requires_video) + candidates = candidates.filter((format) => format.width === best_width); + + if (requires_audio && !requires_video) { + const audio_only = candidates.filter((format) => { + if (language !== 'original') { + return !format.has_video && format.language === language; + } + return !format.has_video && format.is_original; + + }); + if (audio_only.length > 0) { + candidates = audio_only; + } + } + + if (use_most_efficient) { + // Sort by bitrate (lower is better) + candidates.sort((a, b) => a.bitrate - b.bitrate); + } else { + // Sort by bitrate (higher is better) + candidates.sort((a, b) => b.bitrate - a.bitrate); + } + + return candidates[0]; +} + +export { toDash } from './DashManifest.js'; diff --git a/deno/src/utils/HTTPClient.ts b/deno/src/utils/HTTPClient.ts index 2636893f..dd7c3b25 100644 --- a/deno/src/utils/HTTPClient.ts +++ b/deno/src/utils/HTTPClient.ts @@ -95,6 +95,8 @@ export default class HTTPClient { if (Platform.shim.server) { if (n_body.context.client.clientName === 'ANDROID' || n_body.context.client.clientName === 'ANDROID_MUSIC') { request_headers.set('User-Agent', Constants.CLIENTS.ANDROID.USER_AGENT); + } else if (n_body.context.client.clientName === 'iOS') { + request_headers.set('User-Agent', Constants.CLIENTS.iOS.USER_AGENT); } } @@ -139,7 +141,8 @@ export default class HTTPClient { // Check if 2xx if (response.ok) { return response; - } throw new InnertubeError(`Request to ${response.url} failed with status ${response.status}`, await response.text()); + } + throw new InnertubeError(`Request to ${response.url} failed with status ${response.status}`, await response.text()); } #adjustContext(ctx: Context, client: string): void { @@ -157,6 +160,12 @@ export default class HTTPClient { } switch (client) { + case 'iOS': + ctx.client.deviceModel = Constants.CLIENTS.iOS.DEVICE_MODEL; + ctx.client.clientVersion = Constants.CLIENTS.iOS.VERSION; + ctx.client.clientName = Constants.CLIENTS.iOS.NAME; + ctx.client.platform = 'MOBILE'; + break; case 'YTMUSIC': ctx.client.clientVersion = Constants.CLIENTS.YTMUSIC.VERSION; ctx.client.clientName = Constants.CLIENTS.YTMUSIC.NAME; diff --git a/deno/src/utils/StreamingInfo.ts b/deno/src/utils/StreamingInfo.ts new file mode 100644 index 00000000..2e0603dd --- /dev/null +++ b/deno/src/utils/StreamingInfo.ts @@ -0,0 +1,633 @@ +import type Actions from '../core/Actions.ts'; +import type Player from '../core/Player.ts'; +import type { StoryboardData } from '../parser/classes/PlayerStoryboardSpec.ts'; +import type { IStreamingData } from '../parser/index.ts'; +import type { Format } from '../parser/misc.ts'; +import type { PlayerStoryboardSpec } from '../parser/nodes.ts'; +import type { FormatFilter, URLTransformer } from '../types/FormatUtils.ts'; +import { InnertubeError, Platform, getStringBetweenStrings } from './Utils.ts'; +import { Constants } from './index.ts'; + +export interface StreamingInfo { + duration: number; + audio_sets: AudioSet[]; + video_sets: VideoSet[]; + image_sets: ImageSet[]; +} + +export interface AudioSet { + mime_type: string; + language?: string; + codecs?: string; + audio_sample_rate?: number; + track_name?: string; + track_role?: 'main' | 'dub' | 'description' | 'alternate'; + channels?: number; + representations: AudioRepresentation[]; +} + +export interface Range { + start: number; + end: number; +} + +export type SegmentInfo = { + is_oft: false, + base_url: string; + index_range: Range; + init_range: Range; +} | { + is_oft: true, + getSegmentTemplate(): Promise +} + +export interface Segment { + duration: number, + repeat_count?: number +} + +export interface SegmentTemplate { + init_url: string, + media_url: string, + timeline: Segment[] +} + +export interface AudioRepresentation { + uid: string; + bitrate: number; + codecs?: string; + audio_sample_rate?: number; + channels?: number; + segment_info: SegmentInfo; +} + +export interface VideoSet { + mime_type: string; + color_info: ColorInfo; + codecs?: string; + fps?: number; + representations: VideoRepresentation[] +} + +export interface VideoRepresentation { + uid: string; + bitrate: number; + width: number; + height: number; + fps?: number; + codecs?: string; + segment_info: SegmentInfo; +} + +export interface ColorInfo { + primaries?: '1' | '9', + transfer_characteristics?: '1' | '14' | '16' | '18', + matrix_coefficients?: '1' | '14' +} + +export interface ImageSet { + probable_mime_type: string; + /** + * Sometimes youtube returns webp instead of jpg despite the file extension being jpg + * So we need to update the mime type to reflect the actual mime type of the response + */ + getMimeType(): Promise; + representations: ImageRepresentation[] +} + +export interface ImageRepresentation { + uid: string; + getBitrate(): Promise; + sheet_width: number; + sheet_height: number; + thumbnail_width: number; + thumbnail_height: number; + rows: number; + columns: number; + template_url: string; + template_duration: number; + getURL(n: number): string; +} + +function getFormatGroupings(formats: Format[]) { + const group_info = new Map(); + + const has_multiple_audio_tracks = formats.some((fmt) => !!fmt.audio_track); + + for (const format of formats) { + if ((!format.index_range || !format.init_range) && !format.is_type_otf) { + continue; + } + const mime_type = format.mime_type.split(';')[0]; + + // Codec without any profile or level information + const just_codec = getStringBetweenStrings(format.mime_type, 'codecs="', '"')?.split('.')[0]; + + // HDR videos have both SDR and HDR vp9 formats, so we want to stick them in different groups + const color_info = format.color_info ? Object.values(format.color_info).join('-') : ''; + + const audio_track_id = format.audio_track?.id || ''; + + const group_id = `${mime_type}-${just_codec}-${color_info}-${audio_track_id}`; + + if (!group_info.has(group_id)) { + group_info.set(group_id, []); + } + group_info.get(group_id)?.push(format); + } + + return { + groups: Array.from(group_info.values()), + has_multiple_audio_tracks + }; +} + +function hoistCodecsIfPossible(formats: Format[], hoisted: string[]) { + if ( + formats.length > 1 && + new Set(formats.map((format) => getStringBetweenStrings(format.mime_type, 'codecs="', '"'))).size === 1 + ) { + hoisted.push('codecs'); + return getStringBetweenStrings(formats[0].mime_type, 'codecs="', '"'); + } +} + +function hoistNumberAttributeIfPossible( + formats: Format[], + property: 'audio_sample_rate' | 'fps', + hoisted: string[] +) { + if (formats.length > 1 && new Set(formats.map((format) => format.fps)).size === 1) { + hoisted.push(property); + return Number(formats[0][property]); + } +} + +function hoistAudioChannelsIfPossible(formats: Format[], hoisted: string[]) { + if (formats.length > 1 && new Set(formats.map((format) => format.audio_channels || 2)).size === 1) { + hoisted.push('AudioChannelConfiguration'); + return formats[0].audio_channels; + } +} + +async function getOTFSegmentTemplate(url: string, actions: Actions): Promise { + // Fetch the first segment as it contains the segment durations which we need to generate the manifest + const response = await actions.session.http.fetch_function(`${url}&rn=0&sq=0`, { + method: 'GET', + headers: Constants.STREAM_HEADERS, + redirect: 'follow' + }); + + // Example OTF video: https://www.youtube.com/watch?v=DJ8GQUNUXGM + + // There might have been redirects, if there were we want to write the resolved URL to the manifest + // So that the player doesn't have to follow the redirects every time it requests a segment + const resolved_url = response.url.replace('&rn=0', '').replace('&sq=0', ''); + + // In this function we only need the segment durations and how often the durations are repeated + // The segment count could be useful for other stuff though + // The response body contains a lot of junk but the useful stuff looks like this: + // Segment-Count: 922\r\n' + + // 'Segment-Durations-Ms: 5120(r=920),3600,\r\n' + const response_text = await response.text(); + + const segment_duration_strings = getStringBetweenStrings(response_text, 'Segment-Durations-Ms:', '\r\n')?.split(','); + + if (!segment_duration_strings) { + throw new InnertubeError('Failed to extract the segment durations from this OTF stream', { url }); + } + + const segment_durations = []; + for (const segment_duration_string of segment_duration_strings) { + const trimmed_segment_duration = segment_duration_string.trim(); + if (trimmed_segment_duration.length === 0) { + continue; + } + + let repeat_count; + + const repeat_count_string = getStringBetweenStrings(trimmed_segment_duration, '(r=', ')'); + if (repeat_count_string) { + repeat_count = parseInt(repeat_count_string); + } + + segment_durations.push({ + duration: parseInt(trimmed_segment_duration), + repeat_count + }); + } + + return { + init_url: `${resolved_url}&sq=0`, + media_url: `${resolved_url}&sq=$Number$`, + timeline: segment_durations + }; +} + +function getSegmentInfo( + format: Format, + url_transformer: URLTransformer, + actions?: Actions, + player?: Player, + cpn?: string +) { + const url = new URL(format.decipher(player)); + url.searchParams.set('cpn', cpn || ''); + + const transformed_url = url_transformer(url).toString(); + + if (format.is_type_otf) { + if (!actions) + throw new InnertubeError('Unable to get segment durations for this OTF stream without an Actions instance', { format }); + + const info: SegmentInfo = { + is_oft: true, + getSegmentTemplate() { + return getOTFSegmentTemplate(transformed_url, actions); + } + }; + + return info; + } + + if (!format.index_range || !format.init_range) + throw new InnertubeError('Index and init ranges not available', { format }); + + const info: SegmentInfo = { + is_oft: false, + base_url: transformed_url, + index_range: format.index_range, + init_range: format.init_range + }; + + return info; +} + +function getAudioRepresentation( + format: Format, + hoisted: string[], + url_transformer: URLTransformer, + actions?: Actions, + player?: Player, + cpn?: string +) { + const url = new URL(format.decipher(player)); + url.searchParams.set('cpn', cpn || ''); + + const rep: AudioRepresentation = { + uid: format.audio_track ? `${format.itag}-${format.audio_track.id}` : format.itag.toString(), + bitrate: format.bitrate, + codecs: !hoisted.includes('codecs') ? getStringBetweenStrings(format.mime_type, 'codecs="', '"') : undefined, + audio_sample_rate: !hoisted.includes('audio_sample_rate') ? format.audio_sample_rate : undefined, + channels: !hoisted.includes('AudioChannelConfiguration') ? format.audio_channels || 2 : undefined, + segment_info: getSegmentInfo(format, url_transformer, actions, player, cpn) + }; + + return rep; +} + +function getTrackRole(format: Format) { + const { audio_track } = format; + + if (!audio_track) + return; + + if (audio_track.audio_is_default) + return 'main'; + + if (format.is_dubbed) + return 'dub'; + + if (format.is_descriptive) + return 'description'; + + return 'alternate'; +} + +function getAudioSet( + formats: Format[], + url_transformer: URLTransformer, + actions?: Actions, + player?: Player, + cpn?: string +) { + const first_format = formats[0]; + const { audio_track } = first_format; + const hoisted: string[] = []; + + const set: AudioSet = { + mime_type: first_format.mime_type.split(';')[0], + language: first_format.language ?? undefined, + codecs: hoistCodecsIfPossible(formats, hoisted), + audio_sample_rate: hoistNumberAttributeIfPossible(formats, 'audio_sample_rate', hoisted), + track_name: audio_track?.display_name, + track_role: getTrackRole(first_format), + channels: hoistAudioChannelsIfPossible(formats, hoisted), + representations: formats.map((format) => getAudioRepresentation(format, hoisted, url_transformer, actions, player, cpn)) + }; + + return set; +} + +const COLOR_PRIMARIES: Record = { + BT709: '1', + BT2020: '9' +}; + +const COLOR_TRANSFER_CHARACTERISTICS: Record = { + BT709: '1', + BT2020_10: '14', + SMPTEST2084: '16', + ARIB_STD_B67: '18' +}; + +// This list is incomplete, as the player.js doesn't currently have any code for matrix coefficients, +// So it doesn't have a list like with the other two, so this is just based on what we've seen in responses +const COLOR_MATRIX_COEFFICIENTS: Record = { + BT709: '1', + BT2020_NCL: '14' +}; + +function getColorInfo(format: Format) { + // Section 5.5 Video source metadata signalling https://dashif.org/docs/IOP-Guidelines/DASH-IF-IOP-Part7-v5.0.0.pdf + // Section 8 Video code points https://www.itu.int/rec/T-REC-H.273-202107-I/en + // The player.js file was also helpful + + const color_info = format.color_info; + const primaries = + color_info?.primaries ? COLOR_PRIMARIES[color_info.primaries] : undefined; + + const transfer_characteristics = + color_info?.transfer_characteristics ? COLOR_TRANSFER_CHARACTERISTICS[color_info.transfer_characteristics] : undefined; + + const matrix_coefficients = + color_info?.matrix_coefficients ? COLOR_MATRIX_COEFFICIENTS[color_info.matrix_coefficients] : undefined; + + if (color_info?.matrix_coefficients && !matrix_coefficients) { + const url = new URL(format.url as string); + + const anonymisedFormat = JSON.parse(JSON.stringify(format)); + anonymisedFormat.url = 'REDACTED'; + anonymisedFormat.signature_cipher = 'REDACTED'; + anonymisedFormat.cipher = 'REDACTED'; + + console.warn(`YouTube.js toDash(): Unknown matrix coefficients "${color_info.matrix_coefficients}", the DASH manifest is still usuable without this.\n` + + `Please report it at ${Platform.shim.info.bugs_url} so we can add support for it.\n` + + `Innertube client: ${url.searchParams.get('c')}\nformat:`, anonymisedFormat); + } + + const info: ColorInfo = { + primaries, + transfer_characteristics, + matrix_coefficients + }; + + return info; +} + +function getVideoRepresentation( + format: Format, + url_transformer: URLTransformer, + hoisted: string[], + player?: Player, + actions?: Actions, + cpn?: string +) { + const rep: VideoRepresentation = { + uid: format.itag.toString(), + bitrate: format.bitrate, + width: format.width, + height: format.height, + codecs: !hoisted.includes('codecs') ? getStringBetweenStrings(format.mime_type, 'codecs="', '"') : undefined, + fps: !hoisted.includes('fps') ? format.fps : undefined, + segment_info: getSegmentInfo(format, url_transformer, actions, player, cpn) + }; + + return rep; +} + +function getVideoSet( + formats: Format[], + url_transformer: URLTransformer, + player?: Player, + actions?: Actions, + cpn?: string +) { + const first_format = formats[0]; + const color_info = getColorInfo(first_format); + const hoisted: string[] = []; + + const set: VideoSet = { + mime_type: first_format.mime_type.split(';')[0], + color_info, + codecs: hoistCodecsIfPossible(formats, hoisted), + fps: hoistNumberAttributeIfPossible(formats, 'fps', hoisted), + representations: formats.map((format) => getVideoRepresentation(format, url_transformer, hoisted, player, actions, cpn)) + }; + + return set; +} + +function getStoryboardInfo( + storyboards: PlayerStoryboardSpec +) { + const mime_info = new Map(); + + for (const storyboard of storyboards.boards) { + const extension = new URL(storyboard.template_url).pathname.split('.').pop(); + + const mime_type = `image/${extension === 'jpg' ? 'jpeg' : extension}`; + + if (!mime_info.has(mime_type)) { + mime_info.set(mime_type, []); + } + mime_info.get(mime_type)?.push(storyboard); + } + + return mime_info; +} + +interface SharedStoryboardResponse { + response?: Promise +} + +async function getStoryboardMimeType( + actions: Actions, + board: StoryboardData, + transform_url: URLTransformer, + probable_mime_type: string, + shared_response: SharedStoryboardResponse +) { + const url = board.template_url; + + const req_url = transform_url(new URL(url.replace('$M', '0'))); + + const res_promise = shared_response.response ? shared_response.response : actions.session.http.fetch_function(req_url, { + method: 'HEAD', + headers: Constants.STREAM_HEADERS + }); + + shared_response.response = res_promise; + + const res = await res_promise; + + return res.headers.get('Content-Type') || probable_mime_type; +} + +async function getStoryboardBitrate( + actions: Actions, + board: StoryboardData, + shared_response: SharedStoryboardResponse +) { + const url = board.template_url; + + const response_promises: Promise[] = []; + + // Set a limit so we don't take forever for long videos + const request_limit = Math.min(board.storyboard_count, 10); + for (let i = 0; i < request_limit; i++) { + const req_url = new URL(url.replace('$M', i.toString())); + + const response_promise = + i === 0 && shared_response.response ? + shared_response.response : + actions.session.http.fetch_function(req_url, { + method: 'HEAD', + headers: Constants.STREAM_HEADERS + }); + + if (i === 0) + shared_response.response = response_promise; + + response_promises.push(response_promise); + } + + // Run the requests in parallel to avoid causing too much delay + const responses = await Promise.all(response_promises); + + const content_lengths = []; + + for (const response of responses) { + content_lengths.push(parseInt(response.headers.get('Content-Length') || '0')); + } + + // This is a rough estimate, so it probably won't reflect that actual peak bitrate + // Hopefully it's close enough, because figuring out the actual peak bitrate would require downloading and analysing all storyboard tiles + const bandwidth = Math.ceil((Math.max(...content_lengths) / (board.rows * board.columns)) * 8); + + return bandwidth; +} + +function getImageRepresentation( + duration: number, + actions: Actions, + board: StoryboardData, + transform_url: URLTransformer, + shared_response: SharedStoryboardResponse +) { + const url = board.template_url; + const template_url = new URL(url.replace('$M', '$Number$')); + + const rep: ImageRepresentation = { + uid: `thumbnails_${board.thumbnail_width}x${board.thumbnail_height}`, + getBitrate() { + return getStoryboardBitrate(actions, board, shared_response); + }, + sheet_width: board.thumbnail_width * board.columns, + sheet_height: board.thumbnail_height * board.rows, + thumbnail_height: board.thumbnail_height, + thumbnail_width: board.thumbnail_width, + rows: board.rows, + columns: board.columns, + template_duration: duration / board.storyboard_count, + template_url: transform_url(template_url).toString(), + getURL(n) { + return template_url.toString().replace('$Number$', n.toString()); + } + }; + + return rep; +} + +function getImageSets( + duration: number, + actions: Actions, + storyboards: PlayerStoryboardSpec, + transform_url: URLTransformer +) { + const mime_info = getStoryboardInfo(storyboards); + + const shared_response: SharedStoryboardResponse = {}; + + return Array.from(mime_info.entries()).map(([ type, boards ]) => ({ + probable_mime_type: type, + getMimeType() { + return getStoryboardMimeType(actions, boards[0], transform_url, type, shared_response); + }, + representations: boards.map((board) => getImageRepresentation(duration, actions, board, transform_url, shared_response)) + })); +} + +export function getStreamingInfo( + streaming_data?: IStreamingData, + url_transformer: URLTransformer = (url) => url, + format_filter?: FormatFilter, + cpn?: string, + player?: Player, + actions?: Actions, + storyboards?: PlayerStoryboardSpec +) { + if (!streaming_data) + throw new InnertubeError('Streaming data not available'); + + const formats = format_filter ? + streaming_data.adaptive_formats.filter((fmt) => !format_filter(fmt)) : + streaming_data.adaptive_formats; + + const duration = formats[0].approx_duration_ms / 1000; + + const { + groups, + has_multiple_audio_tracks + } = getFormatGroupings(formats); + + const { + video_groups, + audio_groups + } = groups.reduce((acc, formats) => { + if (formats[0].has_audio) { + // Some videos with multiple audio tracks, have a broken one, that doesn't have any audio track information + // It seems to be the same as default audio track but broken + // We want to ignore it, as it messes up audio track selection in players and YouTube ignores it too + // At the time of writing, this video has a broken audio track: https://youtu.be/UJeSWbR6W04 + if (has_multiple_audio_tracks && !formats[0].audio_track) + return acc; + + acc.audio_groups.push(formats); + return acc; + } + + acc.video_groups.push(formats); + + return acc; + }, { + video_groups: [] as Format[][], + audio_groups: [] as Format[][] + }); + + const audio_sets = audio_groups.map((formats) => getAudioSet(formats, url_transformer, actions, player, cpn)); + + const video_sets = video_groups.map((formats) => getVideoSet(formats, url_transformer, player, actions, cpn)); + + // XXX: We need to make requests to get the image sizes, so we'll skip the storyboards if we don't have an Actions instance + const image_sets = storyboards && actions ? getImageSets(duration, actions, storyboards, url_transformer) : []; + + const info : StreamingInfo = { + duration, + audio_sets, + video_sets, + image_sets + }; + + return info; +} diff --git a/deno/src/utils/index.ts b/deno/src/utils/index.ts index 5cc4cb2d..2c5208f3 100644 --- a/deno/src/utils/index.ts +++ b/deno/src/utils/index.ts @@ -4,8 +4,7 @@ export * as Constants from './Constants.ts'; export { default as EventEmitter } from './EventEmitterLike.ts'; -export { default as FormatUtils } from './FormatUtils.ts'; -export * from './FormatUtils.ts'; +export * as FormatUtils from './FormatUtils.ts'; export { default as HTTPClient } from './HTTPClient.ts'; export * from './HTTPClient.ts';