From 317bca261c2974a6fb5996212c8685ef62d25ce7 Mon Sep 17 00:00:00 2001 From: LuanRT Date: Mon, 29 Aug 2022 04:48:33 -0300 Subject: [PATCH] feat(download): bring back `WEB` client (#156) * refactor: remove dead code and integrate with Jinter * chore: tidy up --- package-lock.json | 34 ++- package.json | 1 + src/Innertube.ts | 29 +- src/core/Player.ts | 124 ++++---- src/deciphers/NToken.ts | 465 ------------------------------ src/deciphers/Signature.ts | 140 --------- src/parser/classes/misc/Format.ts | 6 +- src/parser/youtube/VideoInfo.ts | 25 +- src/utils/Constants.ts | 36 +-- 9 files changed, 132 insertions(+), 728 deletions(-) delete mode 100644 src/deciphers/NToken.ts delete mode 100644 src/deciphers/Signature.ts diff --git a/package-lock.json b/package-lock.json index dc731144..06f4fa44 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "license": "MIT", "dependencies": { "@protobuf-ts/runtime": "^2.7.0", + "jintr": "^0.1.9", "linkedom": "^0.14.12", "undici": "^5.7.0" }, @@ -1563,10 +1564,9 @@ } }, "node_modules/acorn": { - "version": "8.7.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.7.1.tgz", - "integrity": "sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A==", - "dev": true, + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz", + "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==", "bin": { "acorn": "bin/acorn" }, @@ -3973,6 +3973,17 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/jintr": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/jintr/-/jintr-0.1.9.tgz", + "integrity": "sha512-0fB9LtzM/Pf7rGI8TViLM2cfLQTRWAdRjwmYWpMamHPAmX083zR2kj6cXTqtJ4v90+//r+SdZY9RSwTQsNetQw==", + "funding": [ + "https://github.com/sponsors/LuanRT" + ], + "dependencies": { + "acorn": "^8.8.0" + } + }, "node_modules/jju": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/jju/-/jju-1.4.0.tgz", @@ -6491,10 +6502,9 @@ } }, "acorn": { - "version": "8.7.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.7.1.tgz", - "integrity": "sha512-Xx54uLJQZ19lKygFXOWsscKUbsBZW0CPykPhVQdhIeIwrbPmJzqeASDInc8nKBnp/JT6igTs82qPXz069H8I/A==", - "dev": true + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz", + "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==" }, "acorn-jsx": { "version": "5.3.2", @@ -8195,6 +8205,14 @@ } } }, + "jintr": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/jintr/-/jintr-0.1.9.tgz", + "integrity": "sha512-0fB9LtzM/Pf7rGI8TViLM2cfLQTRWAdRjwmYWpMamHPAmX083zR2kj6cXTqtJ4v90+//r+SdZY9RSwTQsNetQw==", + "requires": { + "acorn": "^8.8.0" + } + }, "jju": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/jju/-/jju-1.4.0.tgz", diff --git a/package.json b/package.json index 7b60a4e6..9dfff8c9 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "license": "MIT", "dependencies": { "@protobuf-ts/runtime": "^2.7.0", + "jintr": "^0.1.9", "linkedom": "^0.14.12", "undici": "^5.7.0" }, diff --git a/src/Innertube.ts b/src/Innertube.ts index 273a837f..6fa45a77 100644 --- a/src/Innertube.ts +++ b/src/Innertube.ts @@ -48,6 +48,8 @@ export interface SearchFilters { sort_by?: 'relevance' | 'rating' | 'upload_date' | 'view_count'; } +export type InnerTubeClient = 'ANDROID' | 'WEB' | 'YTMUSIC'; + class Innertube { session; account; @@ -74,16 +76,10 @@ class Innertube { /** * Retrieves video info. */ - async getInfo(video_id: string | undefined) { - throwIfMissing({ video_id }); - + async getInfo(video_id: string, client?: InnerTubeClient) { const cpn = generateRandomString(16); - const initial_info = await this.actions.execute('/player', { - client: 'ANDROID', - videoId: video_id - }); - + const initial_info = await this.actions.getVideoInfo(video_id, cpn, client); const continuation = this.actions.next({ video_id }); const response = await Promise.all([ initial_info, continuation ]); @@ -93,15 +89,9 @@ class Innertube { /** * Retrieves basic video info. */ - async getBasicInfo(video_id: string | undefined) { - throwIfMissing({ video_id }); - + async getBasicInfo(video_id: string, client?: InnerTubeClient) { const cpn = generateRandomString(16); - - const response = await this.actions.execute('/player', { - client: 'ANDROID', - videoId: video_id - }); + const response = await this.actions.getVideoInfo(video_id, cpn, client); return new VideoInfo([ response ], this.actions, this.session.player, cpn); } @@ -246,13 +236,12 @@ class Innertube { } /** - * Downloads a given video. If you only need the direct download link take a look at {@link getStreamingData}. + * Downloads a given video. If you only need the direct download link see {@link getStreamingData}. * * If you wish to retrieve the video info too, have a look at {@link getBasicInfo} or {@link getInfo}. */ - async download(video_id: string | undefined, options?: DownloadOptions) { - throwIfMissing({ video_id }); - const info = await this.getBasicInfo(video_id); + async download(video_id: string, options?: DownloadOptions) { + const info = await this.getBasicInfo(video_id, options?.client); return info.download(options); } diff --git a/src/core/Player.ts b/src/core/Player.ts index 06e0b047..c79019f7 100644 --- a/src/core/Player.ts +++ b/src/core/Player.ts @@ -1,25 +1,24 @@ + import { FetchFunction } from '../utils/HTTPClient'; import { getRandomUserAgent, getStringBetweenStrings, PlayerError } from '../utils/Utils'; import Constants from '../utils/Constants'; import UniversalCache from '../utils/Cache'; - -// Import NToken from '../deciphers/NToken'; -// Import Signature from '../deciphers/Signature'; +// See https://github.com/LuanRT/Jinter +import Jinter from 'jintr'; export default class Player { - // #ntoken; - // #signature; - - #signature_timestamp; + #nsig_sc; + #sig_sc; + #sig_sc_timestamp; #player_id; - constructor(signature_timestamp: number, player_id: string) { - // This.#ntoken = ntoken; - // This.#signature = signature; + constructor(signature_timestamp: number, sig_sc: string, nsig_sc: string, player_id: string) { + this.#nsig_sc = nsig_sc; + this.#sig_sc = sig_sc; - this.#signature_timestamp = signature_timestamp; + this.#sig_sc_timestamp = signature_timestamp; this.#player_id = player_id; } @@ -61,14 +60,13 @@ export default class Player { const sig_timestamp = this.extractSigTimestamp(player_js); - // Const sig_decipher_sc = this.extractSigDecipherSc(player_js); - // Const ntoken_sc = this.extractNTokenSc(player_js); + const sig_sc = this.extractSigSourceCode(player_js); + const nsig_sc = this.extractNSigSourceCode(player_js); - return await Player.fromSource(cache, sig_timestamp, player_id); + return await Player.fromSource(cache, sig_timestamp, sig_sc, nsig_sc, player_id); } - /* - Decipher(url?: string, signature_cipher?: string, cipher?: string) { + decipher(url?: string, signature_cipher?: string, cipher?: string) { url = url || signature_cipher || cipher; if (!url) @@ -80,7 +78,11 @@ export default class Player { url_components.searchParams.set('ratebypass', 'yes'); if (signature_cipher || cipher) { - const signature = this.#signature.decipher(url); + const sig_decipher = new Jinter(this.#sig_sc); + sig_decipher.scope.set('sig', args.get('s')); + + const signature = sig_decipher.interpret(); + const sp = args.get('sp'); sp ? @@ -91,12 +93,20 @@ export default class Player { const n = url_components.searchParams.get('n'); if (n) { - const ntoken = this.#ntoken.transform(n); - url_components.searchParams.set('n', ntoken); + const nsig_decipher = new Jinter(this.#nsig_sc); + nsig_decipher.scope.set('nsig', n); + + const nsig = nsig_decipher.interpret(); + + if (nsig.startsWith('enhanced_except_')) { + console.warn('Warning:\nCould not transform nsig, download may be throttled.\nChanging the InnerTube client to "ANDROID" might help!'); + } + + url_components.searchParams.set('n', nsig); } return url_components.toString(); - }*/ + } static async fromCache(cache: UniversalCache, player_id: string) { const buffer = await cache.get(player_id); @@ -112,17 +122,20 @@ export default class Player { const sig_timestamp = view.getUint32(4, true); - /* - * Const sig_decipher_len = view.getUint32(8, true); - * const sig_decipher_buf = buffer.slice(12, 12 + sig_decipher_len); - * const ntoken_transform_buf = buffer.slice(12 + sig_decipher_len); - */ + const sig_len = view.getUint32(8, true); + const sig_buf = buffer.slice(12, 12 + sig_len); + const nsig_buf = buffer.slice(12 + sig_len); - return new Player(sig_timestamp, player_id); + const decoder = new TextDecoder(); + + const sig_sc = decoder.decode(sig_buf); + const nsig_sc = decoder.decode(nsig_buf); + + return new Player(sig_timestamp, sig_sc, nsig_sc, player_id); } - static async fromSource(cache: UniversalCache | undefined, sig_timestamp: number, player_id: string) { - const player = new Player(sig_timestamp, player_id); + static async fromSource(cache: UniversalCache | undefined, sig_timestamp: number, sig_sc: string, nsig_sc: string, player_id: string) { + const player = new Player(sig_timestamp, sig_sc, nsig_sc, player_id); await player.cache(cache); return player; } @@ -130,52 +143,41 @@ export default class Player { async cache(cache?: UniversalCache) { if (!cache) return; - /** - * Const ntoken_buf = this.#ntoken.toArrayBuffer(); - * const sig_decipher_buf = this.#signature.toArrayBuffer(); - */ + const encoder = new TextEncoder(); - const buffer = new ArrayBuffer(12 /* + sig_decipher_buf.byteLength + ntoken_buf.byteLength */); + const sig_buf = encoder.encode(this.#sig_sc); + const nsig_buf = encoder.encode(this.#nsig_sc); + + const buffer = new ArrayBuffer(12 + sig_buf.byteLength + nsig_buf.byteLength); const view = new DataView(buffer); view.setUint32(0, Player.LIBRARY_VERSION, true); - view.setUint32(4, this.#signature_timestamp, true); + view.setUint32(4, this.#sig_sc_timestamp, true); + view.setUint32(8, sig_buf.byteLength, true); - // View.setUint32(8, sig_decipher_buf.byteLength, true); - - // New Uint8Array(buffer).set(new Uint8Array(sig_decipher_buf), 12); - // New Uint8Array(buffer).set(new Uint8Array(ntoken_buf), 12 + sig_decipher_buf.byteLength); + new Uint8Array(buffer).set(sig_buf, 12); + new Uint8Array(buffer).set(nsig_buf, 12 + sig_buf.byteLength); await cache.set(this.#player_id, new Uint8Array(buffer)); } - /** - * Extracts the signature timestamp from the player source code. - */ static extractSigTimestamp(data: string) { return parseInt(getStringBetweenStrings(data, 'signatureTimestamp:', ',') || '0'); } - /** - * Extracts the signature decipher algorithm. - */ - static extractSigDecipherSc(data: string) { - const sig_alg_sc = getStringBetweenStrings(data, 'this.audioTracks};var', '};'); - const sig_data = getStringBetweenStrings(data, 'function(a){a=a.split("")', 'return a.join("")}'); + static extractSigSourceCode(data: string) { + const funcs = getStringBetweenStrings(data, 'this.audioTracks};var', '};'); + const calls = getStringBetweenStrings(data, 'function(a){a=a.split("")', 'return a.join("")}'); - if (!sig_alg_sc || !sig_data) + if (!funcs || !calls) throw new PlayerError('Failed to extract signature decipher algorithm'); - return sig_alg_sc + sig_data; + return `function descramble_sig(a) { a = a.split(""); ${funcs}}${calls} return a.join("") } descramble_sig(sig);`; } - /** - * Extracts the n-token decipher algorithm. - */ - static extractNTokenSc(data: string) { - const sc = `var b=a.split("")${getStringBetweenStrings(data, 'b=a.split("")', '}return b.join("")}')}} return b.join("");`; + static extractNSigSourceCode(data: string) { + const sc = `function descramble_nsig(a) { let b=a.split("")${getStringBetweenStrings(data, 'b=a.split("")', '}return b.join("")}')}} return b.join(""); } descramble_nsig(nsig)`; - console.log(sc); if (!sc) throw new PlayerError('Failed to extract n-token decipher algorithm'); @@ -187,10 +189,18 @@ export default class Player { } get sts() { - return this.#signature_timestamp; + return this.#sig_sc_timestamp; + } + + get nsig_sc() { + return this.#nsig_sc; + } + + get sig_sc() { + return this.#sig_sc; } static get LIBRARY_VERSION() { - return 1; + return 2; } } \ No newline at end of file diff --git a/src/deciphers/NToken.ts b/src/deciphers/NToken.ts deleted file mode 100644 index 28167ec8..00000000 --- a/src/deciphers/NToken.ts +++ /dev/null @@ -1,465 +0,0 @@ -import { NTOKEN_REGEX, BASE64_DIALECT } from '../utils/Constants'; -import { InnertubeError } from '../utils/Utils'; - -export enum NTokenTransformOperation { - NO_OP = 0, - PUSH, - REVERSE_1, - REVERSE_2, - SPLICE, - SWAP0_1, - SWAP0_2, - ROTATE_1, - ROTATE_2, - BASE64_DIA, - TRANSLATE_1, - TRANSLATE_2, -} - -export enum NTokenTransformOpType { - FUNC, - N_ARR, - LITERAL, - REF -} - -const OP_LOOKUP: Record = { - 'd.push(e)': NTokenTransformOperation.PUSH, - 'd.reverse()': NTokenTransformOperation.REVERSE_1, - 'function(d){for(var': NTokenTransformOperation.REVERSE_2, - 'd.length;d.splice(e,1)': NTokenTransformOperation.SPLICE, - 'd[0])[0])': NTokenTransformOperation.SWAP0_1, - 'f=d[0];d[0]': NTokenTransformOperation.SWAP0_2, - 'reverse().forEach': NTokenTransformOperation.ROTATE_1, - 'unshift(d.pop())': NTokenTransformOperation.ROTATE_2, - 'function(){for(var': NTokenTransformOperation.BASE64_DIA, - 'function(d,e){for(var f': NTokenTransformOperation.TRANSLATE_1, - 'function(d,e,f){var': NTokenTransformOperation.TRANSLATE_2 -}; - -export class NTokenTransforms { - /** - * Gets a base64 alphabet and uses it as a lookup table to modify n. - */ - static translate1(arr: any[], token: string, is_reverse_base64: boolean) { - const characters = is_reverse_base64 && BASE64_DIALECT.REVERSE || BASE64_DIALECT.NORMAL; - const that = token.split(''); - arr.forEach((char, index, loc) => { - that.push(loc[index] = characters[(characters.indexOf(char) - characters.indexOf(that[index]) + 64) % characters.length]); - }); - } - - static translate2(arr: any[], token: string, characters: string[]) { - let chars_length = characters.length; - const that = token.split(''); - arr.forEach((char, index, loc) => { - that.push(loc[index] = characters[(characters.indexOf(char) - characters.indexOf(that[index]) + index + chars_length--) % characters.length]); - }); - } - - /** - * Returns the requested base64 dialect, currently this is only used by 'translate2'. - */ - static getBase64Dia(is_reverse_base64: boolean) { - const characters = is_reverse_base64 && BASE64_DIALECT.REVERSE || BASE64_DIALECT.NORMAL; - return characters; - } - - /** - * Swaps the first element with the one at the given index. - */ - static swap0(arr: any[], index: number) { - const old_elem = arr[0]; - index = (index % arr.length + arr.length) % arr.length; - arr[0] = arr[index]; - arr[index] = old_elem; - } - - /** - * Rotates elements of the array. - */ - static rotate(arr: any[], index: number) { - index = (index % arr.length + arr.length) % arr.length; - arr.splice(-index).reverse().forEach((el) => arr.unshift(el)); - } - - /** - * Deletes one element at the given index. - */ - static splice(arr: any[], index: number) { - index = (index % arr.length + arr.length) % arr.length; - arr.splice(index, 1); - } - - static reverse(arr: any[]) { - arr.reverse(); - } - - static push(arr: any[], item: any) { - if (Array.isArray(arr?.[0])) { - arr.push([ NTokenTransformOpType.LITERAL, item ]); - } else { - arr.push(item); - } - } -} - -const TRANSFORM_FUNCTIONS: [Record, Record] = [ { - [NTokenTransformOperation.PUSH]: NTokenTransforms.push, - [NTokenTransformOperation.SPLICE]: NTokenTransforms.splice, - [NTokenTransformOperation.SWAP0_1]: NTokenTransforms.swap0, - [NTokenTransformOperation.SWAP0_2]: NTokenTransforms.swap0, - [NTokenTransformOperation.ROTATE_1]: NTokenTransforms.rotate, - [NTokenTransformOperation.ROTATE_2]: NTokenTransforms.rotate, - [NTokenTransformOperation.REVERSE_1]: NTokenTransforms.reverse, - [NTokenTransformOperation.REVERSE_2]: NTokenTransforms.reverse, - [NTokenTransformOperation.BASE64_DIA]: () => NTokenTransforms.getBase64Dia(false), - [NTokenTransformOperation.TRANSLATE_1]: (...args: any[]) => NTokenTransforms.translate1.apply(null, [ ...args, false ] as any), - [NTokenTransformOperation.TRANSLATE_2]: NTokenTransforms.translate2 -}, { - [NTokenTransformOperation.PUSH]: NTokenTransforms.push, - [NTokenTransformOperation.SPLICE]: NTokenTransforms.splice, - [NTokenTransformOperation.SWAP0_1]: NTokenTransforms.swap0, - [NTokenTransformOperation.SWAP0_2]: NTokenTransforms.swap0, - [NTokenTransformOperation.ROTATE_1]: NTokenTransforms.rotate, - [NTokenTransformOperation.ROTATE_2]: NTokenTransforms.rotate, - [NTokenTransformOperation.REVERSE_1]: NTokenTransforms.reverse, - [NTokenTransformOperation.REVERSE_2]: NTokenTransforms.reverse, - [NTokenTransformOperation.BASE64_DIA]: () => NTokenTransforms.getBase64Dia(true), - [NTokenTransformOperation.TRANSLATE_1]: (...args: any[]) => NTokenTransforms.translate1.apply(null, [ ...args, true ] as any), - [NTokenTransformOperation.TRANSLATE_2]: NTokenTransforms.translate2 -} ]; - -export type NTokenCall = [number, number[]]; -export type NTokenInstruction = [NTokenTransformOpType, (NTokenTransformOperation | number)?, number?]; -export type NTokenTransformer = [NTokenInstruction[], NTokenCall[]]; - -export default class NToken { - private transformer: NTokenTransformer; - - constructor(transformer: NTokenTransformer) { - this.transformer = transformer; - } - - static fromSourceCode(raw: string) { - const transformation_data = NToken.getTransformationData(raw); - - const transformations = transformation_data.map((el) => { - if (el != null && typeof el != 'number') { - const is_reverse_base64 = el.includes('case 65:'); - - const func = NToken.getFunc(el)?.[0]; - - const opcode = func ? OP_LOOKUP[func] : undefined; - - if (opcode) { - el = [ NTokenTransformOpType.FUNC, opcode, 0 + is_reverse_base64 ]; - } else if (el == 'b') { - el = [ NTokenTransformOpType.N_ARR ]; - } else { - el = [ NTokenTransformOpType.LITERAL, el ]; - } - } else if (el != null) { - el = [ NTokenTransformOpType.LITERAL, el ]; - } - - return el; - }); - - // Fills all placeholders with the transformations array - const placeholder_indexes = [ ...raw.matchAll(NTOKEN_REGEX.PLACEHOLDERS) ].map((item) => parseInt(item[1])); - placeholder_indexes.forEach((i) => transformations[i] = [ NTokenTransformOpType.REF ]); - - // Parses and emulates calls to the functions of the transformations array - const function_body = raw.replace(/\n/g, '').match(/try\{(.*?)\}catch/s)?.[1]; - - if (!function_body) { - throw new InnertubeError('Invalid NToken transformation function.', { transformation: raw }); - } - - const function_calls = [ ...function_body.matchAll(NTOKEN_REGEX.CALLS) ].map((params) => [ - parseInt(params[1]), - params[2].split(',').map((param: string) => { - const param_value = param.match(/c\[(.*?)\]/)?.[1]; - - if (!param_value) { - throw new InnertubeError('Unexpected NToken transformation function parameter.', { transformation: raw, param }); - } - - return parseInt(param_value); - }) - ] as NTokenCall - ); - - return new NToken([ transformations, function_calls ]); - } - - private evaluate(i: NTokenInstruction, nToken: string[], transformer: NTokenTransformer) { - switch (i[0]) { - case NTokenTransformOpType.FUNC: - if (i[1] === undefined || i[2] === undefined) - throw new InnertubeError('Invalid NTokenInstruction.', { transformation: nToken, instruction: i }); - return TRANSFORM_FUNCTIONS[i[2]][i[1]]; - case NTokenTransformOpType.N_ARR: - return nToken; - case NTokenTransformOpType.LITERAL: - return i[1]; - case NTokenTransformOpType.REF: - return transformer[0]; - } - } - - transform(n: string) { - const nToken = n.split(''); - - // We must copy since we will modify the array - const transformer: NTokenTransformer = this.getTransformerClone(); - - try { - transformer[1].forEach(([ index, param_index ]) => { - const base64_dia = (param_index[2] !== undefined && this.evaluate(transformer[0][param_index[2]], nToken, transformer)()); - this.evaluate(transformer[0][index], nToken, transformer)( - param_index[0] !== undefined && this.evaluate(transformer[0][param_index[0]], nToken, transformer) || undefined, - param_index[1] !== undefined && this.evaluate(transformer[0][param_index[1]], nToken, transformer) || undefined, - base64_dia || undefined - ); - }); - } catch (e) { - console.error(new Error(`Could not transform n-token, download may be throttled.\nOriginal Token:${n}\nError:\n${(e as Error).stack}`)); - return n; - } - - return nToken.join(''); - } - - private getTransformerClone(): NTokenTransformer { - return [ [ ...this.transformer[0] ], [ ...this.transformer[1] ] ]; - } - - toJSON() { - return this.getTransformerClone(); - } - - toArrayBuffer() { - // (16 bit FUNC instructions) 2 bit op - 1 bit is_reverse_base64 - 4 bit nonce - 8 bit operation - // (8 bit N_ARG and REF) 2 bit op - 6 bit nonce - // (40 bit LITERAL) 2 bit op - 6 bit nonce - 32 bit value - - // NTokenCall will be 8 bit for the index, 8 bit for the number of parameters, and 8 bit for each parameter - - // We've got a 3 * 32 bit header to store the library version and the size of the two arrays - - let size = 4 * 3; - - for (const instruction of this.transformer[0]) { - switch (instruction[0]) { - case NTokenTransformOpType.FUNC: - size += 2; - break; - case NTokenTransformOpType.N_ARR: - case NTokenTransformOpType.REF: - size += 1; - break; - case NTokenTransformOpType.LITERAL: - if (typeof instruction[1] === 'string') { - size += 1 + 4 + new TextEncoder().encode(instruction[1] as string).byteLength; - } - size += 4 + 1; - break; - } - } - - for (const call of this.transformer[1]) { - size += 2 + call[1].length; - } - - const buffer = new ArrayBuffer(size); - const view = new DataView(buffer); - - let offset = 0; - - view.setUint32(offset, NToken.LIBRARY_VERSION, true); - offset += 4; - - view.setUint32(offset, this.transformer[0].length, true); - offset += 4; - - view.setUint32(offset, this.transformer[1].length, true); - offset += 4; - - for (const instruction of this.transformer[0]) { - switch (instruction[0]) { - case NTokenTransformOpType.FUNC: - { - if (instruction[1] === undefined || instruction[2] === undefined) - throw new InnertubeError('Invalid NTokenInstruction.', { transformation: this.transformer, instruction }); - - const opcode = (instruction[0] << 6) | instruction[2]; - - view.setUint8(offset, opcode); - offset += 1; - - view.setUint8(offset, instruction[1]); - offset += 1; - } - break; - case NTokenTransformOpType.N_ARR: - case NTokenTransformOpType.REF: - { - const opcode = (instruction[0] << 6); - view.setUint8(offset, opcode); - offset += 1; - } - break; - case NTokenTransformOpType.LITERAL: - { - if (instruction[1] === undefined) - throw new InnertubeError('Invalid NTokenInstruction.', { transformation: this.transformer, instruction }); - - const type = typeof instruction[1] === 'string' ? 1 : 0; - - const opcode = (instruction[0] << 6) | type; - - view.setUint8(offset, opcode); - offset += 1; - - if (type === 0) { - view.setInt32(offset, instruction[1], true); - offset += 4; - } else { - const encoded = new TextEncoder().encode(instruction[1] as any); - - view.setUint32(offset, encoded.byteLength, true); - offset += 4; - - for (let i = 0; i < encoded.byteLength; i++) { - view.setUint8(offset, encoded[i]); - offset += 1; - } - } - } - break; - } - } - - for (const call of this.transformer[1]) { - view.setUint8(offset, call[0]); - offset += 1; - - view.setUint8(offset, call[1].length); - offset += 1; - - for (const param of call[1]) { - view.setUint8(offset, param); - offset += 1; - } - } - - return buffer; - } - - static fromArrayBuffer(buffer: ArrayBuffer) { - const view = new DataView(buffer); - let offset = 0; - - const version = view.getUint32(offset, true); - offset += 4; - - if (version !== NToken.LIBRARY_VERSION) - throw new TypeError('Invalid library version'); - - const transformations_length = view.getUint32(offset, true); - offset += 4; - - const function_calls_length = view.getUint32(offset, true); - offset += 4; - - const transformations = new Array(transformations_length); - - for (let i = 0; i < transformations_length; i++) { - const opcode = view.getUint8(offset++); - const op = opcode >> 6; - - switch (op) { - case NTokenTransformOpType.FUNC: - { - const is_reverse_base64 = opcode & 0b00000001; - const operation = view.getUint8(offset++); - transformations[i] = [ op, operation, is_reverse_base64 ]; - } - break; - case NTokenTransformOpType.N_ARR: - case NTokenTransformOpType.REF: - { - transformations[i] = [ op ]; - } - break; - case NTokenTransformOpType.LITERAL: - { - const type = opcode & 0b00000001; - - if (type === 0) { - const literal = view.getInt32(offset, true); - offset += 4; - transformations[i] = [ op, literal ]; - } else { - const length = view.getUint32(offset, true); - offset += 4; - - const literal = new Uint8Array(length); - - for (let i = 0; i < length; i++) { - literal[i] = view.getUint8(offset++); - } - - transformations[i] = [ op, new TextDecoder().decode(literal) as any ]; - } - } - break; - default: - throw new Error('Invalid opcode'); - } - } - - const function_calls = new Array(function_calls_length); - - for (let i = 0; i < function_calls_length; i++) { - const index = view.getUint8(offset++); - - const num_params = view.getUint8(offset++); - const params = new Array(num_params); - - for (let j = 0; j < num_params; j++) { - params[j] = view.getUint8(offset++); - } - - function_calls[i] = [ index, params ]; - } - - return new NToken([ transformations, function_calls ]); - } - - static get LIBRARY_VERSION(): number { - return 1; - } - - static getFunc(el: string) { - return el.match(NTOKEN_REGEX.FUNCTIONS); - } - - static getTransformationData(raw: string) { - const data = `[${raw.replace(/\n/g, '').match(/c=\[(.*?)\];c/s)?.[1]}]`; - return JSON.parse(this.refineNTokenData(data)) as any[]; - } - - static refineNTokenData(data: string) { - // TODO: refactor this - return data - .replace(/function\(d,e\)/g, '"function(d,e)').replace(/function\(d\)/g, '"function(d)') - .replace(/function\(\)/g, '"function()').replace(/function\(d,e,f\)/g, '"function(d,e,f)') - .replace(/\[function\(d,e,f\)/g, '["function(d,e,f)').replace(/,b,/g, ',"b",') - .replace(/,b/g, ',"b"').replace(/b,/g, '"b",').replace(/b]/g, '"b"]') - .replace(/\[b/g, '["b"').replace(/}]/g, '"]').replace(/},/g, '}",') - .replace(/""/g, '').replace(/length]\)}"/g, 'length])}'); - } -} \ No newline at end of file diff --git a/src/deciphers/Signature.ts b/src/deciphers/Signature.ts deleted file mode 100644 index 7999aa53..00000000 --- a/src/deciphers/Signature.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { SIG_REGEX } from '../utils/Constants'; - -export enum SignatureOperation { - REVERSE, - SPLICE, - SWAP -} - -export type SignatureInstruction = [ SignatureOperation, number ]; - -export default class Signature { - private action_sequence; - - constructor(action_sequence: SignatureInstruction[]) { - this.action_sequence = action_sequence; - } - - static fromSourceCode(raw_sc: string) { - let func: RegExpExecArray | null; - const functions = []; - - while ((func = SIG_REGEX.FUNCTIONS.exec(raw_sc)) !== null) { - if (func[0].includes('reverse')) { - functions[0] = func[1]; - } else if (func[0].includes('splice')) { - functions[1] = func[1]; - } else { - functions[2] = func[1]; - } - } - - let actions: RegExpExecArray | null; - - const action_sequence: SignatureInstruction[] = []; - - while ((actions = SIG_REGEX.ACTIONS.exec(raw_sc)) !== null) { - const action = actions.groups; - if (!action) continue; - switch (action.name) { - case functions[0]: - action_sequence.push([ SignatureOperation.REVERSE, 0 ]); - break; - case functions[1]: - action_sequence.push([ SignatureOperation.SPLICE, parseInt(action.param) ]); - break; - case functions[2]: - action_sequence.push([ SignatureOperation.SWAP, parseInt(action.param) ]); - break; - default: - } - } - - return new Signature(action_sequence); - } - - decipher(url: string) { - const args = new URLSearchParams(url); - const signature = args.get('s')?.split(''); - - if (!signature) - throw new TypeError('Invalid input signature'); - - for (const action of this.action_sequence) { - switch (action[0]) { - case SignatureOperation.REVERSE: - signature.reverse(); - break; - case SignatureOperation.SPLICE: - signature.splice(0, action[1]); - break; - case SignatureOperation.SWAP: - const index = action[1]; - const orig_arr = signature[0]; - signature[0] = signature[index % signature.length]; - signature[index % signature.length] = orig_arr; - break; - default: - break; - } - } - - return signature.join(''); - } - - toJSON() { - return [ ...this.action_sequence ]; - } - - toArrayBuffer() { - // Array buffer encoding assumes that the index of the action is a short (16 bit unsigned) - const buffer = new ArrayBuffer(4 + 4 + this.action_sequence.length * (1 + 2)); - const view = new DataView(buffer); - - let offset = 0; - - view.setUint32(offset, Signature.LIBRARY_VERSION, true); - offset += 4; - - view.setUint32(offset, this.action_sequence.length, true); - offset += 4; - - for (let i = 0; i < this.action_sequence.length; i++) { - view.setUint8(offset, this.action_sequence[i][0]); - offset += 1; - - view.setUint16(offset, this.action_sequence[i][1], true); - offset += 2; - } - - return buffer; - } - - static fromArrayBuffer(buffer: ArrayBuffer) { - const view = new DataView(buffer); - - let offset = 0; - - const version = view.getUint32(offset, true); - offset += 4; - - if (version !== Signature.LIBRARY_VERSION) - throw new TypeError('Invalid library version'); - - const action_sequenceLength = view.getUint32(offset, true); - offset += 4; - - const action_sequence = new Array(action_sequenceLength); - - for (let i = 0; i < action_sequenceLength; i++) { - action_sequence[i] = [ view.getUint8(offset), view.getUint16(offset + 1, true) ]; - offset += 3; - } - - return new Signature(action_sequence); - } - - static get LIBRARY_VERSION() { - return 1; - } -} \ No newline at end of file diff --git a/src/parser/classes/misc/Format.ts b/src/parser/classes/misc/Format.ts index 4512e168..df561358 100644 --- a/src/parser/classes/misc/Format.ts +++ b/src/parser/classes/misc/Format.ts @@ -1,3 +1,5 @@ +import Player from '../../../core/Player'; + class Format { itag: string; mime_type: string; @@ -71,8 +73,8 @@ class Format { * Decipher the streaming url of the format. * @returns Deciphered URL. */ - decipher(): string { - return this.url; + decipher(player: Player): string { + return player.decipher(this.url, this.signature_cipher, this.cipher); } } diff --git a/src/parser/youtube/VideoInfo.ts b/src/parser/youtube/VideoInfo.ts index df195341..02dff195 100644 --- a/src/parser/youtube/VideoInfo.ts +++ b/src/parser/youtube/VideoInfo.ts @@ -17,6 +17,7 @@ import PlayerOverlay from '../classes/PlayerOverlay'; import ToggleButton from '../classes/ToggleButton'; import CommentsEntryPointHeader from '../classes/comments/CommentsEntryPointHeader'; import ContinuationItem from '../classes/ContinuationItem'; +import PlayerMicroformat from '../classes/PlayerMicroformat'; import LiveChat from '../classes/LiveChat'; import LiveChatWrap from './LiveChat'; @@ -42,6 +43,10 @@ export interface FormatOptions { * File format, use 'any' to download any format */ format?: string; + /** + * InnerTube client, can be ANDROID, WEB or YTMUSIC + */ + client?: 'ANDROID' | 'WEB' | 'YTMUSIC' } export interface DownloadOptions extends FormatOptions { @@ -95,8 +100,22 @@ class VideoInfo { if (info.playability_status?.status === 'ERROR') throw new InnertubeError('This video is unavailable', info.playability_status); + if (info.microformat && !info.microformat?.is(PlayerMicroformat)) + throw new InnertubeError('Invalid microformat', info.microformat); + this.basic_info = { // This type is inferred so no need for an explicit type ...info.video_details, + ...{ + /** + * Microformat is a bit redundant, so only + * a few things there are interesting to us. + */ + embed: info.microformat?.embed, + channel: info.microformat?.channel, + is_unlisted: info.microformat?.is_unlisted, + is_family_safe: info.microformat?.is_family_safe, + has_ypc_metadata: info.microformat?.has_ypc_metadata + }, like_count: undefined as number | undefined, is_liked: undefined as boolean | undefined, is_disliked: undefined as boolean | undefined @@ -423,7 +442,7 @@ class VideoInfo { if (!format.index_range || !format.init_range) throw new InnertubeError('Index and init ranges not available', { format }); - const url = new URL(format.decipher()); + const url = new URL(format.decipher(this.#player)); url.searchParams.set('cpn', this.#cpn); set.appendChild(this.#el(document, 'Representation', { @@ -453,7 +472,7 @@ class VideoInfo { if (!format.index_range || !format.init_range) throw new InnertubeError('Index and init ranges not available', { format }); - const url = new URL(format.decipher()); + const url = new URL(format.decipher(this.#player)); url.searchParams.set('cpn', this.#cpn); set.appendChild(this.#el(document, 'Representation', { @@ -498,7 +517,7 @@ class VideoInfo { }; const format = this.chooseFormat(opts); - const format_url = format.decipher(); + const format_url = format.decipher(this.#player); // If we're not downloading the video in chunks, we just use fetch once. if (opts.type === 'video+audio' && !options.range) { diff --git a/src/utils/Constants.ts b/src/utils/Constants.ts index f322470d..e70e4c2e 100644 --- a/src/utils/Constants.ts +++ b/src/utils/Constants.ts @@ -47,7 +47,6 @@ export const CLIENTS = Object.freeze({ }); export const STREAM_HEADERS = Object.freeze({ 'accept': '*/*', - // XXX: undici doesnt like this, 'connection': 'keep-alive', 'origin': 'https://www.youtube.com', 'referer': 'https://www.youtube.com', 'DNT': '?1' @@ -69,41 +68,12 @@ export const ACCOUNT_SETTINGS = Object.freeze({ PLAYLISTS_PRIVACY: 'PRIVACY_DISCOVERABLE_SAVED_PLAYLISTS', SUBSCRIPTIONS_PRIVACY: 'PRIVACY_DISCOVERABLE_SUBSCRIPTIONS' }); -export const BASE64_DIALECT = Object.freeze({ - NORMAL: 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'.split(''), - REVERSE: '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_'.split('') -}); -export const SIG_REGEX = Object.freeze({ - ACTIONS: /;.{2}\.(?.{2})\(.*?,(?.*?)\)/g, - FUNCTIONS: /(?.{2}):function\(.*?\){(.*?)}/g -}); -export const NTOKEN_REGEX = Object.freeze({ - CALLS: /c\[(.*?)\]\((.+?)\)/g, - PLACEHOLDERS: /c\[(.*?)\]=c/g, - FUNCTIONS: /d\.push\(e\)|d\.reverse\(\)|d\[0\]\)\[0\]\)|f=d\[0];d\[0\]|d\.length;d\.splice\(e,1\)|function\(\){for\(var|function\(d,e,f\){var|function\(d\){for\(var|reverse\(\)\.forEach|unshift\(d\.pop\(\)\)|function\(d,e\){for\(var f/ -}); -export const FUNCS = Object.freeze({ - PUSH: 'd.push(e)', - REVERSE_1: 'd.reverse()', - REVERSE_2: 'function(d){for(var', - SPLICE: 'd.length;d.splice(e,1)', - SWAP0_1: 'd[0])[0])', - SWAP0_2: 'f=d[0];d[0]', - ROTATE_1: 'reverse().forEach', - ROTATE_2: 'unshift(d.pop())', - BASE64_DIA: 'function(){for(var', - TRANSLATE_1: 'function(d,e){for(var f', - TRANSLATE_2: 'function(d,e,f){var' -}); + export default { URLS, OAUTH, CLIENTS, STREAM_HEADERS, INNERTUBE_HEADERS_BASE, - ACCOUNT_SETTINGS, - BASE64_DIALECT, - SIG_REGEX, - NTOKEN_REGEX, - FUNCS -}; + ACCOUNT_SETTINGS +}; \ No newline at end of file