feat: add support for generating sessions locally (#277)

* feat: add visitor data proto

* feat: add support for generating session data locally

* chore: add test
This commit is contained in:
LuanRT
2023-01-06 03:06:49 -03:00
committed by GitHub
parent d36389c865
commit 00fa514b03
6 changed files with 303 additions and 105 deletions

View File

@@ -1,12 +1,13 @@
import UniversalCache from '../utils/Cache';
import Constants from '../utils/Constants';
import Constants, { CLIENTS } from '../utils/Constants';
import EventEmitterLike from '../utils/EventEmitterLike';
import Actions from './Actions';
import Player from './Player';
import HTTPClient, { FetchFunction } from '../utils/HTTPClient';
import { DeviceCategory, getRandomUserAgent, InnertubeError, SessionError } from '../utils/Utils';
import { DeviceCategory, generateRandomString, getRandomUserAgent, InnertubeError, SessionError } from '../utils/Utils';
import OAuth, { Credentials, OAuthAuthErrorEventHandler, OAuthAuthEventHandler, OAuthAuthPendingEventHandler } from './OAuth';
import Proto from '../proto';
export enum ClientType {
WEB = 'WEB',
@@ -21,7 +22,7 @@ export interface Context {
client: {
hl: string;
gl: string;
remoteHost: string;
remoteHost?: string;
screenDensityFloat: number;
screenHeightPoints: number;
screenPixelDensity: number;
@@ -38,8 +39,8 @@ export interface Context {
clientFormFactor: string;
userInterfaceTheme: string;
timeZone: string;
browserName: string;
browserVersion: string;
browserName?: string;
browserVersion?: string;
originalUrl: string;
deviceMake: string;
deviceModel: string;
@@ -58,25 +59,72 @@ export interface Context {
}
export interface SessionOptions {
/**
* Language.
*/
lang?: string;
/**
* Geolocation.
*/
location?: string;
/**
* The account index to use. This is useful if you have multiple accounts logged in.
* **NOTE:**
* Only works if you are signed in with cookies.
*/
account_index?: number;
/**
* Specifies whether to retrieve the JS player. Disabling this will make session creation faster.
* **NOTE:** Deciphering formats is not possible without the JS player.
*/
retrieve_player?: boolean;
/**
* Specifies whether to enable safety mode. This will prevent the session from loading any potentially unsafe content.
*/
enable_safety_mode?: boolean;
/**
* Specifies whether to generate the session data locally or retrieve it from YouTube.
* This can be useful if you need more performance.
*/
generate_session_locally?: boolean;
/**
* Platform to use for the session.
*/
device_category?: DeviceCategory;
/**
* InnerTube client type.
*/
client_type?: ClientType;
/**
* The time zone.
*/
timezone?: string;
/**
* Used to cache the deciphering functions from the JS player.
*/
cache?: UniversalCache;
/**
* YouTube cookies.
*/
cookie?: string;
/**
* Fetch function to use.
*/
fetch?: FetchFunction;
}
export interface SessionData {
context: Context;
api_key: string;
api_version: string;
}
export default class Session extends EventEmitterLike {
#api_version;
#key;
#context;
#account_index;
#player;
#api_version: string;
#key: string;
#context: Context;
#account_index: number;
#player?: Player;
oauth: OAuth;
http: HTTPClient;
@@ -121,6 +169,7 @@ export default class Session extends EventEmitterLike {
options.location,
options.account_index,
options.enable_safety_mode,
options.generate_session_locally,
options.device_category,
options.client_type,
options.timezone,
@@ -135,30 +184,49 @@ export default class Session extends EventEmitterLike {
}
static async getSessionData(
lang = 'en-US',
lang = '',
location = '',
account_index = 0,
enable_safety_mode = false,
generate_session_locally = false,
device_category: DeviceCategory = 'desktop',
client_name: ClientType = ClientType.WEB,
tz: string = Intl.DateTimeFormat().resolvedOptions().timeZone,
fetch: FetchFunction = globalThis.fetch
) {
let session_data: SessionData;
if (generate_session_locally) {
session_data = this.#generateSessionData({ lang, location, time_zone: tz, device_category, client_name, enable_safety_mode });
} else {
session_data = await this.#retrieveSessionData({ lang, location, time_zone: tz, device_category, client_name, enable_safety_mode }, fetch);
}
return { ...session_data, account_index };
}
static async #retrieveSessionData(options: {
lang: string;
location: string;
time_zone: string;
device_category: string;
client_name: string;
enable_safety_mode: boolean;
}, fetch: FetchFunction = globalThis.fetch): Promise<SessionData> {
const url = new URL('/sw.js_data', Constants.URLS.YT_BASE);
const res = await fetch(url, {
headers: {
'accept-language': lang,
'accept-language': options.lang || 'en-US',
'user-agent': getRandomUserAgent('desktop'),
'accept': '*/*',
'referer': 'https://www.youtube.com/sw.js',
'cookie': `PREF=tz=${tz.replace('/', '.')}`
'cookie': `PREF=tz=${options.time_zone.replace('/', '.')}`
}
});
if (!res.ok) {
throw new SessionError(`Failed to get session data: ${res.status}`);
}
if (!res.ok)
throw new SessionError(`Failed to retrieve session data: ${res.status}`);
const text = await res.text();
const data = JSON.parse(text.replace(/^\)\]\}'/, ''));
@@ -172,22 +240,22 @@ export default class Session extends EventEmitterLike {
const context: Context = {
client: {
hl: device_info[0],
gl: location || device_info[2],
gl: options.location || device_info[2],
remoteHost: device_info[3],
screenDensityFloat: 1,
screenHeightPoints: 720,
screenHeightPoints: 1080,
screenPixelDensity: 1,
screenWidthPoints: 1280,
screenWidthPoints: 1920,
visitorData: device_info[13],
userAgent: device_info[14],
clientName: client_name,
clientName: options.client_name,
clientVersion: device_info[16],
osName: device_info[17],
osVersion: device_info[18],
platform: device_category.toUpperCase(),
platform: options.device_category.toUpperCase(),
clientFormFactor: 'UNKNOWN_FORM_FACTOR',
userInterfaceTheme: 'USER_INTERFACE_THEME_LIGHT',
timeZone: device_info[79],
timeZone: device_info[79] || options.time_zone,
browserName: device_info[86],
browserVersion: device_info[87],
originalUrl: Constants.URLS.YT_BASE,
@@ -196,7 +264,7 @@ export default class Session extends EventEmitterLike {
utcOffsetMinutes: new Date().getTimezoneOffset()
},
user: {
enableSafetyMode: enable_safety_mode,
enableSafetyMode: options.enable_safety_mode,
lockedSafetyMode: false
},
request: {
@@ -204,7 +272,53 @@ export default class Session extends EventEmitterLike {
}
};
return { context, api_key, api_version, account_index };
return { context, api_key, api_version };
}
static #generateSessionData(options: {
lang: string;
location: string;
time_zone: string;
device_category: DeviceCategory;
client_name: string;
enable_safety_mode: boolean
}): SessionData {
const id = generateRandomString(11);
const timestamp = Math.floor(Date.now() / 1000);
const context: Context = {
client: {
hl: options.lang || 'en',
gl: options.location || 'US',
screenDensityFloat: 1,
screenHeightPoints: 1080,
screenPixelDensity: 1,
screenWidthPoints: 1920,
visitorData: Proto.encodeVisitorData(id, timestamp),
userAgent: getRandomUserAgent('desktop'),
clientName: options.client_name,
clientVersion: CLIENTS.WEB.VERSION,
osName: 'Windows',
osVersion: '10.0',
platform: options.device_category.toUpperCase(),
clientFormFactor: 'UNKNOWN_FORM_FACTOR',
userInterfaceTheme: 'USER_INTERFACE_THEME_LIGHT',
timeZone: options.time_zone,
originalUrl: Constants.URLS.YT_BASE,
deviceMake: '',
deviceModel: '',
utcOffsetMinutes: new Date().getTimezoneOffset()
},
user: {
enableSafetyMode: options.enable_safety_mode,
lockedSafetyMode: false
},
request: {
useSsl: true
}
};
return { context, api_key: CLIENTS.WEB.API_KEY, api_version: CLIENTS.WEB.API_VERSION };
}
async signIn(credentials?: Credentials): Promise<void> {

View File

@@ -2,9 +2,14 @@ import { CLIENTS } from '../utils/Constants';
import { u8ToBase64 } from '../utils/Utils';
import { VideoMetadata } from '../core/Studio';
import { ChannelAnalytics, CreateCommentParams, GetCommentsSectionParams, InnertubePayload, LiveMessageParams, MusicSearchFilter, NotificationPreferences, PeformCommentActionParams, SearchFilter, SearchFilter_Filters } from './youtube';
import { ChannelAnalytics, CreateCommentParams, GetCommentsSectionParams, InnertubePayload, LiveMessageParams, MusicSearchFilter, NotificationPreferences, PeformCommentActionParams, SearchFilter, SearchFilter_Filters, VisitorData } from './youtube';
class Proto {
static encodeVisitorData(id: string, timestamp: number): string {
const buf = VisitorData.toBinary({ id, timestamp });
return encodeURIComponent(u8ToBase64(buf).replace(/\+/g, '-').replace(/\//g, '_'));
}
static encodeChannelAnalyticsParams(channel_id: string): string {
const buf = ChannelAnalytics.toBinary({
params: {

View File

@@ -3,12 +3,9 @@
syntax = "proto2";
package youtube;
message ChannelAnalytics {
message Params {
required string channel_id = 1001;
}
required Params params = 32;
message VisitorData {
required string id = 1;
required int32 timestamp = 5;
}
message InnertubePayload {
@@ -91,6 +88,14 @@ message InnertubePayload {
optional VideoThumbnail video_thumbnail = 20;
}
message ChannelAnalytics {
message Params {
required string channel_id = 1001;
}
required Params params = 32;
}
message SoundInfoParams {
message Sound {
message Params {

View File

@@ -15,22 +15,17 @@ import { reflectionMergePartial } from "@protobuf-ts/runtime";
import { MESSAGE_TYPE } from "@protobuf-ts/runtime";
import { MessageType } from "@protobuf-ts/runtime";
/**
* @generated from protobuf message youtube.ChannelAnalytics
* @generated from protobuf message youtube.VisitorData
*/
export interface ChannelAnalytics {
export interface VisitorData {
/**
* @generated from protobuf field: youtube.ChannelAnalytics.Params params = 32;
* @generated from protobuf field: string id = 1;
*/
params?: ChannelAnalytics_Params;
}
/**
* @generated from protobuf message youtube.ChannelAnalytics.Params
*/
export interface ChannelAnalytics_Params {
id: string;
/**
* @generated from protobuf field: string channel_id = 1001;
* @generated from protobuf field: int32 timestamp = 5;
*/
channelId: string;
timestamp: number;
}
/**
* @generated from protobuf message youtube.InnertubePayload
@@ -213,6 +208,24 @@ export interface InnertubePayload_VideoThumbnail_Thumbnail {
*/
imageData: Uint8Array;
}
/**
* @generated from protobuf message youtube.ChannelAnalytics
*/
export interface ChannelAnalytics {
/**
* @generated from protobuf field: youtube.ChannelAnalytics.Params params = 32;
*/
params?: ChannelAnalytics_Params;
}
/**
* @generated from protobuf message youtube.ChannelAnalytics.Params
*/
export interface ChannelAnalytics_Params {
/**
* @generated from protobuf field: string channel_id = 1001;
*/
channelId: string;
}
/**
* @generated from protobuf message youtube.SoundInfoParams
*/
@@ -646,26 +659,30 @@ export interface SearchFilter_Filters {
featuresVr180?: number;
}
// @generated message type with reflection information, may provide speed optimized methods
class ChannelAnalytics$Type extends MessageType<ChannelAnalytics> {
class VisitorData$Type extends MessageType<VisitorData> {
constructor() {
super("youtube.ChannelAnalytics", [
{ no: 32, name: "params", kind: "message", T: () => ChannelAnalytics_Params }
super("youtube.VisitorData", [
{ no: 1, name: "id", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
{ no: 5, name: "timestamp", kind: "scalar", T: 5 /*ScalarType.INT32*/ }
]);
}
create(value?: PartialMessage<ChannelAnalytics>): ChannelAnalytics {
const message = {};
create(value?: PartialMessage<VisitorData>): VisitorData {
const message = { id: "", timestamp: 0 };
globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this });
if (value !== undefined)
reflectionMergePartial<ChannelAnalytics>(this, message, value);
reflectionMergePartial<VisitorData>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: ChannelAnalytics): ChannelAnalytics {
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: VisitorData): VisitorData {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* youtube.ChannelAnalytics.Params params */ 32:
message.params = ChannelAnalytics_Params.internalBinaryRead(reader, reader.uint32(), options, message.params);
case /* string id */ 1:
message.id = reader.string();
break;
case /* int32 timestamp */ 5:
message.timestamp = reader.int32();
break;
default:
let u = options.readUnknownField;
@@ -678,10 +695,13 @@ class ChannelAnalytics$Type extends MessageType<ChannelAnalytics> {
}
return message;
}
internalBinaryWrite(message: ChannelAnalytics, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* youtube.ChannelAnalytics.Params params = 32; */
if (message.params)
ChannelAnalytics_Params.internalBinaryWrite(message.params, writer.tag(32, WireType.LengthDelimited).fork(), options).join();
internalBinaryWrite(message: VisitorData, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string id = 1; */
if (message.id !== "")
writer.tag(1, WireType.LengthDelimited).string(message.id);
/* int32 timestamp = 5; */
if (message.timestamp !== 0)
writer.tag(5, WireType.Varint).int32(message.timestamp);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
@@ -689,56 +709,9 @@ class ChannelAnalytics$Type extends MessageType<ChannelAnalytics> {
}
}
/**
* @generated MessageType for protobuf message youtube.ChannelAnalytics
* @generated MessageType for protobuf message youtube.VisitorData
*/
export const ChannelAnalytics = new ChannelAnalytics$Type();
// @generated message type with reflection information, may provide speed optimized methods
class ChannelAnalytics_Params$Type extends MessageType<ChannelAnalytics_Params> {
constructor() {
super("youtube.ChannelAnalytics.Params", [
{ no: 1001, name: "channel_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ }
]);
}
create(value?: PartialMessage<ChannelAnalytics_Params>): ChannelAnalytics_Params {
const message = { channelId: "" };
globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this });
if (value !== undefined)
reflectionMergePartial<ChannelAnalytics_Params>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: ChannelAnalytics_Params): ChannelAnalytics_Params {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string channel_id */ 1001:
message.channelId = reader.string();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: ChannelAnalytics_Params, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string channel_id = 1001; */
if (message.channelId !== "")
writer.tag(1001, WireType.LengthDelimited).string(message.channelId);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message youtube.ChannelAnalytics.Params
*/
export const ChannelAnalytics_Params = new ChannelAnalytics_Params$Type();
export const VisitorData = new VisitorData$Type();
// @generated message type with reflection information, may provide speed optimized methods
class InnertubePayload$Type extends MessageType<InnertubePayload> {
constructor() {
@@ -1456,6 +1429,100 @@ class InnertubePayload_VideoThumbnail_Thumbnail$Type extends MessageType<Innertu
*/
export const InnertubePayload_VideoThumbnail_Thumbnail = new InnertubePayload_VideoThumbnail_Thumbnail$Type();
// @generated message type with reflection information, may provide speed optimized methods
class ChannelAnalytics$Type extends MessageType<ChannelAnalytics> {
constructor() {
super("youtube.ChannelAnalytics", [
{ no: 32, name: "params", kind: "message", T: () => ChannelAnalytics_Params }
]);
}
create(value?: PartialMessage<ChannelAnalytics>): ChannelAnalytics {
const message = {};
globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this });
if (value !== undefined)
reflectionMergePartial<ChannelAnalytics>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: ChannelAnalytics): ChannelAnalytics {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* youtube.ChannelAnalytics.Params params */ 32:
message.params = ChannelAnalytics_Params.internalBinaryRead(reader, reader.uint32(), options, message.params);
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: ChannelAnalytics, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* youtube.ChannelAnalytics.Params params = 32; */
if (message.params)
ChannelAnalytics_Params.internalBinaryWrite(message.params, writer.tag(32, WireType.LengthDelimited).fork(), options).join();
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message youtube.ChannelAnalytics
*/
export const ChannelAnalytics = new ChannelAnalytics$Type();
// @generated message type with reflection information, may provide speed optimized methods
class ChannelAnalytics_Params$Type extends MessageType<ChannelAnalytics_Params> {
constructor() {
super("youtube.ChannelAnalytics.Params", [
{ no: 1001, name: "channel_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ }
]);
}
create(value?: PartialMessage<ChannelAnalytics_Params>): ChannelAnalytics_Params {
const message = { channelId: "" };
globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this });
if (value !== undefined)
reflectionMergePartial<ChannelAnalytics_Params>(this, message, value);
return message;
}
internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: ChannelAnalytics_Params): ChannelAnalytics_Params {
let message = target ?? this.create(), end = reader.pos + length;
while (reader.pos < end) {
let [fieldNo, wireType] = reader.tag();
switch (fieldNo) {
case /* string channel_id */ 1001:
message.channelId = reader.string();
break;
default:
let u = options.readUnknownField;
if (u === "throw")
throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`);
let d = reader.skip(wireType);
if (u !== false)
(u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d);
}
}
return message;
}
internalBinaryWrite(message: ChannelAnalytics_Params, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter {
/* string channel_id = 1001; */
if (message.channelId !== "")
writer.tag(1001, WireType.LengthDelimited).string(message.channelId);
let u = options.writeUnknownFields;
if (u !== false)
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
return writer;
}
}
/**
* @generated MessageType for protobuf message youtube.ChannelAnalytics.Params
*/
export const ChannelAnalytics_Params = new ChannelAnalytics_Params$Type();
// @generated message type with reflection information, may provide speed optimized methods
class SoundInfoParams$Type extends MessageType<SoundInfoParams> {
constructor() {
super("youtube.SoundInfoParams", [

View File

@@ -35,7 +35,9 @@ export const OAUTH = Object.freeze({
export const CLIENTS = Object.freeze({
WEB: {
NAME: 'WEB',
VERSION: '2.20220902.01.00'
VERSION: '2.20230104.01.00',
API_KEY: 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
API_VERSION: 'v1'
},
YTMUSIC: {
NAME: 'WEB_REMIX',

View File

@@ -141,6 +141,11 @@ describe('YouTube.js Tests', () => {
expect(nop_yt.session.player).toBeUndefined();
});
it('should create a session from data generated locally', async () => {
const loc_yt = await Innertube.create({ generate_session_locally: true, retrieve_player: false });
expect(loc_yt.session.context).toBeDefined();
});
it('should resolve a URL', async () => {
const url = await yt.resolveURL('https://www.youtube.com/@linustechtips');
expect(url.payload.browseId).toBe(CHANNELS[0].ID);