Files
YouTube.js/lib/utils/Cache.ts
Daniel Wykerd fb68e6bcfe feat!: better cross runtime support (#97)
* refactor: remove dependancies

removes node-forge and uuid in favor of Web APIs

* refactor!: commonjs to es6

To aid with #93 I will make all my changes in TypeScript instead.
This is the first step into making that happen.

Used: https://github.com/wessberg/cjstoesm

* refactor!: NToken and Signature TS files

Bring this PR up to speed with #93

* feat: cross platform cache (WIP)

this is untested!
should remove idb as dependecy.

* feat: EventEmitter polyfill

* refactor: remove events

* feat: HTTPClient based on Fetch API (WIP)

* refactor!: parsers refactor (WIP)

Initial TS support for parsers as per #93

This adds several type safety checks to the parser which'll help to
ensure valid data is returned by the parser.

* refactor!: parsers refactor (WIP)

Bring more in line with the existing implementations & make less verbose

* refactor!: parser refactor

I was overcomplicating things, this is much simpler and compatible with
the existing JS API

* fix: some missed parsers while refactoring

* fix: better type inferance for parseResponse

* feat(TS): typesafe YTNode casts

* feat: more type safety in YTNode and Parser

* refactor: VideoInfo download with fetch & TS (WIP)

Again, this also does some work for #93

* fix: LiveChat in VideoInfo

* refactor!: more typesafety in parser

* refactor!: VideoInfo almost completed

* refactor!: player and session refactors

- Remove the Player class' dependance on Session.
- Add additional context to the Session.

* refactor!: move auth logic to Session (WIP)

* refactor: TS port for Actions and Innertube

My fingers hurt from typing out all those types :-P

* refactor: NavigationEndpoint TS

this is still a WIP and should be improved.
NavigationEndpoint should probably be refactored further.

* refactor!: VideoInfo compiles without errors

* chore: delete old player

* fix: import errors

It compiles and runs!!

* fix: Utils import fixes

* fix: several runtime errors

* fix: video streaming

* chore: remove console.log debugging

Whoops, forgot to remove these before I pushed the previous commit

* chore: remove old unused dependencies

* fix: typescript errors

Now emitting declarations and source maps

* refactor: TS feed

* chore: delete old Feed

* refactor: move streamToIterable into Utils

* refactor: AccountManager TS

* refactor: FilterableFeed to TS

* refactor: InteractionManager to TS

* refactor: PlaylistManager to TS

* refactor: TabbedFeed to TS

* refactor: Music to TS (WIP)

more work to be done, see TODO comments

* fix: getting the tests to pass (6/12)

YouTube.js Tests
    Search
      ✓ Should search on YouTube (1152 ms)
      ✕ Should search on YouTube Music (705 ms)
      ✕ Should retrieve YouTube search suggestions (722 ms)
      ✓ Should retrieve YouTube Music search suggestions (233 ms)
    Comments
      ✓ Should retrieve comments (585 ms)
      ✕ Should retrieve next batch of comments (221 ms)
      ✕ Should retrieve comment replies (1 ms)
    General
      ✕ Should retrieve playlist with YouTube (732 ms)
      ✓ Should retrieve home feed (838 ms)
      ✓ Should retrieve trending content (543 ms)
      ✓ Should retrieve video info (639 ms)
      ✕ Should download video (5 ms)

* fix: tests (7/12)

YouTube.js Tests
    Search
      ✓ Should search on YouTube (1984 ms)
      ✕ Should search on YouTube Music (1139 ms)
      ✕ Should retrieve YouTube search suggestions (1433 ms)
      ✓ Should retrieve YouTube Music search suggestions (529 ms)
    Comments
      ✓ Should retrieve comments (324 ms)
      ✓ Should retrieve next batch of comments (395 ms)
      ✕ Should retrieve comment replies
    General
      ✕ Should retrieve playlist with YouTube (653 ms)
      ✓ Should retrieve home feed (1085 ms)
      ✓ Should retrieve trending content (513 ms)
      ✓ Should retrieve video info (921 ms)
      ✕ Should download video (3 ms)

* fix: download tests (8/12)

YouTube.js Tests
    Search
      ✓ Should search on YouTube (1293 ms)
      ✕ Should search on YouTube Music (927 ms)
      ✕ Should retrieve YouTube search suggestions (1250 ms)
      ✓ Should retrieve YouTube Music search suggestions (258 ms)
    Comments
      ✓ Should retrieve comments (803 ms)
      ✓ Should retrieve next batch of comments (511 ms)
      ✕ Should retrieve comment replies
    General
      ✕ Should retrieve playlist with YouTube (528 ms)
      ✓ Should retrieve home feed (1047 ms)
      ✓ Should retrieve trending content (548 ms)
      ✓ Should retrieve video info (825 ms)
      ✓ Should download video (1779 ms)

* fix: tests (9/12)

YouTube.js Tests
    Search
      ✓ Should search on YouTube (1276 ms)
      ✕ Should search on YouTube Music (955 ms)
      ✓ Should retrieve YouTube search suggestions (661 ms)
      ✓ Should retrieve YouTube Music search suggestions (491 ms)
    Comments
      ✓ Should retrieve comments (624 ms)
      ✓ Should retrieve next batch of comments (353 ms)
      ✕ Should retrieve comment replies
    General
      ✕ Should retrieve playlist with YouTube (672 ms)
      ✓ Should retrieve home feed (1277 ms)
      ✓ Should retrieve trending content (999 ms)
      ✓ Should retrieve video info (1106 ms)
      ✓ Should download video (2514 ms)

* feat: key based type validation for parsers

* fix: comments tests pass (10/12)

YouTube.js Tests
    Search
      ✓ Should search on YouTube (938 ms)
      ✕ Should search on YouTube Music (850 ms)
      ✓ Should retrieve YouTube search suggestions (528 ms)
      ✓ Should retrieve YouTube Music search suggestions (224 ms)
    Comments
      ✓ Should retrieve comments (518 ms)
      ✓ Should retrieve next batch of comments (337 ms)
      ✓ Should retrieve comment replies (358 ms)
    General
      ✕ Should retrieve playlist with YouTube (466 ms)
      ✓ Should retrieve home feed (1051 ms)
      ✓ Should retrieve trending content (623 ms)
      ✓ Should retrieve video info (863 ms)
      ✓ Should download video (2656 ms)

* refactor: type safety checks removing @ts-ignore

* fix: playlist tests pass (11/12)

YouTube.js Tests
    Search
      ✓ Should search on YouTube (991 ms)
      ✕ Should search on YouTube Music (924 ms)
      ✓ Should retrieve YouTube search suggestions (606 ms)
      ✓ Should retrieve YouTube Music search suggestions (225 ms)
    Comments
      ✓ Should retrieve comments (393 ms)
      ✓ Should retrieve next batch of comments (284 ms)
      ✓ Should retrieve comment replies (252 ms)
    General
      ✓ Should retrieve playlist with YouTube (578 ms)
      ✓ Should retrieve home feed (1148 ms)
      ✓ Should retrieve trending content (541 ms)
      ✓ Should retrieve video info (799 ms)
      ✓ Should download video (1419 ms)

* fix: all tests pass for node 🎉

YouTube.js Tests
    Search
      ✓ Should search on YouTube (1053 ms)
      ✓ Should search on YouTube Music (761 ms)
      ✓ Should retrieve YouTube search suggestions (453 ms)
      ✓ Should retrieve YouTube Music search suggestions (221 ms)
    Comments
      ✓ Should retrieve comments (627 ms)
      ✓ Should retrieve next batch of comments (412 ms)
      ✓ Should retrieve comment replies (268 ms)
    General
      ✓ Should retrieve playlist with YouTube (565 ms)
      ✓ Should retrieve home feed (775 ms)
      ✓ Should retrieve trending content (498 ms)
      ✓ Should retrieve video info (875 ms)
      ✓ Should download video (1364 ms)

* build: working Deno bundle

Still need to test whether this bundle works in the browser

* docs: update deno example to download video

* refactor: MusicResponsiveListItem to TS

* docs: TSDoc for Parser helpers

* docs: Parser documentation for TS

* docs: add note about parseItem and parseArray

* test: remove browser tests since they're identical

* feat: browser support and proxy example

* fix: PlaylistManager TS after merge

* feat: in-browser video streaming

* refactor: cleanup the Dash example

* feat: allow custom fetch implementations

* feat: fetch debugger

* fix: OAuth login

* refactor: remove file extensions from imports

* refactor: build scripts

* fix: CustomEvent on node

* fix: LiveChat

* fix: linting

* fix: liniting in build-parser-json

* chore: update test workflow

* fix: NToken errors after lint fixes

* fix: codacy complaints

* docs: update to reflect changes

Definitly needs more work but its a start

* refactor: cleanup imports/exports

* fix: browser example

- Remove user-agent before making request.
- Fix cache on browsers

* fix: cache on node

* fix: stupid mistake

* refactor: Session#signIn to wait untill success

This also splits the 'auth' event up into 3 distinct events:
- 'auth' -> fired on success
- 'auth-pending' -> fired when pending authentication
- 'auth-error' -> fired when an error occurred

* refactor: freeze Constants

* refactor: cleanup HTTPClient Request

* refactor: debugFetch readability

* chore: lint

* refactor: replace jsdoc with tsdoc eslint plugin

remove @param annotations without descriptions

* fix: bunch of liniting warnings

* refactor: better inference on YTNode#is

As suggested by @MasterOfBob777

* fix: linting warnings

* revert: undici import

* refactor: rename `list_type` to `item_type`
2022-07-20 14:06:12 -03:00

241 lines
7.0 KiB
TypeScript

import { getRuntime } from './Utils';
// Browser Cache is based off:
// https://github.com/elias551/simple-kvs/blob/master/src/index.ts
export default class UniversalCache {
#persistent_directory: string;
#persistent: boolean;
constructor(persistent = false, persistent_directory?: string) {
this.#persistent_directory = persistent_directory || UniversalCache.default_persistent_directory;
this.#persistent = persistent;
}
static get temp_directory() {
switch (getRuntime()) {
case 'deno':
const Deno: any = Reflect.get(globalThis, 'Deno');
return `${Deno.env.get('TMPDIR') || Deno.env.get('TMP') || Deno.env.get('TEMP') || '/tmp'}/youtubei.js`;
case 'node':
return `${Reflect.get(module, 'require')('os').tmpdir()}/youtubei.js`;
default:
return '';
}
}
static get default_persistent_directory() {
switch (getRuntime()) {
case 'deno':
const Deno: any = Reflect.get(globalThis, 'Deno');
return `${Deno.cwd()}/.cache/youtubei.js`;
case 'node':
return Reflect.get(module, 'require')('path').resolve(__dirname, '..', '..', '.cache', 'youtubei.js');
default:
return '';
}
}
get cache_dir() {
return this.#persistent ? this.#persistent_directory : UniversalCache.temp_directory;
}
async #createCache() {
const dir = this.cache_dir;
switch (getRuntime()) {
case 'deno':
const Deno: any = Reflect.get(globalThis, 'Deno');
try {
const cwd = await Deno.stat(dir);
if (!cwd.isDirectory)
throw new Error('An unexpected file was found in place of the cache directory');
} catch (e) {
if (e instanceof Deno.errors.NotFound)
await Deno.mkdir(dir, { recursive: true });
else
throw e;
}
break;
case 'node':
const fs = Reflect.get(module, 'require')('fs/promises');
try {
const cwd = await fs.stat(dir);
if (!cwd.isDirectory())
throw new Error('An unexpected file was found in place of the cache directory');
} catch (e: any) {
if (e?.code === 'ENOENT')
await fs.mkdir(dir, { recursive: true });
else
throw e;
}
break;
}
}
#getBrowserDB() {
const indexedDB: IDBFactory = Reflect.get(globalThis, 'indexedDB') || Reflect.get(globalThis, 'webkitIndexedDB') || Reflect.get(globalThis, 'mozIndexedDB') || Reflect.get(globalThis, 'msIndexedDB');
if (!indexedDB) return console.log('IndexedDB is not supported. No cache will be used.');
return new Promise<IDBDatabase>((resolve, reject) => {
const request = indexedDB.open('youtubei.js', 1);
request.onsuccess = function () {
resolve(this.result);
};
request.onerror = function (event) {
reject('indexedDB request error');
console.error(event);
};
request.onupgradeneeded = function () {
const store = this.result.createObjectStore('kv-store', {
keyPath: 'k'
});
store.transaction.oncomplete = function () {
resolve(this.db);
};
};
});
}
async get(key: string) {
await this.#createCache();
switch (getRuntime()) {
case 'deno':
{
const file = `${this.cache_dir}/${key}`;
const Deno: any = Reflect.get(globalThis, 'Deno');
try {
const stat = await Deno.stat(file);
if (stat.isFile) {
const data: Uint8Array = await Deno.readFile(file);
return data.buffer;
}
throw new Error('An unexpected file was found in place of the cache key');
} catch (e) {
if (e instanceof Deno.errors.NotFound)
return undefined;
throw e;
}
}
case 'node':
{
const fs = Reflect.get(module, 'require')('fs/promises');
const file = Reflect.get(module, 'require')('path').resolve(this.cache_dir, key);
try {
const stat = await fs.stat(file);
if (stat.isFile()) {
const data: Buffer = await fs.readFile(file);
return data.buffer;
}
throw new Error('An unexpected file was found in place of the cache key');
} catch (e: any) {
if (e?.code === 'ENOENT')
return undefined;
throw e;
}
}
case 'browser':
{
const db = await this.#getBrowserDB();
if (!db) return;
return new Promise<ArrayBuffer | undefined>((resolve, reject) => {
const request = db.transaction('kv-store', 'readonly').objectStore('kv-store').get(key);
request.onerror = reject;
request.onsuccess = function () {
const result: Uint8Array | undefined = this.result?.v;
resolve(result ? result.buffer : undefined);
};
});
}
}
}
async set(key: string, value: ArrayBuffer) {
await this.#createCache();
switch (getRuntime()) {
case 'deno':
{
const Deno: any = Reflect.get(globalThis, 'Deno');
const file = `${this.cache_dir}/${key}`;
await Deno.writeFile(file, new Uint8Array(value));
}
break;
case 'node':
{
const fs = Reflect.get(module, 'require')('fs/promises');
const file = Reflect.get(module, 'require')('path').resolve(this.cache_dir, key);
await fs.writeFile(file, new Uint8Array(value));
}
break;
case 'browser':
{
const db = await this.#getBrowserDB();
if (!db) return;
return new Promise<void>((resolve, reject) => {
const request = db.transaction('kv-store', 'readwrite').objectStore('kv-store').put({ k: key, v: value });
request.onerror = reject;
request.onsuccess = () => resolve();
});
}
break;
}
}
async remove(key: string) {
await this.#createCache();
switch (getRuntime()) {
case 'deno':
{
const file = `${this.cache_dir}/${key}`;
const Deno: any = Reflect.get(globalThis, 'Deno');
try {
await Deno.remove(file);
} catch (e) {
if (e instanceof Deno.errors.NotFound) return undefined;
throw e;
}
}
break;
case 'node':
{
const fs = Reflect.get(module, 'require')('fs/promises');
const file = Reflect.get(module, 'require')('path').resolve(this.cache_dir, key);
try {
await fs.unlink(file);
} catch (e: any) {
if (e?.code === 'ENOENT') return;
throw e;
}
}
break;
case 'browser':
{
const db = await this.#getBrowserDB();
if (!db) return;
return new Promise<void>((resolve, reject) => {
const request = db.transaction('kv-store', 'readwrite').objectStore('kv-store').delete(key);
request.onerror = reject;
request.onsuccess = () => resolve();
});
}
}
}
}