refactor(cache): Get rid of custom LZW compression (#1168)

It's just buggy and not very efficient, and I can't realistically improve it. Compression algorithms are complicated and would require quite a bit of effort to get right...

fflate is tiny, works anywhere, and is more efficient.

Also, here is an interesting article about compression from fflate's author :  https://gist.github.com/101arrowz/253f31eb5abc3d9275ab943003ffecad

package-lock.json

@@ -13,6 +13,7 @@
       "license": "MIT",
       "dependencies": {
         "@bufbuild/protobuf": "^2.0.0",
+        "fflate": "^0.8.2",
         "meriyah": "^6.1.4"
       },
       "devDependencies": {
@@ -2404,6 +2405,12 @@
         "reusify": "^1.0.4"
       }
     },
+    "node_modules/fflate": {
+      "version": "0.8.2",
+      "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz",
+      "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==",
+      "license": "MIT"
+    },
     "node_modules/file-entry-cache": {
       "version": "8.0.0",
       "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz",
@@ -4141,6 +4148,7 @@
       "integrity": "sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "esbuild": "^0.27.0",
         "fdir": "^6.5.0",

package.json

@@ -103,6 +103,7 @@
   "license": "MIT",
   "dependencies": {
     "@bufbuild/protobuf": "^2.0.0",
+    "fflate": "^0.8.2",
     "meriyah": "^6.1.4"
   },
   "devDependencies": {

src/core/Session.ts

@@ -528,7 +528,7 @@ export default class Session extends EventEmitter {
 
     const buffer = BinarySerializer.serialize({
       ...session_data,
-      library_version: parseInt(packageInfo.version)
+      library_version: parseInt(packageInfo.version.split('.', 1)[0])
     });
 
     await cache.set('innertube_session_data', buffer);

src/utils/BinarySerializer.ts

@@ -1,30 +1,31 @@
-import { compress, decompress } from './LZW.js';
+import { gunzipSync, gzipSync } from 'fflate';
 
 export const MAGIC_HEADER = 0x594254; // 'YTB' in hex...
-export const VERSION = 1;
+export const VERSION = 2;
 
 export function serialize(data: any): ArrayBuffer {
-  const json_str = JSON.stringify(data);
-  const compressed = compress(json_str);
-  const compressed_bytes = new TextEncoder().encode(compressed);
+  const json = JSON.stringify(data);
+  const jsonBytes = new TextEncoder().encode(json);
+  const compressed = gzipSync(jsonBytes);
 
-  const buffer = new ArrayBuffer(12 + compressed_bytes.byteLength);
+  const buffer = new ArrayBuffer(12 + compressed.byteLength);
   const view = new DataView(buffer);
 
   view.setUint32(0, MAGIC_HEADER, true);
   view.setUint32(4, VERSION, true);
-  view.setUint32(8, compressed_bytes.byteLength, true);
+  view.setUint32(8, compressed.byteLength, true);
 
-  new Uint8Array(buffer).set(compressed_bytes, 12);
+  new Uint8Array(buffer).set(compressed, 12);
 
   return buffer;
 }
 
 export function deserialize<T>(buffer: Uint8Array): T {
-  if (buffer.byteLength < 12)
+  if (buffer.byteLength < 12) {
     throw new Error('Invalid binary format: buffer too short');
+  }
 
-  const view = new DataView(buffer.buffer, buffer.byteOffset);
+  const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
 
   const magic = view.getUint32(0, true);
   if (magic !== MAGIC_HEADER) {
@@ -36,11 +37,14 @@ export function deserialize<T>(buffer: Uint8Array): T {
     throw new Error(`Unsupported binary format version: ${version}`);
   }
 
-  const data_length = view.getUint32(8, true);
-  const compressed_data = buffer.slice(12, 12 + data_length);
+  const length = view.getUint32(8, true);
+  if (12 + length > buffer.byteLength) {
+    throw new Error('Invalid binary format: data length out of bounds');
+  }
 
-  const compressed = new TextDecoder().decode(compressed_data);
-  const json_str = decompress(compressed);
+  const compressed = buffer.subarray(12, 12 + length);
+  const decompressed = gunzipSync(compressed);
+  const json = new TextDecoder().decode(decompressed);
 
-  return JSON.parse(json_str);
+  return JSON.parse(json) as T;
 }

src/utils/LZW.ts

@@ -1,64 +0,0 @@
-/**
- * Compresses a string using the LZW compression algorithm.
- * @param input - The data to compress.
- */
-export function compress(input: string): string {
-  const output: number[] = [];
-  const dictionary: Record<string, number> = {};
-
-  for (let i = 0; i < 256; i++) {
-    dictionary[String.fromCharCode(i)] = i;
-  }
-
-  let current_string = '';
-  let dictionary_size = 256;
-
-  for (let i = 0; i < input.length; i++) {
-    const current_char = input[i];
-    const combined_string = current_string + current_char;
-
-    if (dictionary.hasOwnProperty(combined_string)) {
-      current_string = combined_string;
-    } else {
-      output.push(dictionary[current_string]);
-      dictionary[combined_string] = dictionary_size++;
-      current_string = current_char;
-    }
-  }
-
-  if (current_string !== '') {
-    output.push(dictionary[current_string]);
-  }
-
-  return output.map((code) => String.fromCharCode(code)).join('');
-}
-
-/**
- * Decompresses data that was compressed using the LZW compression algorithm.
- * @param input - The data to be decompressed.
- */
-export function decompress(input: string): string {
-  const dictionary: Record<number, string> = {};
-  const input_data = input.split('');
-  const output: string[] = [ input_data.shift() as string ];
-  const input_length = input_data.length >>> 0; // Convert to unsigned 32-bit integer
-
-  let dictionary_code = 256;
-  let current_char = output[0];
-  let current_string = current_char;
-
-  for (let i = 0; i < input_length; ++i) {
-    const current_code = input_data[i].charCodeAt(0);
-    const entry =
-      current_code < 256 ? input_data[i] : (dictionary[current_code] ?
-        dictionary[current_code] : (current_string + current_char));
-
-    output.push(entry);
-
-    current_char = entry.charAt(0);
-    dictionary[dictionary_code++] = current_string + current_char;
-    current_string = entry;
-  }
-
-  return output.join('');
-}

src/utils/index.ts

@@ -13,7 +13,6 @@ export { Platform } from './Utils.js';
 export * as Utils from './Utils.js';
 
 export * as Log from './Log.js';
-export * as LZW from './LZW.js';
 export * as BinarySerializer from './BinarySerializer.js';
 
 export * as ProtoUtils from './ProtoUtils.js';