From 95e047974507d503ea4f2f30a2c3c58dbdb6c455 Mon Sep 17 00:00:00 2001 From: LuanRT Date: Wed, 28 Sep 2022 03:08:51 -0300 Subject: [PATCH] docs: add parser ytnode instructions & other minor changes (#206) * docs: add instructions on implementing ytnodes * docs(parser): fix grammar & other minor improvements * docs: update guidelines * chore: update parser warning messages --- CONTRIBUTING.md | 2 +- docs/updating-the-parser.md | 54 +++++++++++++++++++++++++++++++++++++ src/parser/README.md | 50 +++++++++++++++++++++++++--------- src/parser/index.ts | 7 ++--- 4 files changed, 96 insertions(+), 17 deletions(-) create mode 100644 docs/updating-the-parser.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e8eaa1f3..98461075 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,7 +22,7 @@ If you find a problem, search if an issue already exists. If a related issue doe #### Solve an issue -Scan through the existing issues to find one that interests you. You can narrow down the search using labels as filters. If you find an issue to work on, you are welcome to open a PR with a fix. +Scan through the existing issues to find one that interests you. You can narrow down the search using labels as filters. If you find an issue to work on, you are welcome to open a PR with a fix. Documentation updates and grammar fixes are also appreciated! ## Make changes diff --git a/docs/updating-the-parser.md b/docs/updating-the-parser.md new file mode 100644 index 00000000..ec80e8d1 --- /dev/null +++ b/docs/updating-the-parser.md @@ -0,0 +1,54 @@ +# Updating the parser + +YouTube is constantly changing, so it is not rare to see YouTube crawlers/scrapers breaking every now and then. + +Our parser, on the other hand, was written so that it behaves similarly to an official client, parsing and mapping renderers (a.k.a YTNodes) dynamically without hard-coding their path in the response. This way, whenever a new renderer pops up (e.g; YouTube adds a new feature / minor UI changes) the library will print a warning like this in the console: +``` +InnertubeError: SomeRenderer not found! +This is a bug, want to help us fix it? Follow the instructions at https://github.com/LuanRT/YouTube.js/blob/main/docs/updating-the-parser.md or report it at https://github.com/LuanRT/YouTube.js/issues! + at Parser.printError (...) + at Parser.parseItem (...) + at Parser.parseArray (...) { + info: { + // renderer data, can be used as a reference to implement the renderer parser + }, + date: 2022-05-22T22:16:06.831Z, + version: '2.2.3' +} +``` + +This warning, however, **does not** throw an error. The parser itself will continue working normally, even if a parsing error occurs in an existing renderer parser. + +## Adding a new renderer parser + +Thanks to the modularity of the parser, a renderer can be implemented by simply adding a new file anywhere in the [classes directory](../src/parser/classes)! + +For example, say we found a new renderer named `verticalListRenderer`, to let the parser know it exists we would have to create a file with the following structure: + +> `../classes/VerticalList.ts` +```ts +import Parser from '..'; +import { YTNode } from '../helpers'; + +class VerticalList extends YTNode { + static type = 'VerticalList'; + + header; + contents; + + constructor(data: any) { + // parse the data here, ex; + this.header = Parser.parseItem(data.header); + this.contents = Parser.parseArray(data.contents); + } +} + +export default VerticalList; +``` + +Then update the parser map: +```bash +npm run build:parser-map +``` + +And that's it! \ No newline at end of file diff --git a/src/parser/README.md b/src/parser/README.md index 209e8c75..c1586ca7 100644 --- a/src/parser/README.md +++ b/src/parser/README.md @@ -1,6 +1,25 @@ # Parser -Sanitizes and standardizes InnerTube responses while maintaining the integrity of the data. Also [drastically improves](https://github.com/LuanRT/YouTube.js/blob/main/lib/parser/youtube/Library.js#L44) how API calls are made and handled. +Sanitizes and standardizes InnerTube responses while maintaining the integrity of the data. Also [drastically improves](https://github.com/LuanRT/YouTube.js/blob/main/src/parser/youtube/Library.ts#L69) how API calls are made and handled. + +
    +
  1. + API +
    2. +
  2. + Usage + +
    3. +
  3. Adding new nodes
    4. +
  4. How it works
    5. +
+ +___ ## API @@ -43,6 +62,8 @@ Unlike `parse`, this can be used to parse the entire response object. | --- | --- | --- | | data | `object` | Raw InnerTube response | +## Usage + ## ObservedArray You may use `ObservedArray` as a normal array, but it provides additional methods for typesafe access and casting. @@ -58,13 +79,13 @@ const firstVideo = feed.firstOfType(GridVideo); // We may cast the whole array to a GridVideo[] and throw if we have any non-GridVideo elements: const allVideos = feed.as(GridVideo); -// There's some extra methods for ObservedArray +// There are some extra methods for ObservedArray // which we use internally but not documented here (yet). // see the source code for more details. ``` ## SuperParsedResponse -Represents a parsed response in an unknown state. Either a `YTNode` or a `ObservedArray` or `null`. +Represents a parsed response in an unknown state. Either a `YTNode` or an `ObservedArray` or `null`. You will need to assert the type and unwrap the response to get the actual value. @@ -116,14 +137,14 @@ if (node.is(TwoColumnSearchResults, VideoList)) { ``` ### Accessing properties without casting -Sometimes multiple nodes have the same properties and we don't want to check the type of the node before accessing the property, for example the property "contents" is used by many node types, and we may add more in the future, as such we want to only assert the property instead of casting to a specific type. +Sometimes multiple nodes have the same properties and we don't want to check the type of the node before accessing the property, for example, the property "contents" is used by many node types, and we may add more in the future, as such we want to only assert the property instead of casting to a specific type. ```ts -// Accesing a property on a node which you aren't sure if it exists. +// Accessing a property on a node which you aren't sure if it exists. const prop = node.key("contents"); // This returns the value wrapped into a `Maybe` type // which you can use to find the type of the value -// note however, this throws an error if the key doesn't exist +// note, however, this throws an error if the key doesn't exist // we may want to check for the key before accessing it. if (node.hasKey("contents")) { const prop = node.key("contents"); @@ -146,7 +167,7 @@ if (prop.isInstanceof(Text)) { }); } -// There's some special methods for using with the parser — +// There are some special methods for using with the parser — // such as getting the value as a YTNode. const prop = node.key("contents"); if (prop.isNode()) { @@ -171,7 +192,7 @@ const prop = node.key("contents"); if (prop.isObserved()) { const array = prop.observed(); - // Now we may use the all the ObservedArray methods as normal, + // Now we may use all the ObservedArray methods as normal, // like finding nodes of a certain type for example. const results = array.filterType(GridVideo); } @@ -187,7 +208,7 @@ if (prop.isParsed()) { const videos = results.filterType(Video); } -// Sometimes we just want to debug something and not interested in finding the type. +// Sometimes we just want to debug something and are not interested in finding the type. // This will, however, warn you when being used. const prop = node.key("contents"); const value = prop.any(); @@ -200,7 +221,7 @@ if (prop.isArray()) { // This will return Maybe[] } -// Or if you want zero typesafety you can use the `array` method. +// Or if you want zero type safety you can use the `array` method. const prop = node.key("contents"); if (prop.isArray()) { const array = prop.array(); @@ -221,13 +242,16 @@ const videos = response.contents_memo.getType(Video); `Memo` extends `Map` and can be used as a normal `Map` too if you want. +## Adding new nodes +Instructions can be found [here](https://github.com/LuanRT/YouTube.js/blob/main/docs/updating-the-parser.md). + ## How it works -If you decompile a YouTube client and analize it for a while you will notice that it has classes named `protos/youtube/api/innertube/MusicItemRenderer`, `protos/youtube/api/innertube/SectionListRenderer`, etc. +If you decompile a YouTube client and analyze it for a while you will notice that it has classes named `protos/youtube/api/innertube/MusicItemRenderer`, `protos/youtube/api/innertube/SectionListRenderer`, etc. -These classes are used to parse objects from the response, map them into models and generate the UI. The website works in a similar way, the difference is that it uses plain JSON (likely converted from protobuf server-side, hence the weird structure of the response). +These classes are used to parse objects from the response, map them into models and generate the UI. The website works similarly, the difference is that it uses plain JSON (likely converted from protobuf server-side, hence the weird structure of the response). -Here we're taking a similar approach, the parser goes through all the renderers and parses their inner element(s). The final result is a nicely structured JSON, and on top of that it also parses navigation endpoints which allows us to make an API call with all required parameters in one line and even emulate client actions (eg; clicking a button). +Here we're taking a similar approach, the parser goes through all the renderers and parses their inner element(s). The final result is a nicely structured JSON, and on top of that, it also parses navigation endpoints which allow us to make an API call with all required parameters in one line and even emulate client actions (eg; clicking a button). Here is your average, arguably ugly InnerTube response:
diff --git a/src/parser/index.ts b/src/parser/index.ts index 0c5d067a..7f707ed3 100644 --- a/src/parser/index.ts +++ b/src/parser/index.ts @@ -347,7 +347,7 @@ export default class Parser { return result as T; } catch (err) { - this.formatError({ classname, classdata: data[keys[0]], err }); + this.printError({ classname, classdata: data[keys[0]], err }); return null; } } @@ -436,12 +436,13 @@ export default class Parser { } } - static formatError({ classname, classdata, err }: { classname: string, classdata: any, err: any }) { + static printError({ classname, classdata, err }: { classname: string, classdata: any, err: any }) { if (err.code == 'MODULE_NOT_FOUND') { return console.warn( new InnertubeError( `${classname} not found!\n` + - `This is a bug, please report it at ${package_json.bugs.url}`, classdata) + `This is a bug, want to help us fix it? Follow the instructions at ${package_json.homepage.split('#')[0]}/blob/main/docs/updating-the-parser.md or report it at ${package_json.bugs.url}!`, classdata + ) ); }