feat(declib): initial mastodon bot experiment r/7499

No default.nix yet, just for development.

Change-Id: Ib8bd0057d697fecd083d5961e635c770b7638e08
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10803
Reviewed-by: Profpatsch <mail@profpatsch.de>
Tested-by: BuildkiteCI

12 files changed, 383 insertions, 3 deletions

diff --git a/users/Profpatsch/.envrc b/users/Profpatsch/.envrc
index 051d09d292..832afd26bd 100644
--- a/users/Profpatsch/.envrc
+++ b/users/Profpatsch/.envrc
@@ -1 +1,5 @@
+if pass apps/declib/mastodon_access_token; then
+    export DECLIB_MASTODON_ACCESS_TOKEN=$(pass apps/declib/mastodon_access_token)
+fi
+
 eval "$(lorri direnv)"
diff --git a/users/Profpatsch/.vscode/launch.json b/users/Profpatsch/.vscode/launch.json
new file mode 100644
index 0000000000..baa087d437
--- /dev/null
+++ b/users/Profpatsch/.vscode/launch.json
@@ -0,0 +1,18 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "run declib",
+            "type": "node",
+            "cwd": "${workspaceFolder}/declib",
+            "request": "launch",
+            "runtimeExecutable": "ninja",
+            "runtimeArgs": [
+                "run",
+            ],
+        }
+    ]
+}
diff --git a/users/Profpatsch/.vscode/settings.json b/users/Profpatsch/.vscode/settings.json
index 81546964eb..7984076c16 100644
--- a/users/Profpatsch/.vscode/settings.json
+++ b/users/Profpatsch/.vscode/settings.json
@@ -8,7 +8,18 @@
         }
     ],
     "sqltools.useNodeRuntime": true,
-    "[haskell]": {
-        "editor.formatOnSave": true
-    }
+    "editor.formatOnSave": true,
+    "[typescript]": {
+        "editor.defaultFormatter": "esbenp.prettier-vscode"
+    },
+    "[javascript]": {
+        "editor.defaultFormatter": "esbenp.prettier-vscode"
+    },
+    "[json]": {
+        "editor.defaultFormatter": "esbenp.prettier-vscode"
+    },
+    "purescript.codegenTargets": [
+        "corefn"
+    ],
+    "purescript.foreignExt": "nix"
 }
diff --git a/users/Profpatsch/declib/.eslintrc.json b/users/Profpatsch/declib/.eslintrc.json
new file mode 100644
index 0000000000..9cffc711db
--- /dev/null
+++ b/users/Profpatsch/declib/.eslintrc.json
@@ -0,0 +1,14 @@
+{
+  "extends": ["eslint:recommended", "plugin:@typescript-eslint/strict-type-checked"],
+  "parser": "@typescript-eslint/parser",
+  "plugins": ["@typescript-eslint"],
+  "parserOptions": {
+    "project": true
+  },
+  "root": true,
+  "rules": {
+    "no-unused-vars": "warn",
+    "prefer-const": "warn",
+    "@typescript-eslint/no-unused-vars": "warn"
+  }
+}
diff --git a/users/Profpatsch/declib/.gitignore b/users/Profpatsch/declib/.gitignore
new file mode 100644
index 0000000000..8b56bf4ede
--- /dev/null
+++ b/users/Profpatsch/declib/.gitignore
@@ -0,0 +1,6 @@
+/node_modules/
+/.ninja/
+/output/
+
+# ignore for now
+/package.lock.json
diff --git a/users/Profpatsch/declib/.prettierrc b/users/Profpatsch/declib/.prettierrc
new file mode 100644
index 0000000000..7258fb81e0
--- /dev/null
+++ b/users/Profpatsch/declib/.prettierrc
@@ -0,0 +1,8 @@
+{
+  "trailingComma": "all",
+  "tabWidth": 2,
+  "semi": true,
+  "singleQuote": true,
+  "printWidth": 100,
+  "arrowParens": "avoid"
+}
diff --git a/users/Profpatsch/declib/README.md b/users/Profpatsch/declib/README.md
new file mode 100644
index 0000000000..11a8bf21a5
--- /dev/null
+++ b/users/Profpatsch/declib/README.md
@@ -0,0 +1,4 @@
+# Decentralized Library
+
+https://en.wikipedia.org/wiki/Distributed_library
+https://faculty.ist.psu.edu/jjansen/academic/pubs/ride98/ride98.html
diff --git a/users/Profpatsch/declib/build.ninja b/users/Profpatsch/declib/build.ninja
new file mode 100644
index 0000000000..f8844fc9be
--- /dev/null
+++ b/users/Profpatsch/declib/build.ninja
@@ -0,0 +1,16 @@
+
+builddir = .ninja
+
+outdir = ./output
+jsdir = $outdir/js
+
+rule tsc
+  command = node_modules/.bin/tsc
+
+build $outdir/index.js: tsc | index.ts tsconfig.json
+
+rule run
+  command = node $in
+
+build run: run $outdir/index.js
+  pool = console
diff --git a/users/Profpatsch/declib/index.ts b/users/Profpatsch/declib/index.ts
new file mode 100644
index 0000000000..c6a26f0922
--- /dev/null
+++ b/users/Profpatsch/declib/index.ts
@@ -0,0 +1,245 @@
+import generator, { MegalodonInterface } from 'megalodon';
+import { Account } from 'megalodon/lib/src/entities/account';
+import * as masto from 'megalodon/lib/src/entities/notification';
+import { Status } from 'megalodon/lib/src/entities/status';
+import * as rxjs from 'rxjs';
+import { Observable } from 'rxjs';
+import { NodeEventHandler } from 'rxjs/internal/observable/fromEvent';
+import * as sqlite from 'sqlite';
+import sqlite3 from 'sqlite3';
+import * as parse5 from 'parse5';
+import { mergeMap } from 'rxjs/operators';
+
+type Events =
+  | { type: 'connect'; event: [] }
+  | { type: 'update'; event: Status }
+  | { type: 'notification'; event: Notification }
+  | { type: 'delete'; event: number }
+  | { type: 'error'; event: Error }
+  | { type: 'heartbeat'; event: [] }
+  | { type: 'close'; event: [] }
+  | { type: 'parser-error'; event: Error };
+
+type Notification = masto.Notification & {
+  type: 'favourite' | 'reblog' | 'status' | 'mention' | 'poll' | 'update';
+  status: NonNullable<masto.Notification['status']>;
+  account: NonNullable<masto.Notification['account']>;
+};
+
+class Main {
+  private client: MegalodonInterface;
+  private socket: Observable<Events>;
+  private state!: State;
+  private config: {
+    databaseFile?: string;
+    baseServer: string;
+  };
+
+  private constructor() {
+    this.config = {
+      databaseFile: process.env['DECLIB_DATABASE_FILE'],
+      baseServer: process.env['DECLIB_MASTODON_SERVER'] ?? 'mastodon.xyz',
+    };
+    const ACCESS_TOKEN = process.env['DECLIB_MASTODON_ACCESS_TOKEN'];
+
+    if (!ACCESS_TOKEN) {
+      console.error('Please set DECLIB_MASTODON_ACCESS_TOKEN');
+      process.exit(1);
+    }
+    this.client = generator('mastodon', `https://${this.config.baseServer}`, ACCESS_TOKEN);
+    const websocket = this.client.publicSocket();
+    function mk<Name extends string, Type>(name: Name): Observable<{ type: Name; event: Type }> {
+      const wrap =
+        (h: NodeEventHandler) =>
+        (event: Type): void => {
+          h({ type: name, event });
+        };
+      return rxjs.fromEventPattern<{ type: Name; event: Type }>(
+        hdl => websocket.on(name, wrap(hdl)),
+        hdl => websocket.removeListener(name, wrap(hdl)),
+      );
+    }
+    this.socket = rxjs.merge(
+      mk<'connect', []>('connect'),
+      mk<'update', Status>('update'),
+      mk<'notification', Notification>('notification'),
+      mk<'delete', number>('delete'),
+      mk<'error', Error>('error'),
+      mk<'heartbeat', []>('heartbeat'),
+      mk<'close', []>('close'),
+      mk<'parser-error', Error>('parser-error'),
+    );
+  }
+
+  static async init(): Promise<Main> {
+    const self = new Main();
+    self.state = await State.init(self.config);
+    return self;
+  }
+
+  public main() {
+    // const res = await this.getAcc({ username: 'grindhold', server: 'chaos.social' });
+    // const res = await this.getAcc({ username: 'Profpatsch', server: 'mastodon.xyz' });
+    // const res = await this.getStatus('111862170899069698');
+    this.socket
+      .pipe(
+        mergeMap(async event => {
+          switch (event.type) {
+            case 'update': {
+              await this.state.addStatus(event.event);
+              console.log(`${event.event.account.acct}: ${event.event.content}`);
+              console.log(await this.state.databaseInternal.all(`SELECT * from status`));
+              break;
+            }
+            case 'notification': {
+              console.log(`NOTIFICATION (${event.event.type}):`);
+              console.log(event.event);
+              console.log(event.event.status.content);
+              const content = parseContent(event.event.status.content);
+              if (content) {
+                switch (content.command) {
+                  case 'addbook': {
+                    if (content.content[0]) {
+                      const book = {
+                        $owner: event.event.account.acct,
+                        $bookid: content.content[0],
+                      };
+                      console.log('adding book', book);
+                      await this.state.addBook(book);
+                      await this.client.postStatus(
+                        `@${event.event.account.acct} I have inserted book "${book.$bookid}" for you.`,
+                        {
+                          in_reply_to_id: event.event.status.id,
+                          visibility: 'direct',
+                        },
+                      );
+                    }
+                  }
+                }
+              }
+              break;
+            }
+            default: {
+              console.log(event);
+            }
+          }
+        }),
+      )
+      .subscribe();
+  }
+
+  private async getStatus(id: string): Promise<Status | null> {
+    return (await this.client.getStatus(id)).data;
+  }
+
+  private async getAcc(user: { username: string; server: string }): Promise<Account | null> {
+    const fullAccount = `${user.username}@${user.server}`;
+    const res = await this.client.searchAccount(fullAccount, {
+      limit: 10,
+    });
+    const accs = res.data.filter(acc =>
+      this.config.baseServer === user.server
+        ? (acc.acct = user.username)
+        : acc.acct === fullAccount,
+    );
+    return accs[0] ?? null;
+  }
+}
+
+type Interaction = {
+  originalStatus: { id: string };
+  lastStatus: { id: string };
+};
+
+class State {
+  db!: sqlite.Database;
+  private constructor() {}
+
+  static async init(config: { databaseFile?: string }): Promise<State> {
+    const s = new State();
+    s.db = await sqlite.open({
+      filename: config.databaseFile ?? ':memory:',
+      driver: sqlite3.Database,
+    });
+    await s.db.run('CREATE TABLE books (owner text, bookid text)');
+    await s.db.run('CREATE TABLE status (id text primary key, content json)');
+    return s;
+  }
+
+  async addBook(opts: { $owner: string; $bookid: string }) {
+    return await this.db.run('INSERT INTO books (owner, bookid) VALUES ($owner, $bookid)', opts);
+  }
+
+  async addStatus($status: Status) {
+    return await this.db.run(
+      `
+      INSERT INTO status (id, content) VALUES ($id, $status)
+      ON CONFLICT (id) DO UPDATE SET id = $id, content = $status
+      `,
+      {
+        $id: $status.id,
+        $status: JSON.stringify($status),
+      },
+    );
+  }
+
+  get databaseInternal() {
+    return this.db;
+  }
+}
+
+/** Parse the message; take the plain text, first line is the command any any successive lines are content */
+function parseContent(html: string): { command: string; content: string[] } | null {
+  const plain = contentToPlainText(html).split('\n');
+  if (plain[0]) {
+    return { command: plain[0].replace(' ', '').trim(), content: plain.slice(1) };
+  } else {
+    return null;
+  }
+}
+
+/** Convert the Html content to a plain text (best effort), keeping line breaks */
+function contentToPlainText(html: string): string {
+  const queue: parse5.DefaultTreeAdapterMap['childNode'][] = [];
+  queue.push(...parse5.parseFragment(html).childNodes);
+  let res = '';
+  let endOfP = false;
+  for (const el of queue) {
+    switch (el.nodeName) {
+      case '#text': {
+        res += (el as parse5.DefaultTreeAdapterMap['textNode']).value;
+        break;
+      }
+      case 'br': {
+        res += '\n';
+        break;
+      }
+      case 'p': {
+        if (endOfP) {
+          res += '\n';
+          endOfP = false;
+        }
+        queue.push(...el.childNodes);
+        endOfP = true;
+        break;
+      }
+      case 'span': {
+        break;
+      }
+      default: {
+        console.warn('unknown element in message: ', el);
+        break;
+      }
+    }
+  }
+  return res.trim();
+}
+
+Main.init().then(
+  m => {
+    m.main();
+  },
+  rej => {
+    throw rej;
+  },
+);
diff --git a/users/Profpatsch/declib/package.json b/users/Profpatsch/declib/package.json
new file mode 100644
index 0000000000..93176e8581
--- /dev/null
+++ b/users/Profpatsch/declib/package.json
@@ -0,0 +1,25 @@
+{
+  "name": "declib",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.ts",
+  "type": "commonjs",
+  "scripts": {
+    "run": "ninja run"
+  },
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "megalodon": "^9.2.2",
+    "parse5": "^7.1.2",
+    "rxjs": "^7.8.1",
+    "sqlite": "^5.1.1",
+    "sqlite3": "^5.1.7"
+  },
+  "devDependencies": {
+    "@typescript-eslint/eslint-plugin": "^6.21.0",
+    "@typescript-eslint/parser": "^6.21.0",
+    "eslint": "^8.56.0",
+    "typescript": "^5.3.3"
+  }
+}
diff --git a/users/Profpatsch/declib/tsconfig.json b/users/Profpatsch/declib/tsconfig.json
new file mode 100644
index 0000000000..b7f2f4c18b
--- /dev/null
+++ b/users/Profpatsch/declib/tsconfig.json
@@ -0,0 +1,25 @@
+{
+  "compilerOptions": {
+    "strict": true,
+    "module": "NodeNext",
+    "sourceMap": true,
+    "outDir": "output",
+    "target": "ES6",
+    "lib": [],
+    "typeRoots": ["node_modules/@types", "shims/@types"],
+    "moduleResolution": "NodeNext",
+
+    // importHelpers & downlevelIteration will reduce the generated javascript for new language features.
+    // `importHelpers` requires the `tslib` dependency.
+    // "downlevelIteration": true,
+    // "importHelpers": true
+    "noFallthroughCasesInSwitch": true,
+    "noImplicitOverride": true,
+    "noImplicitReturns": true,
+    "noPropertyAccessFromIndexSignature": true,
+    "noUncheckedIndexedAccess": true,
+
+  },
+
+  "files": ["index.ts"]
+}
diff --git a/users/Profpatsch/shell.nix b/users/Profpatsch/shell.nix
index 367ab01e1d..1e36dbeda6 100644
--- a/users/Profpatsch/shell.nix
+++ b/users/Profpatsch/shell.nix
@@ -61,6 +61,8 @@ pkgs.mkShell {
     pkgs.pkg-config
     pkgs.fuse
     pkgs.postgresql
+    pkgs.nodejs
+    pkgs.ninja
   ];
 
   WHATCD_RESOLVER_TOOLS = pkgs.linkFarm "whatcd-resolver-tools" [
@@ -70,6 +72,8 @@ pkgs.mkShell {
     }
   ];
 
+  # DECLIB_MASTODON_ACCESS_TOKEN read from `pass` in .envrc.
+
   RUSTC_WRAPPER =
     let
       wrapperArgFile = libs: pkgs.writeText "rustc-wrapper-args"