From 80c683c9ecf0129b5468028856a1443d87c6c64a Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Sun, 11 Feb 2024 17:31:42 +0100 Subject: feat(declib): initial mastodon bot experiment No default.nix yet, just for development. Change-Id: Ib8bd0057d697fecd083d5961e635c770b7638e08 Reviewed-on: https://cl.tvl.fyi/c/depot/+/10803 Reviewed-by: Profpatsch Tested-by: BuildkiteCI --- users/Profpatsch/.envrc | 4 + users/Profpatsch/.vscode/launch.json | 18 +++ users/Profpatsch/.vscode/settings.json | 17 ++- users/Profpatsch/declib/.eslintrc.json | 14 ++ users/Profpatsch/declib/.gitignore | 6 + users/Profpatsch/declib/.prettierrc | 8 ++ users/Profpatsch/declib/README.md | 4 + users/Profpatsch/declib/build.ninja | 16 +++ users/Profpatsch/declib/index.ts | 245 +++++++++++++++++++++++++++++++++ users/Profpatsch/declib/package.json | 25 ++++ users/Profpatsch/declib/tsconfig.json | 25 ++++ users/Profpatsch/shell.nix | 4 + 12 files changed, 383 insertions(+), 3 deletions(-) create mode 100644 users/Profpatsch/.vscode/launch.json create mode 100644 users/Profpatsch/declib/.eslintrc.json create mode 100644 users/Profpatsch/declib/.gitignore create mode 100644 users/Profpatsch/declib/.prettierrc create mode 100644 users/Profpatsch/declib/README.md create mode 100644 users/Profpatsch/declib/build.ninja create mode 100644 users/Profpatsch/declib/index.ts create mode 100644 users/Profpatsch/declib/package.json create mode 100644 users/Profpatsch/declib/tsconfig.json diff --git a/users/Profpatsch/.envrc b/users/Profpatsch/.envrc index 051d09d292a8..832afd26bd69 100644 --- a/users/Profpatsch/.envrc +++ b/users/Profpatsch/.envrc @@ -1 +1,5 @@ +if pass apps/declib/mastodon_access_token; then + export DECLIB_MASTODON_ACCESS_TOKEN=$(pass apps/declib/mastodon_access_token) +fi + eval "$(lorri direnv)" diff --git a/users/Profpatsch/.vscode/launch.json b/users/Profpatsch/.vscode/launch.json new file mode 100644 index 000000000000..baa087d437d1 --- /dev/null +++ b/users/Profpatsch/.vscode/launch.json @@ -0,0 +1,18 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "run declib", + "type": "node", + "cwd": "${workspaceFolder}/declib", + "request": "launch", + "runtimeExecutable": "ninja", + "runtimeArgs": [ + "run", + ], + } + ] +} diff --git a/users/Profpatsch/.vscode/settings.json b/users/Profpatsch/.vscode/settings.json index 81546964eb03..7984076c1647 100644 --- a/users/Profpatsch/.vscode/settings.json +++ b/users/Profpatsch/.vscode/settings.json @@ -8,7 +8,18 @@ } ], "sqltools.useNodeRuntime": true, - "[haskell]": { - "editor.formatOnSave": true - } + "editor.formatOnSave": true, + "[typescript]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[javascript]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[json]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "purescript.codegenTargets": [ + "corefn" + ], + "purescript.foreignExt": "nix" } diff --git a/users/Profpatsch/declib/.eslintrc.json b/users/Profpatsch/declib/.eslintrc.json new file mode 100644 index 000000000000..9cffc711dbd1 --- /dev/null +++ b/users/Profpatsch/declib/.eslintrc.json @@ -0,0 +1,14 @@ +{ + "extends": ["eslint:recommended", "plugin:@typescript-eslint/strict-type-checked"], + "parser": "@typescript-eslint/parser", + "plugins": ["@typescript-eslint"], + "parserOptions": { + "project": true + }, + "root": true, + "rules": { + "no-unused-vars": "warn", + "prefer-const": "warn", + "@typescript-eslint/no-unused-vars": "warn" + } +} diff --git a/users/Profpatsch/declib/.gitignore b/users/Profpatsch/declib/.gitignore new file mode 100644 index 000000000000..8b56bf4ede77 --- /dev/null +++ b/users/Profpatsch/declib/.gitignore @@ -0,0 +1,6 @@ +/node_modules/ +/.ninja/ +/output/ + +# ignore for now +/package.lock.json diff --git a/users/Profpatsch/declib/.prettierrc b/users/Profpatsch/declib/.prettierrc new file mode 100644 index 000000000000..7258fb81e063 --- /dev/null +++ b/users/Profpatsch/declib/.prettierrc @@ -0,0 +1,8 @@ +{ + "trailingComma": "all", + "tabWidth": 2, + "semi": true, + "singleQuote": true, + "printWidth": 100, + "arrowParens": "avoid" +} diff --git a/users/Profpatsch/declib/README.md b/users/Profpatsch/declib/README.md new file mode 100644 index 000000000000..11a8bf21a510 --- /dev/null +++ b/users/Profpatsch/declib/README.md @@ -0,0 +1,4 @@ +# Decentralized Library + +https://en.wikipedia.org/wiki/Distributed_library +https://faculty.ist.psu.edu/jjansen/academic/pubs/ride98/ride98.html diff --git a/users/Profpatsch/declib/build.ninja b/users/Profpatsch/declib/build.ninja new file mode 100644 index 000000000000..f8844fc9be16 --- /dev/null +++ b/users/Profpatsch/declib/build.ninja @@ -0,0 +1,16 @@ + +builddir = .ninja + +outdir = ./output +jsdir = $outdir/js + +rule tsc + command = node_modules/.bin/tsc + +build $outdir/index.js: tsc | index.ts tsconfig.json + +rule run + command = node $in + +build run: run $outdir/index.js + pool = console diff --git a/users/Profpatsch/declib/index.ts b/users/Profpatsch/declib/index.ts new file mode 100644 index 000000000000..c6a26f09226f --- /dev/null +++ b/users/Profpatsch/declib/index.ts @@ -0,0 +1,245 @@ +import generator, { MegalodonInterface } from 'megalodon'; +import { Account } from 'megalodon/lib/src/entities/account'; +import * as masto from 'megalodon/lib/src/entities/notification'; +import { Status } from 'megalodon/lib/src/entities/status'; +import * as rxjs from 'rxjs'; +import { Observable } from 'rxjs'; +import { NodeEventHandler } from 'rxjs/internal/observable/fromEvent'; +import * as sqlite from 'sqlite'; +import sqlite3 from 'sqlite3'; +import * as parse5 from 'parse5'; +import { mergeMap } from 'rxjs/operators'; + +type Events = + | { type: 'connect'; event: [] } + | { type: 'update'; event: Status } + | { type: 'notification'; event: Notification } + | { type: 'delete'; event: number } + | { type: 'error'; event: Error } + | { type: 'heartbeat'; event: [] } + | { type: 'close'; event: [] } + | { type: 'parser-error'; event: Error }; + +type Notification = masto.Notification & { + type: 'favourite' | 'reblog' | 'status' | 'mention' | 'poll' | 'update'; + status: NonNullable; + account: NonNullable; +}; + +class Main { + private client: MegalodonInterface; + private socket: Observable; + private state!: State; + private config: { + databaseFile?: string; + baseServer: string; + }; + + private constructor() { + this.config = { + databaseFile: process.env['DECLIB_DATABASE_FILE'], + baseServer: process.env['DECLIB_MASTODON_SERVER'] ?? 'mastodon.xyz', + }; + const ACCESS_TOKEN = process.env['DECLIB_MASTODON_ACCESS_TOKEN']; + + if (!ACCESS_TOKEN) { + console.error('Please set DECLIB_MASTODON_ACCESS_TOKEN'); + process.exit(1); + } + this.client = generator('mastodon', `https://${this.config.baseServer}`, ACCESS_TOKEN); + const websocket = this.client.publicSocket(); + function mk(name: Name): Observable<{ type: Name; event: Type }> { + const wrap = + (h: NodeEventHandler) => + (event: Type): void => { + h({ type: name, event }); + }; + return rxjs.fromEventPattern<{ type: Name; event: Type }>( + hdl => websocket.on(name, wrap(hdl)), + hdl => websocket.removeListener(name, wrap(hdl)), + ); + } + this.socket = rxjs.merge( + mk<'connect', []>('connect'), + mk<'update', Status>('update'), + mk<'notification', Notification>('notification'), + mk<'delete', number>('delete'), + mk<'error', Error>('error'), + mk<'heartbeat', []>('heartbeat'), + mk<'close', []>('close'), + mk<'parser-error', Error>('parser-error'), + ); + } + + static async init(): Promise
{ + const self = new Main(); + self.state = await State.init(self.config); + return self; + } + + public main() { + // const res = await this.getAcc({ username: 'grindhold', server: 'chaos.social' }); + // const res = await this.getAcc({ username: 'Profpatsch', server: 'mastodon.xyz' }); + // const res = await this.getStatus('111862170899069698'); + this.socket + .pipe( + mergeMap(async event => { + switch (event.type) { + case 'update': { + await this.state.addStatus(event.event); + console.log(`${event.event.account.acct}: ${event.event.content}`); + console.log(await this.state.databaseInternal.all(`SELECT * from status`)); + break; + } + case 'notification': { + console.log(`NOTIFICATION (${event.event.type}):`); + console.log(event.event); + console.log(event.event.status.content); + const content = parseContent(event.event.status.content); + if (content) { + switch (content.command) { + case 'addbook': { + if (content.content[0]) { + const book = { + $owner: event.event.account.acct, + $bookid: content.content[0], + }; + console.log('adding book', book); + await this.state.addBook(book); + await this.client.postStatus( + `@${event.event.account.acct} I have inserted book "${book.$bookid}" for you.`, + { + in_reply_to_id: event.event.status.id, + visibility: 'direct', + }, + ); + } + } + } + } + break; + } + default: { + console.log(event); + } + } + }), + ) + .subscribe(); + } + + private async getStatus(id: string): Promise { + return (await this.client.getStatus(id)).data; + } + + private async getAcc(user: { username: string; server: string }): Promise { + const fullAccount = `${user.username}@${user.server}`; + const res = await this.client.searchAccount(fullAccount, { + limit: 10, + }); + const accs = res.data.filter(acc => + this.config.baseServer === user.server + ? (acc.acct = user.username) + : acc.acct === fullAccount, + ); + return accs[0] ?? null; + } +} + +type Interaction = { + originalStatus: { id: string }; + lastStatus: { id: string }; +}; + +class State { + db!: sqlite.Database; + private constructor() {} + + static async init(config: { databaseFile?: string }): Promise { + const s = new State(); + s.db = await sqlite.open({ + filename: config.databaseFile ?? ':memory:', + driver: sqlite3.Database, + }); + await s.db.run('CREATE TABLE books (owner text, bookid text)'); + await s.db.run('CREATE TABLE status (id text primary key, content json)'); + return s; + } + + async addBook(opts: { $owner: string; $bookid: string }) { + return await this.db.run('INSERT INTO books (owner, bookid) VALUES ($owner, $bookid)', opts); + } + + async addStatus($status: Status) { + return await this.db.run( + ` + INSERT INTO status (id, content) VALUES ($id, $status) + ON CONFLICT (id) DO UPDATE SET id = $id, content = $status + `, + { + $id: $status.id, + $status: JSON.stringify($status), + }, + ); + } + + get databaseInternal() { + return this.db; + } +} + +/** Parse the message; take the plain text, first line is the command any any successive lines are content */ +function parseContent(html: string): { command: string; content: string[] } | null { + const plain = contentToPlainText(html).split('\n'); + if (plain[0]) { + return { command: plain[0].replace(' ', '').trim(), content: plain.slice(1) }; + } else { + return null; + } +} + +/** Convert the Html content to a plain text (best effort), keeping line breaks */ +function contentToPlainText(html: string): string { + const queue: parse5.DefaultTreeAdapterMap['childNode'][] = []; + queue.push(...parse5.parseFragment(html).childNodes); + let res = ''; + let endOfP = false; + for (const el of queue) { + switch (el.nodeName) { + case '#text': { + res += (el as parse5.DefaultTreeAdapterMap['textNode']).value; + break; + } + case 'br': { + res += '\n'; + break; + } + case 'p': { + if (endOfP) { + res += '\n'; + endOfP = false; + } + queue.push(...el.childNodes); + endOfP = true; + break; + } + case 'span': { + break; + } + default: { + console.warn('unknown element in message: ', el); + break; + } + } + } + return res.trim(); +} + +Main.init().then( + m => { + m.main(); + }, + rej => { + throw rej; + }, +); diff --git a/users/Profpatsch/declib/package.json b/users/Profpatsch/declib/package.json new file mode 100644 index 000000000000..93176e8581d4 --- /dev/null +++ b/users/Profpatsch/declib/package.json @@ -0,0 +1,25 @@ +{ + "name": "declib", + "version": "1.0.0", + "description": "", + "main": "index.ts", + "type": "commonjs", + "scripts": { + "run": "ninja run" + }, + "author": "", + "license": "MIT", + "dependencies": { + "megalodon": "^9.2.2", + "parse5": "^7.1.2", + "rxjs": "^7.8.1", + "sqlite": "^5.1.1", + "sqlite3": "^5.1.7" + }, + "devDependencies": { + "@typescript-eslint/eslint-plugin": "^6.21.0", + "@typescript-eslint/parser": "^6.21.0", + "eslint": "^8.56.0", + "typescript": "^5.3.3" + } +} diff --git a/users/Profpatsch/declib/tsconfig.json b/users/Profpatsch/declib/tsconfig.json new file mode 100644 index 000000000000..b7f2f4c18be8 --- /dev/null +++ b/users/Profpatsch/declib/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "strict": true, + "module": "NodeNext", + "sourceMap": true, + "outDir": "output", + "target": "ES6", + "lib": [], + "typeRoots": ["node_modules/@types", "shims/@types"], + "moduleResolution": "NodeNext", + + // importHelpers & downlevelIteration will reduce the generated javascript for new language features. + // `importHelpers` requires the `tslib` dependency. + // "downlevelIteration": true, + // "importHelpers": true + "noFallthroughCasesInSwitch": true, + "noImplicitOverride": true, + "noImplicitReturns": true, + "noPropertyAccessFromIndexSignature": true, + "noUncheckedIndexedAccess": true, + + }, + + "files": ["index.ts"] +} diff --git a/users/Profpatsch/shell.nix b/users/Profpatsch/shell.nix index 367ab01e1d80..1e36dbeda6d7 100644 --- a/users/Profpatsch/shell.nix +++ b/users/Profpatsch/shell.nix @@ -61,6 +61,8 @@ pkgs.mkShell { pkgs.pkg-config pkgs.fuse pkgs.postgresql + pkgs.nodejs + pkgs.ninja ]; WHATCD_RESOLVER_TOOLS = pkgs.linkFarm "whatcd-resolver-tools" [ @@ -70,6 +72,8 @@ pkgs.mkShell { } ]; + # DECLIB_MASTODON_ACCESS_TOKEN read from `pass` in .envrc. + RUSTC_WRAPPER = let wrapperArgFile = libs: pkgs.writeText "rustc-wrapper-args" -- cgit 1.4.1