From 5af5414fdf03752fcbfbd11bb4d6dbf4f192b291 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 23 May 2025 19:44:07 -0400 Subject: [PATCH 01/29] enable TypeScript incremental mode to speed up IDEs --- cypress/tsconfig.json | 3 ++- packages/backend/test-federation/tsconfig.json | 2 +- packages/backend/test-server/tsconfig.json | 1 + packages/backend/test/tsconfig.json | 1 + packages/backend/tsconfig.json | 1 + packages/frontend-embed/src/workers/tsconfig.json | 1 + packages/frontend-embed/tsconfig.json | 1 + packages/frontend-shared/tsconfig.json | 1 + packages/frontend/.storybook/tsconfig.json | 1 + packages/frontend/src/workers/tsconfig.json | 1 + packages/frontend/test/tsconfig.json | 1 + packages/frontend/tsconfig.json | 1 + packages/misskey-bubble-game/tsconfig.json | 1 + packages/misskey-js/generator/tsconfig.json | 1 + packages/misskey-js/tsconfig.json | 1 + packages/misskey-reversi/tsconfig.json | 1 + packages/sw/tsconfig.json | 1 + 17 files changed, 18 insertions(+), 2 deletions(-) diff --git a/cypress/tsconfig.json b/cypress/tsconfig.json index 6fe7f32cc4..4435a4fda8 100644 --- a/cypress/tsconfig.json +++ b/cypress/tsconfig.json @@ -2,7 +2,8 @@ "compilerOptions": { "lib": ["dom", "es5"], "target": "es5", - "types": ["cypress", "node"] + "types": ["cypress", "node"], + "incremental": true }, "include": ["./**/*.ts"] } diff --git a/packages/backend/test-federation/tsconfig.json b/packages/backend/test-federation/tsconfig.json index 3a1cb3b9f3..16b333f877 100644 --- a/packages/backend/test-federation/tsconfig.json +++ b/packages/backend/test-federation/tsconfig.json @@ -3,7 +3,7 @@ /* Visit https://aka.ms/tsconfig to read more about this file */ /* Projects */ - // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ diff --git a/packages/backend/test-server/tsconfig.json b/packages/backend/test-server/tsconfig.json index 10313699c2..cb394ecccd 100644 --- a/packages/backend/test-server/tsconfig.json +++ b/packages/backend/test-server/tsconfig.json @@ -23,6 +23,7 @@ "emitDecoratorMetadata": true, "resolveJsonModule": true, "isolatedModules": true, + "incremental": true, "rootDir": "../src", "baseUrl": "./", "paths": { diff --git a/packages/backend/test/tsconfig.json b/packages/backend/test/tsconfig.json index 2b562acda8..f3b6a5108d 100644 --- a/packages/backend/test/tsconfig.json +++ b/packages/backend/test/tsconfig.json @@ -23,6 +23,7 @@ "emitDecoratorMetadata": true, "resolveJsonModule": true, "isolatedModules": true, + "incremental": true, "baseUrl": "./", "paths": { "@/*": ["../src/*"] diff --git a/packages/backend/tsconfig.json b/packages/backend/tsconfig.json index 392da169ad..afed1f186c 100644 --- a/packages/backend/tsconfig.json +++ b/packages/backend/tsconfig.json @@ -23,6 +23,7 @@ "emitDecoratorMetadata": true, "resolveJsonModule": true, "isolatedModules": true, + "incremental": true, "rootDir": "./src", "baseUrl": "./", "paths": { diff --git a/packages/frontend-embed/src/workers/tsconfig.json b/packages/frontend-embed/src/workers/tsconfig.json index 8ee8930465..39ba45ddbb 100644 --- a/packages/frontend-embed/src/workers/tsconfig.json +++ b/packages/frontend-embed/src/workers/tsconfig.json @@ -1,5 +1,6 @@ { "compilerOptions": { "lib": ["esnext", "webworker"], + "incremental": true } } diff --git a/packages/frontend-embed/tsconfig.json b/packages/frontend-embed/tsconfig.json index e0ee08188d..8db5776c91 100644 --- a/packages/frontend-embed/tsconfig.json +++ b/packages/frontend-embed/tsconfig.json @@ -23,6 +23,7 @@ "useDefineForClassFields": true, "verbatimModuleSyntax": true, "skipLibCheck": true, + "incremental": true, "baseUrl": ".", "paths": { "@/*": ["./src/*"], diff --git a/packages/frontend-shared/tsconfig.json b/packages/frontend-shared/tsconfig.json index 8f76763e10..0512b50caf 100644 --- a/packages/frontend-shared/tsconfig.json +++ b/packages/frontend-shared/tsconfig.json @@ -18,6 +18,7 @@ "esModuleInterop": true, "verbatimModuleSyntax": true, "skipLibCheck": true, + "incremental": true, "baseUrl": ".", "paths": { "@/*": ["./*"], diff --git a/packages/frontend/.storybook/tsconfig.json b/packages/frontend/.storybook/tsconfig.json index f325114522..18baf516ba 100644 --- a/packages/frontend/.storybook/tsconfig.json +++ b/packages/frontend/.storybook/tsconfig.json @@ -18,6 +18,7 @@ "esModuleInterop": true, "skipLibCheck": true, "forceConsistentCasingInFileNames": true, + "incremental": true, "jsx": "react", "jsxFactory": "h" }, diff --git a/packages/frontend/src/workers/tsconfig.json b/packages/frontend/src/workers/tsconfig.json index 8ee8930465..39ba45ddbb 100644 --- a/packages/frontend/src/workers/tsconfig.json +++ b/packages/frontend/src/workers/tsconfig.json @@ -1,5 +1,6 @@ { "compilerOptions": { "lib": ["esnext", "webworker"], + "incremental": true } } diff --git a/packages/frontend/test/tsconfig.json b/packages/frontend/test/tsconfig.json index 98ac45211b..1490a66d20 100644 --- a/packages/frontend/test/tsconfig.json +++ b/packages/frontend/test/tsconfig.json @@ -22,6 +22,7 @@ "emitDecoratorMetadata": true, "resolveJsonModule": true, "isolatedModules": true, + "incremental": true, "baseUrl": "./", "paths": { "@/*": ["../src/*"] diff --git a/packages/frontend/tsconfig.json b/packages/frontend/tsconfig.json index 3c7e5e1da3..0616eee5be 100644 --- a/packages/frontend/tsconfig.json +++ b/packages/frontend/tsconfig.json @@ -23,6 +23,7 @@ "useDefineForClassFields": true, "verbatimModuleSyntax": true, "skipLibCheck": true, + "incremental": true, "baseUrl": ".", "paths": { "@/*": ["./src/*"], diff --git a/packages/misskey-bubble-game/tsconfig.json b/packages/misskey-bubble-game/tsconfig.json index f467951ef6..3cf8bb037f 100644 --- a/packages/misskey-bubble-game/tsconfig.json +++ b/packages/misskey-bubble-game/tsconfig.json @@ -16,6 +16,7 @@ "noImplicitReturns": true, "esModuleInterop": true, "skipLibCheck": true, + "incremental": true, "typeRoots": [ "./node_modules/@types" ], diff --git a/packages/misskey-js/generator/tsconfig.json b/packages/misskey-js/generator/tsconfig.json index d65042dc6d..0de2a7fe77 100644 --- a/packages/misskey-js/generator/tsconfig.json +++ b/packages/misskey-js/generator/tsconfig.json @@ -8,6 +8,7 @@ "strictFunctionTypes": true, "strictNullChecks": true, "esModuleInterop": true, + "incremental": true, "lib": [ "esnext", ] diff --git a/packages/misskey-js/tsconfig.json b/packages/misskey-js/tsconfig.json index 95128b8fab..e0603832c7 100644 --- a/packages/misskey-js/tsconfig.json +++ b/packages/misskey-js/tsconfig.json @@ -17,6 +17,7 @@ "esModuleInterop": true, "exactOptionalPropertyTypes": true, "skipLibCheck": true, + "incremental": true, "typeRoots": [ "./node_modules/@types" ], diff --git a/packages/misskey-reversi/tsconfig.json b/packages/misskey-reversi/tsconfig.json index f467951ef6..3cf8bb037f 100644 --- a/packages/misskey-reversi/tsconfig.json +++ b/packages/misskey-reversi/tsconfig.json @@ -16,6 +16,7 @@ "noImplicitReturns": true, "esModuleInterop": true, "skipLibCheck": true, + "incremental": true, "typeRoots": [ "./node_modules/@types" ], diff --git a/packages/sw/tsconfig.json b/packages/sw/tsconfig.json index 112a932e58..3a78106e46 100644 --- a/packages/sw/tsconfig.json +++ b/packages/sw/tsconfig.json @@ -20,6 +20,7 @@ "resolveJsonModule": true, "isolatedModules": true, "skipLibCheck": true, + "incremental": true, "baseUrl": ".", "paths": { "@/*": ["./src/*"], From 566f92ab735b489e660bda1ded6106673ab1d892 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 24 May 2025 18:28:07 -0400 Subject: [PATCH 02/29] add IDX_instance_host_key --- ...48104955717-index_IDX_instance_host_key.js | 20 +++++++++++++++++++ packages/backend/src/models/Instance.ts | 1 + 2 files changed, 21 insertions(+) create mode 100644 packages/backend/migration/1748104955717-index_IDX_instance_host_key.js diff --git a/packages/backend/migration/1748104955717-index_IDX_instance_host_key.js b/packages/backend/migration/1748104955717-index_IDX_instance_host_key.js new file mode 100644 index 0000000000..3c3b779298 --- /dev/null +++ b/packages/backend/migration/1748104955717-index_IDX_instance_host_key.js @@ -0,0 +1,20 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class IndexIDXInstanceHostKey1748104955717 { + name = 'IndexIDXInstanceHostKey1748104955717' + + async up(queryRunner) { + await queryRunner.query(`CREATE UNIQUE INDEX "IDX_instance_host_key" ON "instance" (((lower(reverse("host")) || '.')::text) text_pattern_ops)`); + + // Flush all cached Linear Scan Plans and redo statistics for expression index + // this is important for Postgres to learn that even in highly complex queries, using this index first can reduce the result set significantly + await queryRunner.query(`ANALYZE "instance"`); + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "IDX_instance_host_key"`); + } +} diff --git a/packages/backend/src/models/Instance.ts b/packages/backend/src/models/Instance.ts index c64ebb1b3b..5d8108d423 100644 --- a/packages/backend/src/models/Instance.ts +++ b/packages/backend/src/models/Instance.ts @@ -6,6 +6,7 @@ import { Entity, PrimaryColumn, Index, Column } from 'typeorm'; import { id } from './util/id.js'; +@Index('IDX_instance_host_key', { synchronize: false }) @Entity('instance') export class MiInstance { @PrimaryColumn(id()) From 45e5749cca8ddf11ba01ab3d669cff9517045a3f Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 24 May 2025 18:28:21 -0400 Subject: [PATCH 03/29] add instance properties for persisted block data --- ...748105111513-add_instance_block_columns.js | 89 +++++++++++++++++++ packages/backend/src/models/Instance.ts | 50 +++++++++++ 2 files changed, 139 insertions(+) create mode 100644 packages/backend/migration/1748105111513-add_instance_block_columns.js diff --git a/packages/backend/migration/1748105111513-add_instance_block_columns.js b/packages/backend/migration/1748105111513-add_instance_block_columns.js new file mode 100644 index 0000000000..c9087604e5 --- /dev/null +++ b/packages/backend/migration/1748105111513-add_instance_block_columns.js @@ -0,0 +1,89 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + * @typedef {{ blockedHosts: string[], silencedHosts: string[], mediaSilencedHosts: string[], federationHosts: string[], bubbleInstances: string[] }} Meta + */ + +/** + * @class + * @implements {MigrationInterface} + */ +export class AddInstanceBlockColumns1748105111513 { + name = 'AddInstanceBlockColumns1748105111513' + + async up(queryRunner) { + // Schema migration + await queryRunner.query(`ALTER TABLE "instance" ADD "isBlocked" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "instance"."isBlocked" IS 'True if this instance is blocked from federation.'`); + await queryRunner.query(`ALTER TABLE "instance" ADD "isAllowListed" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "instance"."isAllowListed" IS 'True if this instance is allow-listed.'`); + await queryRunner.query(`ALTER TABLE "instance" ADD "isBubbled" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "instance"."isBubbled" IS 'True if this instance is part of the local bubble.'`); + await queryRunner.query(`ALTER TABLE "instance" ADD "isSilenced" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "instance"."isSilenced" IS 'True if this instance is silenced.'`); + await queryRunner.query(`ALTER TABLE "instance" ADD "isMediaSilenced" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "instance"."isMediaSilenced" IS 'True if this instance is media-silenced.'`); + + // Data migration + /** @type {Meta[]} */ + const metas = await queryRunner.query(`SELECT "blockedHosts", "silencedHosts", "mediaSilencedHosts", "federationHosts", "bubbleInstances" FROM "meta"`); + if (metas.length > 0) { + /** @type {Meta} */ + const meta = metas[0]; + + // Blocked hosts + if (meta.blockedHosts.length > 0) { + const pattern = buildPatterns(meta.blockedHosts); + await queryRunner.query(`UPDATE "instance" SET "isBlocked" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ${pattern}`); + } + + // Silenced hosts + if (meta.silencedHosts.length > 0) { + const pattern = buildPatterns(meta.silencedHosts); + await queryRunner.query(`UPDATE "instance" SET "isSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ${pattern}`); + } + + // Media silenced hosts + if (meta.mediaSilencedHosts.length > 0) { + const pattern = buildPatterns(meta.mediaSilencedHosts); + await queryRunner.query(`UPDATE "instance" SET "isMediaSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ${pattern}`); + } + + // Allow-listed hosts + if (meta.federationHosts.length > 0) { + const pattern = buildPatterns(meta.federationHosts); + await queryRunner.query(`UPDATE "instance" SET "isAllowListed" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ${pattern}`); + } + + // Bubbled hosts + if (meta.bubbleInstances.length > 0) { + const pattern = buildPatterns(meta.bubbleInstances); + await queryRunner.query(`UPDATE "instance" SET "isBubbled" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ${pattern}`); + } + } + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isMediaSilenced"`); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isSilenced"`); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isBubbled"`); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isAllowListed"`); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isBlocked"`); + } +} + +/** + * @param {string[]} input + * @returns {string} + */ +function buildPatterns(input) { + const strings = input + .map(i => i.toLowerCase().split('').reverse().join('') + '.%') + .map(i => `'${i}'`) + .join(', '); + return `(array[${strings}]::text[])`; +} diff --git a/packages/backend/src/models/Instance.ts b/packages/backend/src/models/Instance.ts index 5d8108d423..0022e58933 100644 --- a/packages/backend/src/models/Instance.ts +++ b/packages/backend/src/models/Instance.ts @@ -99,6 +99,56 @@ export class MiInstance { }) public suspensionState: 'none' | 'manuallySuspended' | 'goneSuspended' | 'autoSuspendedForNotResponding'; + /** + * True if this instance is blocked from federation. + */ + @Column('boolean', { + nullable: false, + default: false, + comment: 'True if this instance is blocked from federation.', + }) + public isBlocked: boolean; + + /** + * True if this instance is allow-listed. + */ + @Column('boolean', { + nullable: false, + default: false, + comment: 'True if this instance is allow-listed.', + }) + public isAllowListed: boolean; + + /** + * True if this instance is part of the local bubble. + */ + @Column('boolean', { + nullable: false, + default: false, + comment: 'True if this instance is part of the local bubble.', + }) + public isBubbled: boolean; + + /** + * True if this instance is silenced. + */ + @Column('boolean', { + nullable: false, + default: false, + comment: 'True if this instance is silenced.', + }) + public isSilenced: boolean; + + /** + * True if this instance is media-silenced. + */ + @Column('boolean', { + nullable: false, + default: false, + comment: 'True if this instance is media-silenced.', + }) + public isMediaSilenced: boolean; + @Column('varchar', { length: 64, nullable: true, comment: 'The software of the Instance.', From 373c60b52173e6ed8bcc01b3c4f1c6ac4734398a Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 24 May 2025 18:28:43 -0400 Subject: [PATCH 04/29] add diff-arrays utility for efficient array diffs --- packages/backend/src/misc/diff-arrays.ts | 56 +++++++++++++++++++ .../backend/test/unit/misc/diff-arrays.ts | 53 ++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 packages/backend/src/misc/diff-arrays.ts create mode 100644 packages/backend/test/unit/misc/diff-arrays.ts diff --git a/packages/backend/src/misc/diff-arrays.ts b/packages/backend/src/misc/diff-arrays.ts new file mode 100644 index 0000000000..1f6820481b --- /dev/null +++ b/packages/backend/src/misc/diff-arrays.ts @@ -0,0 +1,56 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export interface DiffResult { + added: T[]; + removed: T[]; +} + +/** + * Calculates the difference between two snapshots of data. + * Null, undefined, and empty arrays are supported, and values do not have to be unique. + * Result sets are de-duplicated, and will be empty if no data was added or removed (respectively). + * @param dataBefore Array containing data before the change + * @param dataAfter Array containing data after the change + */ +export function diffArrays(dataBefore: T[] | null | undefined, dataAfter: T[] | null | undefined): DiffResult { + const before = dataBefore ? new Set(dataBefore) : null; + const after = dataAfter ? new Set(dataAfter) : null; + + // data before AND after => changed + if (before?.size && after?.size) { + const added: T[] = []; + const removed: T[] = []; + + for (const host of before) { + // before and NOT after => removed + if (!after.has(host)) { + removed.push(host); + } + } + + for (const host of after) { + // after and NOT before => added + if (!before.has(host)) { + added.push(host); + } + } + + return { added, removed }; + } + + // data ONLY before => all removed + if (before?.size) { + return { added: [], removed: Array.from(before) }; + } + + // data ONLY after => all added + if (after?.size) { + return { added: Array.from(after), removed: [] }; + } + + // data NEITHER before nor after => no change + return { added: [], removed: [] }; +} diff --git a/packages/backend/test/unit/misc/diff-arrays.ts b/packages/backend/test/unit/misc/diff-arrays.ts new file mode 100644 index 0000000000..3eed5a8913 --- /dev/null +++ b/packages/backend/test/unit/misc/diff-arrays.ts @@ -0,0 +1,53 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { diffArrays } from '@/misc/diff-arrays.js'; + +describe(diffArrays, () => { + it('should return empty result when both inputs are null', () => { + const result = diffArrays(null, null); + expect(result.added).toHaveLength(0); + expect(result.removed).toHaveLength(0); + }); + + it('should return empty result when both inputs are empty', () => { + const result = diffArrays([], []); + expect(result.added).toHaveLength(0); + expect(result.removed).toHaveLength(0); + }); + + it('should remove before when added is empty', () => { + const result = diffArrays([1, 2, 3], []); + expect(result.added).toHaveLength(0); + expect(result.removed).toEqual([1, 2, 3]); + }); + + it('should deduplicate before when added is empty', () => { + const result = diffArrays([1, 1, 2, 2, 3], []); + expect(result.added).toHaveLength(0); + expect(result.removed).toEqual([1, 2, 3]); + }); + + it('should remove after when before is empty', () => { + const result = diffArrays([], [1, 2, 3]); + expect(result.added).toEqual([1, 2, 3]); + expect(result.removed).toHaveLength(0); + }); + + it('should deduplicate after when before is empty', () => { + const result = diffArrays([], [1, 1, 2, 2, 3]); + expect(result.added).toEqual([1, 2, 3]); + expect(result.removed).toHaveLength(0); + }); + + it('should return diff when both have values', () => { + const result = diffArrays( + ['a', 'b', 'c', 'd'], + ['a', 'c', 'e', 'f'], + ); + expect(result.added).toEqual(['e', 'f']); + expect(result.removed).toEqual(['b', 'd']); + }); +}); From 305250d0738b1fc7332ee9556b8d9e6aed5a8237 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 24 May 2025 18:29:10 -0400 Subject: [PATCH 05/29] persist changes to meta host lists to instance table --- packages/backend/src/core/MetaService.ts | 55 +++++++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/MetaService.ts b/packages/backend/src/core/MetaService.ts index 40e7439f5f..16be22ac19 100644 --- a/packages/backend/src/core/MetaService.ts +++ b/packages/backend/src/core/MetaService.ts @@ -4,7 +4,7 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import { DataSource } from 'typeorm'; +import { DataSource, EntityManager } from 'typeorm'; import * as Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; import { MiMeta } from '@/models/Meta.js'; @@ -12,6 +12,8 @@ import { GlobalEventService } from '@/core/GlobalEventService.js'; import { bindThis } from '@/decorators.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; import { FeaturedService } from '@/core/FeaturedService.js'; +import { MiInstance } from '@/models/Instance.js'; +import { diffArrays } from '@/misc/diff-arrays.js'; import type { OnApplicationShutdown } from '@nestjs/common'; @Injectable() @@ -103,7 +105,7 @@ export class MetaService implements OnApplicationShutdown { let before: MiMeta | undefined; const updated = await this.db.transaction(async transactionalEntityManager => { - const metas = await transactionalEntityManager.find(MiMeta, { + const metas: (MiMeta | undefined)[] = await transactionalEntityManager.find(MiMeta, { order: { id: 'DESC', }, @@ -126,6 +128,10 @@ export class MetaService implements OnApplicationShutdown { }, }); + // Propagate changes to blockedHosts, silencedHosts, mediaSilencedHosts, federationInstances, and bubbleInstances to the relevant instance rows + // Do this inside the transaction to avoid potential race condition (when an instance gets registered while we're updating). + await this.persistBlocks(transactionalEntityManager, before ?? {}, afters[0]); + return afters[0]; }); @@ -159,4 +165,49 @@ export class MetaService implements OnApplicationShutdown { public onApplicationShutdown(signal?: string | undefined): void { this.dispose(); } + + private async persistBlocks(tem: EntityManager, before: Partial, after: Partial): Promise { + await this.persistBlock(tem, before.blockedHosts, after.blockedHosts, 'isBlocked'); + await this.persistBlock(tem, before.silencedHosts, after.silencedHosts, 'isSilenced'); + await this.persistBlock(tem, before.mediaSilencedHosts, after.mediaSilencedHosts, 'isMediaSilenced'); + await this.persistBlock(tem, before.federationHosts, after.federationHosts, 'isAllowListed'); + await this.persistBlock(tem, before.bubbleInstances, after.bubbleInstances, 'isBubbled'); + } + + private async persistBlock(tem: EntityManager, before: string[] | undefined, after: string[] | undefined, field: keyof MiInstance): Promise { + const { added, removed } = diffArrays(before, after); + + if (removed.length > 0) { + await this.updateInstancesByHost(tem, field, false, removed); + } + + if (added.length > 0) { + await this.updateInstancesByHost(tem, field, true, added); + } + } + + private async updateInstancesByHost(tem: EntityManager, field: keyof MiInstance, value: boolean, hosts: string[]): Promise { + // Use non-array queries when possible, as they are indexed and can be much faster. + if (hosts.length === 1) { + const pattern = genHostPattern(hosts[0]); + await tem + .createQueryBuilder(MiInstance, 'instance') + .update() + .set({ [field]: value }) + .where('(lower(reverse("host")) || \'.\') LIKE :pattern', { pattern }) + .execute(); + } else if (hosts.length > 1) { + const patterns = hosts.map(host => genHostPattern(host)); + await tem + .createQueryBuilder(MiInstance, 'instance') + .update() + .set({ [field]: value }) + .where('(lower(reverse("host")) || \'.\') LIKE ANY (:patterns)', {patterns}) + .execute(); + } + } +} + +function genHostPattern(host: string): string { + return host.toLowerCase().split('').reverse().join('') + '.%'; } From b422d5bc9d44e146b41f163c0101664a9b49268b Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 24 May 2025 18:42:07 -0400 Subject: [PATCH 06/29] add utility service overloads for quickly checking hosts against meta values --- packages/backend/src/core/UtilityService.ts | 39 +++++++++++++++++---- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/core/UtilityService.ts b/packages/backend/src/core/UtilityService.ts index 170afc72dc..3098367392 100644 --- a/packages/backend/src/core/UtilityService.ts +++ b/packages/backend/src/core/UtilityService.ts @@ -49,22 +49,49 @@ export class UtilityService { return regexp.test(email); } + public isBlockedHost(host: string | null): boolean; + public isBlockedHost(blockedHosts: string[], host: string | null): boolean; @bindThis - public isBlockedHost(blockedHosts: string[], host: string | null): boolean { + public isBlockedHost(blockedHostsOrHost: string[] | string | null, host?: string | null): boolean { + const blockedHosts = Array.isArray(blockedHostsOrHost) ? blockedHostsOrHost : this.meta.blockedHosts; + host = Array.isArray(blockedHostsOrHost) ? host : blockedHostsOrHost; + if (host == null) return false; return blockedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); } + public isSilencedHost(host: string | null): boolean; + public isSilencedHost(silencedHosts: string[], host: string | null): boolean; @bindThis - public isSilencedHost(silencedHosts: string[] | undefined, host: string | null): boolean { - if (!silencedHosts || host == null) return false; + public isSilencedHost(silencedHostsOrHost: string[] | string | null, host?: string | null): boolean { + const silencedHosts = Array.isArray(silencedHostsOrHost) ? silencedHostsOrHost : this.meta.silencedHosts; + host = Array.isArray(silencedHostsOrHost) ? host : silencedHostsOrHost; + + if (host == null) return false; return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); } + public isMediaSilencedHost(host: string | null): boolean; + public isMediaSilencedHost(silencedHosts: string[], host: string | null): boolean; @bindThis - public isMediaSilencedHost(silencedHosts: string[] | undefined, host: string | null): boolean { - if (!silencedHosts || host == null) return false; - return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); + public isMediaSilencedHost(mediaSilencedHostsOrHost: string[] | string | null, host?: string | null): boolean { + const mediaSilencedHosts = Array.isArray(mediaSilencedHostsOrHost) ? mediaSilencedHostsOrHost : this.meta.mediaSilencedHosts; + host = Array.isArray(mediaSilencedHostsOrHost) ? host : mediaSilencedHostsOrHost; + + if (host == null) return false; + return mediaSilencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); + } + + @bindThis + public isAllowListedHost(host: string | null): boolean { + if (host == null) return false; + return this.meta.federationHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); + } + + @bindThis + public isBubbledHost(host: string | null): boolean { + if (host == null) return false; + return this.meta.bubbleInstances.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); } @bindThis From f3eca0b5cfff9b22f0ac90b65c2c979a9ef0b56d Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 24 May 2025 18:43:43 -0400 Subject: [PATCH 07/29] populate block fields when registering a new instance --- .../src/core/FederatedInstanceService.ts | 38 +++++++++---------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/packages/backend/src/core/FederatedInstanceService.ts b/packages/backend/src/core/FederatedInstanceService.ts index 3f7ed99348..e31d802e8c 100644 --- a/packages/backend/src/core/FederatedInstanceService.ts +++ b/packages/backend/src/core/FederatedInstanceService.ts @@ -5,9 +5,9 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Redis from 'ioredis'; -import { QueryFailedError } from 'typeorm'; +import { DataSource, QueryFailedError } from 'typeorm'; import type { InstancesRepository } from '@/models/_.js'; -import type { MiInstance } from '@/models/Instance.js'; +import { MiInstance } from '@/models/Instance.js'; import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; import { IdService } from '@/core/IdService.js'; import { DI } from '@/di-symbols.js'; @@ -26,6 +26,9 @@ export class FederatedInstanceService implements OnApplicationShutdown { @Inject(DI.instancesRepository) private instancesRepository: InstancesRepository, + @Inject(DI.db) + private readonly db: DataSource, + private utilityService: UtilityService, private idService: IdService, ) { @@ -55,34 +58,27 @@ export class FederatedInstanceService implements OnApplicationShutdown { const cached = await this.federatedInstanceCache.get(host); if (cached) return cached; - const index = await this.instancesRepository.findOneBy({ host }); + return await this.db.transaction(async tem => { + let index = await tem.findOneBy(MiInstance, { host }); - if (index == null) { - let i; - try { - i = await this.instancesRepository.insertOne({ + if (index == null) { + await tem.insert(MiInstance, { id: this.idService.gen(), host, firstRetrievedAt: new Date(), + isBlocked: this.utilityService.isBlockedHost(host), + isSilenced: this.utilityService.isSilencedHost(host), + isMediaSilenced: this.utilityService.isMediaSilencedHost(host), + isAllowListed: this.utilityService.isAllowListedHost(host), + isBubbled: this.utilityService.isBubbledHost(host), }); - } catch (e: unknown) { - if (e instanceof QueryFailedError) { - if (isDuplicateKeyValueError(e)) { - i = await this.instancesRepository.findOneBy({ host }); - } - } - if (i == null) { - throw e; - } + index = await tem.findOneByOrFail(MiInstance, { host }); } - this.federatedInstanceCache.set(host, i); - return i; - } else { - this.federatedInstanceCache.set(host, index); + await this.federatedInstanceCache.set(host, index); return index; - } + }); } @bindThis From 59099a2b2a1d570334df3a0770450c8b4b1522c6 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 24 May 2025 19:10:41 -0400 Subject: [PATCH 08/29] add foreign keys to note/user where instance is referenced --- ...1748128176881-add_instance_foreign_keys.js | 30 ++++++++++++++++++ packages/backend/src/models/Note.ts | 31 +++++++++++++++++++ packages/backend/src/models/User.ts | 13 +++++++- 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 packages/backend/migration/1748128176881-add_instance_foreign_keys.js diff --git a/packages/backend/migration/1748128176881-add_instance_foreign_keys.js b/packages/backend/migration/1748128176881-add_instance_foreign_keys.js new file mode 100644 index 0000000000..da2de14f18 --- /dev/null +++ b/packages/backend/migration/1748128176881-add_instance_foreign_keys.js @@ -0,0 +1,30 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + */ + +/** + * @class + * @implements {MigrationInterface} + */ +export class AddInstanceForeignKeys1748128176881 { + name = 'AddInstanceForeignKeys1748128176881' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "user" ADD CONSTRAINT "FK_user_host" FOREIGN KEY ("host") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_userHost" FOREIGN KEY ("userHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_replyUserHost" FOREIGN KEY ("replyUserHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_renoteUserHost" FOREIGN KEY ("renoteUserHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_renoteUserHost"`); + await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_replyUserHost"`); + await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_userHost"`); + await queryRunner.query(`ALTER TABLE "user" DROP CONSTRAINT "FK_user_host"`); + } +} diff --git a/packages/backend/src/models/Note.ts b/packages/backend/src/models/Note.ts index ee2098216d..fa5839b6ec 100644 --- a/packages/backend/src/models/Note.ts +++ b/packages/backend/src/models/Note.ts @@ -5,6 +5,7 @@ import { Entity, Index, JoinColumn, Column, PrimaryColumn, ManyToOne } from 'typeorm'; import { noteVisibilities } from '@/types.js'; +import { MiInstance } from '@/models/Instance.js'; import { id } from './util/id.js'; import { MiUser } from './User.js'; import { MiChannel } from './Channel.js'; @@ -222,6 +223,16 @@ export class MiNote { }) public userHost: string | null; + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'userHost', + foreignKeyConstraintName: 'FK_note_userHost', + referencedColumnName: 'host', + }) + public userInstance: MiInstance | null; + @Column({ ...id(), nullable: true, @@ -235,6 +246,16 @@ export class MiNote { }) public replyUserHost: string | null; + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'replyUserHost', + foreignKeyConstraintName: 'FK_note_replyUserHost', + referencedColumnName: 'host', + }) + public replyUserInstance: MiInstance | null; + @Column({ ...id(), nullable: true, @@ -247,6 +268,16 @@ export class MiNote { comment: '[Denormalized]', }) public renoteUserHost: string | null; + + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'renoteUserHost', + foreignKeyConstraintName: 'FK_note_renoteUserHost', + referencedColumnName: 'host', + }) + public renoteUserInstance: MiInstance | null; //#endregion constructor(data: Partial) { diff --git a/packages/backend/src/models/User.ts b/packages/backend/src/models/User.ts index 46f8e84a94..55b8f4f4f0 100644 --- a/packages/backend/src/models/User.ts +++ b/packages/backend/src/models/User.ts @@ -3,8 +3,9 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Entity, Column, Index, OneToOne, JoinColumn, PrimaryColumn } from 'typeorm'; +import { Entity, Column, Index, OneToOne, JoinColumn, PrimaryColumn, ManyToOne } from 'typeorm'; import { type UserUnsignedFetchOption, userUnsignedFetchOptions } from '@/const.js'; +import { MiInstance } from '@/models/Instance.js'; import { id } from './util/id.js'; import { MiDriveFile } from './DriveFile.js'; @@ -292,6 +293,16 @@ export class MiUser { }) public host: string | null; + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'host', + foreignKeyConstraintName: 'FK_user_host', + referencedColumnName: 'host', + }) + public instance: MiInstance | null; + @Column('varchar', { length: 512, nullable: true, comment: 'The inbox URL of the User. It will be null if the origin of the user is local.', From 51128028f47b9c1b13966c0dbbb0c185e7f91c32 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 24 May 2025 22:00:57 -0400 Subject: [PATCH 09/29] add foreign keys to following where instance is referenced --- ...-add_instance_foreign_keys_to_following.js | 26 +++++++++++++++++++ packages/backend/src/models/Following.ts | 21 +++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 packages/backend/migration/1748137683887-add_instance_foreign_keys_to_following.js diff --git a/packages/backend/migration/1748137683887-add_instance_foreign_keys_to_following.js b/packages/backend/migration/1748137683887-add_instance_foreign_keys_to_following.js new file mode 100644 index 0000000000..8f4a977ff5 --- /dev/null +++ b/packages/backend/migration/1748137683887-add_instance_foreign_keys_to_following.js @@ -0,0 +1,26 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + */ + +/** + * @class + * @implements {MigrationInterface} + */ +export class AddInstanceForeignKeysToFollowing1748137683887 { + name = 'AddInstanceForeignKeysToFollowing1748137683887' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "following" ADD CONSTRAINT "FK_following_followerHost" FOREIGN KEY ("followerHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "following" ADD CONSTRAINT "FK_following_followeeHost" FOREIGN KEY ("followeeHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "following" DROP CONSTRAINT "FK_following_followeeHost"`); + await queryRunner.query(`ALTER TABLE "following" DROP CONSTRAINT "FK_following_followerHost"`); + } +} diff --git a/packages/backend/src/models/Following.ts b/packages/backend/src/models/Following.ts index 62cbc29f26..0aa1b13976 100644 --- a/packages/backend/src/models/Following.ts +++ b/packages/backend/src/models/Following.ts @@ -4,6 +4,7 @@ */ import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm'; +import { MiInstance } from '@/models/Instance.js'; import { id } from './util/id.js'; import { MiUser } from './User.js'; @@ -66,6 +67,16 @@ export class MiFollowing { }) public followerHost: string | null; + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'followerHost', + foreignKeyConstraintName: 'FK_following_followerHost', + referencedColumnName: 'host', + }) + public followerInstance: MiInstance | null; + @Column('varchar', { length: 512, nullable: true, comment: '[Denormalized]', @@ -85,6 +96,16 @@ export class MiFollowing { }) public followeeHost: string | null; + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'followeeHost', + foreignKeyConstraintName: 'FK_following_followeeHost', + referencedColumnName: 'host', + }) + public followeeInstance: MiInstance | null; + @Column('varchar', { length: 512, nullable: true, comment: '[Denormalized]', From abac2f785a2e820f32914cbfa0fb00c4973fd703 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 24 May 2025 22:01:42 -0400 Subject: [PATCH 10/29] fix QueryService.generateMutedUserRenotesQueryForNotes to properly exclude quotes --- packages/backend/src/core/QueryService.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/core/QueryService.ts b/packages/backend/src/core/QueryService.ts index 50a72e8aa6..e87360b00d 100644 --- a/packages/backend/src/core/QueryService.ts +++ b/packages/backend/src/core/QueryService.ts @@ -243,13 +243,12 @@ export class QueryService { q.andWhere(new Brackets(qb => { qb - .where(new Brackets(qb => { - qb.where('note.renoteId IS NOT NULL'); - qb.andWhere('note.text IS NULL'); - qb.andWhere(`note.userId NOT IN (${ mutingQuery.getQuery() })`); - })) .orWhere('note.renoteId IS NULL') - .orWhere('note.text IS NOT NULL'); + .orWhere('note.text IS NOT NULL') + .orWhere('note.cw IS NOT NULL') + .orWhere('note.replyId IS NOT NULL') + .orWhere('note.fileIds != \'{}\'') + .orWhere(`note.userId NOT IN (${ mutingQuery.getQuery() })`); })); q.setParameters(mutingQuery.getParameters()); From fad82000f09ec4a9843e5ffb14924ab1288f2e0c Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 24 May 2025 23:12:52 -0400 Subject: [PATCH 11/29] fix lint error in MetaService --- packages/backend/src/core/MetaService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/MetaService.ts b/packages/backend/src/core/MetaService.ts index 16be22ac19..5b6ee8920e 100644 --- a/packages/backend/src/core/MetaService.ts +++ b/packages/backend/src/core/MetaService.ts @@ -202,7 +202,7 @@ export class MetaService implements OnApplicationShutdown { .createQueryBuilder(MiInstance, 'instance') .update() .set({ [field]: value }) - .where('(lower(reverse("host")) || \'.\') LIKE ANY (:patterns)', {patterns}) + .where('(lower(reverse("host")) || \'.\') LIKE ANY (:patterns)', { patterns }) .execute(); } } From 70641501444820b2e30c01ae152bf396e8cabf7e Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 24 May 2025 23:14:57 -0400 Subject: [PATCH 12/29] use instance block columns instead of checking meta columns --- .../src/core/FanoutTimelineEndpointService.ts | 11 +++-- packages/backend/src/core/QueryService.ts | 46 ++++++++----------- .../src/core/chart/charts/federation.ts | 23 +++++----- .../core/entities/InstanceEntityService.ts | 6 +-- .../server/api/endpoints/antennas/notes.ts | 1 + .../server/api/endpoints/channels/timeline.ts | 2 + .../src/server/api/endpoints/clips/notes.ts | 3 +- .../api/endpoints/notes/bubble-timeline.ts | 2 +- .../api/endpoints/notes/search-by-tag.ts | 5 +- .../src/server/api/endpoints/roles/notes.ts | 2 +- .../server/api/endpoints/users/reactions.ts | 7 ++- 11 files changed, 56 insertions(+), 52 deletions(-) diff --git a/packages/backend/src/core/FanoutTimelineEndpointService.ts b/packages/backend/src/core/FanoutTimelineEndpointService.ts index af2723e99d..f9cf41e854 100644 --- a/packages/backend/src/core/FanoutTimelineEndpointService.ts +++ b/packages/backend/src/core/FanoutTimelineEndpointService.ts @@ -136,10 +136,10 @@ export class FanoutTimelineEndpointService { const parentFilter = filter; filter = (note) => { if (!ps.ignoreAuthorFromInstanceBlock) { - if (this.utilityService.isBlockedHost(this.meta.blockedHosts, note.userHost)) return false; + if (note.userInstance?.isBlocked) return false; } - if (note.userId !== note.renoteUserId && this.utilityService.isBlockedHost(this.meta.blockedHosts, note.renoteUserHost)) return false; - if (note.userId !== note.replyUserId && this.utilityService.isBlockedHost(this.meta.blockedHosts, note.replyUserHost)) return false; + if (note.userId !== note.renoteUserId && note.renoteUserInstance?.isBlocked) return false; + if (note.userId !== note.replyUserId && note.replyUserInstance?.isBlocked) return false; return parentFilter(note); }; @@ -194,7 +194,10 @@ export class FanoutTimelineEndpointService { .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); + .leftJoinAndSelect('note.channel', 'channel') + .leftJoinAndSelect('note.userInstance', 'userInstance') + .leftJoinAndSelect('note.replyUserInstance', 'replyUserInstance') + .leftJoinAndSelect('note.renoteUserInstance', 'renoteUserInstance'); const notes = (await query.getMany()).filter(noteFilter); diff --git a/packages/backend/src/core/QueryService.ts b/packages/backend/src/core/QueryService.ts index e87360b00d..1b00f41d20 100644 --- a/packages/backend/src/core/QueryService.ts +++ b/packages/backend/src/core/QueryService.ts @@ -255,34 +255,28 @@ export class QueryService { } @bindThis - public generateBlockedHostQueryForNote(q: SelectQueryBuilder, excludeAuthor?: boolean): void { - let nonBlockedHostQuery: (part: string) => string; - if (this.meta.blockedHosts.length === 0) { - nonBlockedHostQuery = () => '1=1'; - } else { - nonBlockedHostQuery = (match: string) => `('.' || ${match}) NOT ILIKE ALL(select '%.' || x from (select unnest("blockedHosts") as x from "meta") t)`; + public generateBlockedHostQueryForNote(q: SelectQueryBuilder, excludeAuthor?: boolean, allowSilenced = true): void { + function checkFor(key: 'user' | 'replyUser' | 'renoteUser') { + q.leftJoin(`note.${key}Instance`, `${key}Instance`); + q.andWhere(new Brackets(qb => { + qb.orWhere(`note.${key}Id IS NULL`) // no corresponding user + .orWhere(`note.${key}Host IS NULL`) // local + .orWhere(`${key}Instance.isBlocked = false`); // not blocked + + if (!allowSilenced) { + qb.orWhere(`${key}Instance.isSilenced = false`); // not silenced + } + + if (excludeAuthor) { + qb.orWhere(`note.userId = note.${key}Id`); // author + } + })); } - if (excludeAuthor) { - const instanceSuspension = (user: string) => new Brackets(qb => qb - .where(`note.${user}Id IS NULL`) // no corresponding user - .orWhere(`note.userId = note.${user}Id`) - .orWhere(`note.${user}Host IS NULL`) // local - .orWhere(nonBlockedHostQuery(`note.${user}Host`))); - - q - .andWhere(instanceSuspension('replyUser')) - .andWhere(instanceSuspension('renoteUser')); - } else { - const instanceSuspension = (user: string) => new Brackets(qb => qb - .where(`note.${user}Id IS NULL`) // no corresponding user - .orWhere(`note.${user}Host IS NULL`) // local - .orWhere(nonBlockedHostQuery(`note.${user}Host`))); - - q - .andWhere(instanceSuspension('user')) - .andWhere(instanceSuspension('replyUser')) - .andWhere(instanceSuspension('renoteUser')); + if (!excludeAuthor) { + checkFor('user'); } + checkFor('replyUser'); + checkFor('renoteUser'); } } diff --git a/packages/backend/src/core/chart/charts/federation.ts b/packages/backend/src/core/chart/charts/federation.ts index bf702884ca..b6db6f5454 100644 --- a/packages/backend/src/core/chart/charts/federation.ts +++ b/packages/backend/src/core/chart/charts/federation.ts @@ -44,10 +44,6 @@ export default class FederationChart extends Chart { // eslint-di } protected async tickMinor(): Promise>> { - const suspendedInstancesQuery = this.instancesRepository.createQueryBuilder('instance') - .select('instance.host') - .where('instance.suspensionState != \'none\''); - const pubsubSubQuery = this.followingsRepository.createQueryBuilder('f') .select('f.followerHost') .where('f.followerHost IS NOT NULL'); @@ -64,22 +60,25 @@ export default class FederationChart extends Chart { // eslint-di this.followingsRepository.createQueryBuilder('following') .select('COUNT(DISTINCT following.followeeHost)') .where('following.followeeHost IS NOT NULL') - .andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followeeHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)') - .andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`) + .innerJoin('following.followeeInstance', 'followeeInstance') + .andWhere('followeeInstance.suspensionState = \'none\'') + .andWhere('followeeInstance.isBlocked = false') .getRawOne() .then(x => parseInt(x.count, 10)), this.followingsRepository.createQueryBuilder('following') .select('COUNT(DISTINCT following.followerHost)') .where('following.followerHost IS NOT NULL') - .andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followerHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)') - .andWhere(`following.followerHost NOT IN (${ suspendedInstancesQuery.getQuery() })`) + .innerJoin('following.followerInstance', 'followerInstance') + .andWhere('followerInstance.isBlocked = false') + .andWhere('followerInstance.suspensionState = \'none\'') .getRawOne() .then(x => parseInt(x.count, 10)), this.followingsRepository.createQueryBuilder('following') .select('COUNT(DISTINCT following.followeeHost)') .where('following.followeeHost IS NOT NULL') - .andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followeeHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)') - .andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`) + .innerJoin('following.followeeInstance', 'followeeInstance') + .andWhere('followeeInstance.isBlocked = false') + .andWhere('followeeInstance.suspensionState = \'none\'') .andWhere(`following.followeeHost IN (${ pubsubSubQuery.getQuery() })`) .setParameters(pubsubSubQuery.getParameters()) .getRawOne() @@ -87,7 +86,7 @@ export default class FederationChart extends Chart { // eslint-di this.instancesRepository.createQueryBuilder('instance') .select('COUNT(instance.id)') .where(`instance.host IN (${ subInstancesQuery.getQuery() })`) - .andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || instance.host) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)') + .andWhere('instance.isBlocked = false') .andWhere('instance.suspensionState = \'none\'') .andWhere('instance.isNotResponding = false') .getRawOne() @@ -95,7 +94,7 @@ export default class FederationChart extends Chart { // eslint-di this.instancesRepository.createQueryBuilder('instance') .select('COUNT(instance.id)') .where(`instance.host IN (${ pubInstancesQuery.getQuery() })`) - .andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || instance.host) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)') + .andWhere('instance.isBlocked = false') .andWhere('instance.suspensionState = \'none\'') .andWhere('instance.isNotResponding = false') .getRawOne() diff --git a/packages/backend/src/core/entities/InstanceEntityService.ts b/packages/backend/src/core/entities/InstanceEntityService.ts index fcc9bed3bd..332d2943a4 100644 --- a/packages/backend/src/core/entities/InstanceEntityService.ts +++ b/packages/backend/src/core/entities/InstanceEntityService.ts @@ -43,7 +43,7 @@ export class InstanceEntityService { isNotResponding: instance.isNotResponding, isSuspended: instance.suspensionState !== 'none', suspensionState: instance.suspensionState, - isBlocked: this.utilityService.isBlockedHost(this.meta.blockedHosts, instance.host), + isBlocked: instance.isBlocked, softwareName: instance.softwareName, softwareVersion: instance.softwareVersion, openRegistrations: instance.openRegistrations, @@ -51,8 +51,8 @@ export class InstanceEntityService { description: instance.description, maintainerName: instance.maintainerName, maintainerEmail: instance.maintainerEmail, - isSilenced: this.utilityService.isSilencedHost(this.meta.silencedHosts, instance.host), - isMediaSilenced: this.utilityService.isMediaSilencedHost(this.meta.mediaSilencedHosts, instance.host), + isSilenced: instance.isSilenced, + isMediaSilenced: instance.isMediaSilenced, iconUrl: instance.iconUrl, faviconUrl: instance.faviconUrl, themeColor: instance.themeColor, diff --git a/packages/backend/src/server/api/endpoints/antennas/notes.ts b/packages/backend/src/server/api/endpoints/antennas/notes.ts index b90ba6aa0d..7e79f0dccc 100644 --- a/packages/backend/src/server/api/endpoints/antennas/notes.ts +++ b/packages/backend/src/server/api/endpoints/antennas/notes.ts @@ -121,6 +121,7 @@ export default class extends Endpoint { // eslint- this.queryService.generateVisibilityQuery(query, me); this.queryService.generateMutedUserQueryForNotes(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); const notes = await query.getMany(); if (sinceId != null && untilId == null) { diff --git a/packages/backend/src/server/api/endpoints/channels/timeline.ts b/packages/backend/src/server/api/endpoints/channels/timeline.ts index 6336f43e9f..99ae1c2211 100644 --- a/packages/backend/src/server/api/endpoints/channels/timeline.ts +++ b/packages/backend/src/server/api/endpoints/channels/timeline.ts @@ -138,9 +138,11 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('note.channel', 'channel'); this.queryService.generateBlockedHostQueryForNote(query); + this.queryService.generateVisibilityQuery(query, me); if (me) { this.queryService.generateMutedUserQueryForNotes(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); } if (ps.withRenotes === false) { diff --git a/packages/backend/src/server/api/endpoints/clips/notes.ts b/packages/backend/src/server/api/endpoints/clips/notes.ts index 59513e530d..4758dbad00 100644 --- a/packages/backend/src/server/api/endpoints/clips/notes.ts +++ b/packages/backend/src/server/api/endpoints/clips/notes.ts @@ -92,10 +92,11 @@ export default class extends Endpoint { // eslint- .andWhere('clipNote.clipId = :clipId', { clipId: clip.id }); this.queryService.generateBlockedHostQueryForNote(query); + this.queryService.generateVisibilityQuery(query, me); if (me) { - this.queryService.generateVisibilityQuery(query, me); this.queryService.generateMutedUserQueryForNotes(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); } const notes = await query diff --git a/packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts b/packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts index df030d90aa..be08f84b6b 100644 --- a/packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts @@ -85,7 +85,7 @@ export default class extends Endpoint { // eslint- ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) .andWhere('note.visibility = \'public\'') .andWhere('note.channelId IS NULL') - .andWhere('note.userHost IN (:...hosts)', { hosts: this.serverSettings.bubbleInstances }) + .andWhere('(note.userHost IS NULL OR userInstance.isBubbled = true)') // This comes from generateVisibilityQuery below .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') diff --git a/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts b/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts index 91874a8195..5c1ab0fb78 100644 --- a/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts +++ b/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts @@ -96,10 +96,10 @@ export default class extends Endpoint { // eslint- if (!this.serverSettings.enableBotTrending) query.andWhere('user.isBot = FALSE'); - this.queryService.generateVisibilityQuery(query, me); - this.queryService.generateBlockedHostQueryForNote(query); + this.queryService.generateBlockedHostQueryForNote(query, undefined, false); if (me) this.queryService.generateMutedUserQueryForNotes(query, me); if (me) this.queryService.generateBlockedUserQueryForNotes(query, me); + if (me) this.queryService.generateMutedUserRenotesQueryForNotes(query, me); const followings = me ? await this.cacheService.userFollowingsCache.fetch(me.id) : {}; @@ -160,7 +160,6 @@ export default class extends Endpoint { // eslint- if (note.user?.isSuspended) return false; if (note.userHost) { if (!this.utilityService.isFederationAllowedHost(note.userHost)) return false; - if (this.utilityService.isSilencedHost(this.serverSettings.silencedHosts, note.userHost)) return false; } return true; }); diff --git a/packages/backend/src/server/api/endpoints/roles/notes.ts b/packages/backend/src/server/api/endpoints/roles/notes.ts index d1c2e4b686..536384a381 100644 --- a/packages/backend/src/server/api/endpoints/roles/notes.ts +++ b/packages/backend/src/server/api/endpoints/roles/notes.ts @@ -107,10 +107,10 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('reply.user', 'replyUser') .leftJoinAndSelect('renote.user', 'renoteUser'); - this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedHostQueryForNote(query); this.queryService.generateMutedUserQueryForNotes(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); const notes = await query.getMany(); notes.sort((a, b) => a.id > b.id ? -1 : 1); diff --git a/packages/backend/src/server/api/endpoints/users/reactions.ts b/packages/backend/src/server/api/endpoints/users/reactions.ts index 56f59bd285..553787ad58 100644 --- a/packages/backend/src/server/api/endpoints/users/reactions.ts +++ b/packages/backend/src/server/api/endpoints/users/reactions.ts @@ -105,10 +105,15 @@ export default class extends Endpoint { // eslint- const query = this.queryService.makePaginationQuery(this.noteReactionsRepository.createQueryBuilder('reaction'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) .andWhere('reaction.userId = :userId', { userId: ps.userId }) - .leftJoinAndSelect('reaction.note', 'note'); + .innerJoinAndSelect('reaction.note', 'note'); this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedHostQueryForNote(query); + if (me) { + this.queryService.generateMutedUserQueryForNotes(query, me); + this.queryService.generateBlockedUserQueryForNotes(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + } const reactions = (await query .limit(ps.limit) From 139f458c0b9438029056d73b59c82f985488dd48 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 24 May 2025 23:15:15 -0400 Subject: [PATCH 13/29] fix following feed performance and bugs --- .../src/server/api/endpoints/notes/following.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/notes/following.ts b/packages/backend/src/server/api/endpoints/notes/following.ts index 5f6ee9f903..088b172ba4 100644 --- a/packages/backend/src/server/api/endpoints/notes/following.ts +++ b/packages/backend/src/server/api/endpoints/notes/following.ts @@ -4,7 +4,7 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import { ObjectLiteral, SelectQueryBuilder } from 'typeorm'; +import { IsNull, ObjectLiteral, SelectQueryBuilder } from 'typeorm'; import { SkLatestNote, MiFollowing } from '@/models/_.js'; import type { NotesRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; @@ -130,7 +130,9 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel') + + // Exclude channel notes + .andWhere({ channelId: IsNull() }) ; // Limit to files, if requested @@ -145,11 +147,10 @@ export default class extends Endpoint { // eslint- // Hide blocked users / instances query.andWhere('"user"."isSuspended" = false'); - query.andWhere('("replyUser" IS NULL OR "replyUser"."isSuspended" = false)'); - query.andWhere('("renoteUser" IS NULL OR "renoteUser"."isSuspended" = false)'); this.queryService.generateBlockedHostQueryForNote(query); - // Respect blocks and mutes + // Respect blocks, mutes, and privacy + this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); this.queryService.generateMutedUserQueryForNotes(query, me); @@ -161,7 +162,7 @@ export default class extends Endpoint { // eslint- // Query and return the next page const notes = await query.getMany(); - return await this.noteEntityService.packMany(notes, me); + return await this.noteEntityService.packMany(notes, me, { skipHide: true }); }); } } From 08834f1722c43434166eaf06cb0eea6d9dd19634 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 24 May 2025 23:48:36 -0400 Subject: [PATCH 14/29] register instances before creating a user --- .../backend/src/core/activitypub/models/ApPersonService.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 5c6716a0b8..3ca9c93806 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -398,6 +398,9 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const followerscollection = await _resolver.resolveCollection(person.followers!).catch(() => { return null; }); const followingcollection = await _resolver.resolveCollection(person.following!).catch(() => { return null; }); + // Register the instance first, to avoid FK errors + await this.federatedInstanceService.fetchOrRegister(host); + try { // Start transaction await this.db.transaction(async transactionalEntityManager => { From c0ead9cf11471a07d0906d90fcbe1d17b09862f5 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 25 May 2025 00:16:16 -0400 Subject: [PATCH 15/29] remove broken HTTP users before running add_instance_foreign_keys migration --- .../migration/1748128176881-add_instance_foreign_keys.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/backend/migration/1748128176881-add_instance_foreign_keys.js b/packages/backend/migration/1748128176881-add_instance_foreign_keys.js index da2de14f18..9a970699a0 100644 --- a/packages/backend/migration/1748128176881-add_instance_foreign_keys.js +++ b/packages/backend/migration/1748128176881-add_instance_foreign_keys.js @@ -15,6 +15,11 @@ export class AddInstanceForeignKeys1748128176881 { name = 'AddInstanceForeignKeys1748128176881' async up(queryRunner) { + // Fix-up: Sharkey versions in early-mid 2025 could federate with HTTP URLs, which would produce a user with no matching instance. + // These users are fundamentally broken and can just be removed, which ensures that the FK can create without conflicts. + // But we must also *preserve* those with a matching registered instance, as FireFish allowed federation over HTTP and some older instances may have fully-populated users. + await queryRunner.query(`DELETE FROM "user" WHERE "uri" LIKE 'http:%' AND NOT EXISTS (select 1 from "instance" where "instance"."host" = "user"."host")`); + await queryRunner.query(`ALTER TABLE "user" ADD CONSTRAINT "FK_user_host" FOREIGN KEY ("host") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_userHost" FOREIGN KEY ("userHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_replyUserHost" FOREIGN KEY ("replyUserHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); From 3e7ab07b3caa91c9caff7aeb072cf26f2e13ffc9 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 25 May 2025 08:44:45 -0400 Subject: [PATCH 16/29] avoid race conditions in meta / instance insert --- .../src/core/FederatedInstanceService.ts | 30 +++++------- packages/backend/src/core/MetaService.ts | 48 +++++++++---------- 2 files changed, 37 insertions(+), 41 deletions(-) diff --git a/packages/backend/src/core/FederatedInstanceService.ts b/packages/backend/src/core/FederatedInstanceService.ts index e31d802e8c..662c7f7f2d 100644 --- a/packages/backend/src/core/FederatedInstanceService.ts +++ b/packages/backend/src/core/FederatedInstanceService.ts @@ -5,15 +5,13 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Redis from 'ioredis'; -import { DataSource, QueryFailedError } from 'typeorm'; import type { InstancesRepository } from '@/models/_.js'; -import { MiInstance } from '@/models/Instance.js'; +import type { MiInstance } from '@/models/Instance.js'; import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; import { IdService } from '@/core/IdService.js'; import { DI } from '@/di-symbols.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; -import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; @Injectable() export class FederatedInstanceService implements OnApplicationShutdown { @@ -26,9 +24,6 @@ export class FederatedInstanceService implements OnApplicationShutdown { @Inject(DI.instancesRepository) private instancesRepository: InstancesRepository, - @Inject(DI.db) - private readonly db: DataSource, - private utilityService: UtilityService, private idService: IdService, ) { @@ -58,11 +53,11 @@ export class FederatedInstanceService implements OnApplicationShutdown { const cached = await this.federatedInstanceCache.get(host); if (cached) return cached; - return await this.db.transaction(async tem => { - let index = await tem.findOneBy(MiInstance, { host }); - - if (index == null) { - await tem.insert(MiInstance, { + let index = await this.instancesRepository.findOneBy({ host }); + if (index == null) { + await this.instancesRepository.createQueryBuilder('instance') + .insert() + .values({ id: this.idService.gen(), host, firstRetrievedAt: new Date(), @@ -71,14 +66,15 @@ export class FederatedInstanceService implements OnApplicationShutdown { isMediaSilenced: this.utilityService.isMediaSilencedHost(host), isAllowListed: this.utilityService.isAllowListedHost(host), isBubbled: this.utilityService.isBubbledHost(host), - }); + }) + .orIgnore() + .execute(); - index = await tem.findOneByOrFail(MiInstance, { host }); - } + index = await this.instancesRepository.findOneByOrFail({ host }); + } - await this.federatedInstanceCache.set(host, index); - return index; - }); + await this.federatedInstanceCache.set(host, index); + return index; } @bindThis diff --git a/packages/backend/src/core/MetaService.ts b/packages/backend/src/core/MetaService.ts index 5b6ee8920e..b4ccfec4cc 100644 --- a/packages/backend/src/core/MetaService.ts +++ b/packages/backend/src/core/MetaService.ts @@ -14,6 +14,7 @@ import type { GlobalEvents } from '@/core/GlobalEventService.js'; import { FeaturedService } from '@/core/FeaturedService.js'; import { MiInstance } from '@/models/Instance.js'; import { diffArrays } from '@/misc/diff-arrays.js'; +import type { MetasRepository } from '@/models/_.js'; import type { OnApplicationShutdown } from '@nestjs/common'; @Injectable() @@ -28,6 +29,9 @@ export class MetaService implements OnApplicationShutdown { @Inject(DI.db) private db: DataSource, + @Inject(DI.metasRepository) + private readonly metasRepository: MetasRepository, + private featuredService: FeaturedService, private globalEventService: GlobalEventService, ) { @@ -69,35 +73,31 @@ export class MetaService implements OnApplicationShutdown { public async fetch(noCache = false): Promise { if (!noCache && this.cache) return this.cache; - return await this.db.transaction(async transactionalEntityManager => { - // 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する - const metas = await transactionalEntityManager.find(MiMeta, { + // 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する + let meta = await this.metasRepository.findOne({ + order: { + id: 'DESC', + }, + }); + + if (!meta) { + await this.metasRepository.createQueryBuilder('meta') + .insert() + .values({ + id: 'x', + }) + .orIgnore() + .execute(); + + meta = await this.metasRepository.findOneOrFail({ order: { id: 'DESC', }, }); + } - const meta = metas[0]; - - if (meta) { - this.cache = meta; - return meta; - } else { - // metaが空のときfetchMetaが同時に呼ばれるとここが同時に呼ばれてしまうことがあるのでフェイルセーフなupsertを使う - const saved = await transactionalEntityManager - .upsert( - MiMeta, - { - id: 'x', - }, - ['id'], - ) - .then((x) => transactionalEntityManager.findOneByOrFail(MiMeta, x.identifiers[0])); - - this.cache = saved; - return saved; - } - }); + this.cache = meta; + return meta; } @bindThis From 7385f30903111bbedc2ed8f83f40a63407edcd08 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 25 May 2025 12:16:34 -0400 Subject: [PATCH 17/29] clear federatedInstanceCache when meta host lists change --- .../src/core/FederatedInstanceService.ts | 71 ++++++++++++------- .../src/core/entities/UserEntityService.ts | 2 +- packages/backend/src/misc/cache.ts | 9 +++ 3 files changed, 56 insertions(+), 26 deletions(-) diff --git a/packages/backend/src/core/FederatedInstanceService.ts b/packages/backend/src/core/FederatedInstanceService.ts index 662c7f7f2d..eb634b1d68 100644 --- a/packages/backend/src/core/FederatedInstanceService.ts +++ b/packages/backend/src/core/FederatedInstanceService.ts @@ -5,21 +5,24 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Redis from 'ioredis'; -import type { InstancesRepository } from '@/models/_.js'; +import type { InstancesRepository, MiMeta } from '@/models/_.js'; import type { MiInstance } from '@/models/Instance.js'; -import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; +import { MemoryKVCache } from '@/misc/cache.js'; import { IdService } from '@/core/IdService.js'; import { DI } from '@/di-symbols.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; +import type { GlobalEvents } from '@/core/GlobalEventService.js'; +import { Serialized } from '@/types.js'; +import { diffArrays } from '@/misc/diff-arrays.js'; @Injectable() export class FederatedInstanceService implements OnApplicationShutdown { - public federatedInstanceCache: RedisKVCache; + private readonly federatedInstanceCache: MemoryKVCache; constructor( - @Inject(DI.redis) - private redisClient: Redis.Redis, + @Inject(DI.redisForSub) + private redisForSub: Redis.Redis, @Inject(DI.instancesRepository) private instancesRepository: InstancesRepository, @@ -27,30 +30,15 @@ export class FederatedInstanceService implements OnApplicationShutdown { private utilityService: UtilityService, private idService: IdService, ) { - this.federatedInstanceCache = new RedisKVCache(this.redisClient, 'federatedInstance', { - lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60 * 3, // 3m - fetcher: (key) => this.instancesRepository.findOneBy({ host: key }), - toRedisConverter: (value) => JSON.stringify(value), - fromRedisConverter: (value) => { - const parsed = JSON.parse(value); - if (parsed == null) return null; - return { - ...parsed, - firstRetrievedAt: new Date(parsed.firstRetrievedAt), - latestRequestReceivedAt: parsed.latestRequestReceivedAt ? new Date(parsed.latestRequestReceivedAt) : null, - infoUpdatedAt: parsed.infoUpdatedAt ? new Date(parsed.infoUpdatedAt) : null, - notRespondingSince: parsed.notRespondingSince ? new Date(parsed.notRespondingSince) : null, - }; - }, - }); + this.federatedInstanceCache = new MemoryKVCache(1000 * 60 * 3); // 3m + this.redisForSub.on('message', this.onMessage); } @bindThis public async fetchOrRegister(host: string): Promise { host = this.utilityService.toPuny(host); - const cached = await this.federatedInstanceCache.get(host); + const cached = this.federatedInstanceCache.get(host); if (cached) return cached; let index = await this.instancesRepository.findOneBy({ host }); @@ -73,7 +61,7 @@ export class FederatedInstanceService implements OnApplicationShutdown { index = await this.instancesRepository.findOneByOrFail({ host }); } - await this.federatedInstanceCache.set(host, index); + this.federatedInstanceCache.set(host, index); return index; } @@ -81,7 +69,7 @@ export class FederatedInstanceService implements OnApplicationShutdown { public async fetch(host: string): Promise { host = this.utilityService.toPuny(host); - const cached = await this.federatedInstanceCache.get(host); + const cached = this.federatedInstanceCache.get(host); if (cached !== undefined) return cached; const index = await this.instancesRepository.findOneBy({ host }); @@ -109,8 +97,35 @@ export class FederatedInstanceService implements OnApplicationShutdown { this.federatedInstanceCache.set(result.host, result); } + private syncCache(before: Serialized, after: Serialized): void { + const changed = + hasDiff(before?.blockedHosts, after.blockedHosts) || + hasDiff(before?.silencedHosts, after.silencedHosts) || + hasDiff(before?.mediaSilencedHosts, after.mediaSilencedHosts) || + hasDiff(before?.federationHosts, after.federationHosts) || + hasDiff(before?.bubbleInstances, after.bubbleInstances); + + if (changed) { + // We have to clear the whole thing, otherwise subdomains won't be synced. + this.federatedInstanceCache.clear(); + } + } + + @bindThis + private async onMessage(_: string, data: string): Promise { + const obj = JSON.parse(data); + + if (obj.channel === 'internal') { + const { type, body } = obj.message as GlobalEvents['internal']['payload']; + if (type === 'metaUpdated') { + this.syncCache(body.before, body.after); + } + } + } + @bindThis public dispose(): void { + this.redisForSub.off('message', this.onMessage); this.federatedInstanceCache.dispose(); } @@ -119,3 +134,9 @@ export class FederatedInstanceService implements OnApplicationShutdown { this.dispose(); } } + +function hasDiff(before: string[] | null | undefined, after: string[] | null | undefined): boolean { + const { added, removed } = diffArrays(before, after); + return added.length > 0 || removed.length > 0; +} + diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 56506a5fa4..feddb8fa94 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -609,7 +609,7 @@ export class UserEntityService implements OnModuleInit { requireSigninToViewContents: user.requireSigninToViewContents === false ? undefined : true, makeNotesFollowersOnlyBefore: user.makeNotesFollowersOnlyBefore ?? undefined, makeNotesHiddenBefore: user.makeNotesHiddenBefore ?? undefined, - instance: user.host ? this.federatedInstanceService.federatedInstanceCache.fetch(user.host).then(instance => instance ? { + instance: user.host ? this.federatedInstanceService.fetch(user.host).then(instance => instance ? { name: instance.name, softwareName: instance.softwareName, softwareVersion: instance.softwareVersion, diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 48b8f43678..a6ab96c189 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -308,8 +308,17 @@ export class MemoryKVCache { } } + /** + * Removes all entries from the cache, but does not dispose it. + */ + @bindThis + public clear(): void { + this.cache.clear(); + } + @bindThis public dispose(): void { + this.clear(); clearInterval(this.gcIntervalHandle); } From 788e59f046f80cb16c0098052722738109eea772 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 25 May 2025 12:17:11 -0400 Subject: [PATCH 18/29] fix type errors caused by new User, Note, and Instance fields --- packages/backend/src/core/ReversiService.ts | 2 ++ packages/backend/src/core/WebhookTestService.ts | 5 +++++ packages/backend/test/unit/NoteCreateService.ts | 3 +++ packages/backend/test/unit/misc/is-renote.ts | 3 +++ 4 files changed, 13 insertions(+) diff --git a/packages/backend/src/core/ReversiService.ts b/packages/backend/src/core/ReversiService.ts index 8c0a8f6cc7..e31d9e5b1a 100644 --- a/packages/backend/src/core/ReversiService.ts +++ b/packages/backend/src/core/ReversiService.ts @@ -587,6 +587,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { lastActiveDate: parsed.user1.lastActiveDate != null ? new Date(parsed.user1.lastActiveDate) : null, lastFetchedAt: parsed.user1.lastFetchedAt != null ? new Date(parsed.user1.lastFetchedAt) : null, movedAt: parsed.user1.movedAt != null ? new Date(parsed.user1.movedAt) : null, + instance: null, } : null, user2: parsed.user2 != null ? { ...parsed.user2, @@ -597,6 +598,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { lastActiveDate: parsed.user2.lastActiveDate != null ? new Date(parsed.user2.lastActiveDate) : null, lastFetchedAt: parsed.user2.lastFetchedAt != null ? new Date(parsed.user2.lastFetchedAt) : null, movedAt: parsed.user2.movedAt != null ? new Date(parsed.user2.movedAt) : null, + instance: null, } : null, }; } else { diff --git a/packages/backend/src/core/WebhookTestService.ts b/packages/backend/src/core/WebhookTestService.ts index 2f8cfea7f7..afd011c410 100644 --- a/packages/backend/src/core/WebhookTestService.ts +++ b/packages/backend/src/core/WebhookTestService.ts @@ -63,6 +63,7 @@ function generateDummyUser(override?: Partial): MiUser { emojis: [], score: 0, host: null, + instance: null, inbox: null, sharedInbox: null, featured: null, @@ -114,10 +115,13 @@ function generateDummyNote(override?: Partial): MiNote { channelId: null, channel: null, userHost: null, + userInstance: null, replyUserId: null, replyUserHost: null, + replyUserInstance: null, renoteUserId: null, renoteUserHost: null, + renoteUserInstance: null, updatedAt: null, processErrors: [], ...override, @@ -449,6 +453,7 @@ export class WebhookTestService { isAdmin: false, isModerator: false, isSystem: false, + instance: undefined, ...override, }; } diff --git a/packages/backend/test/unit/NoteCreateService.ts b/packages/backend/test/unit/NoteCreateService.ts index f4ecfef34d..63e3795a84 100644 --- a/packages/backend/test/unit/NoteCreateService.ts +++ b/packages/backend/test/unit/NoteCreateService.ts @@ -57,10 +57,13 @@ describe('NoteCreateService', () => { channelId: null, channel: null, userHost: null, + userInstance: null, replyUserId: null, replyUserHost: null, + replyUserInstance: null, renoteUserId: null, renoteUserHost: null, + renoteUserInstance: null, processErrors: [], }; diff --git a/packages/backend/test/unit/misc/is-renote.ts b/packages/backend/test/unit/misc/is-renote.ts index 24cd2236bb..b6cfa53466 100644 --- a/packages/backend/test/unit/misc/is-renote.ts +++ b/packages/backend/test/unit/misc/is-renote.ts @@ -40,10 +40,13 @@ const base: MiNote = { channelId: null, channel: null, userHost: null, + userInstance: null, replyUserId: null, replyUserHost: null, + replyUserInstance: null, renoteUserId: null, renoteUserHost: null, + renoteUserInstance: null, processErrors: [], }; From 6b3ddc676815e65aee0be6104686fbf6866d176d Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 25 May 2025 12:17:57 -0400 Subject: [PATCH 19/29] add TS caches to gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index b07d195a3f..ea761882da 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,7 @@ vite.config.local-dev.ts.timestamp-* # Sharkey /packages/megalodon/lib + +# TypeScript +.tsbuildinfo +*.tsbuildinfo From 8a2ed3bc86e9531038686d911c3b0975272c911f Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 25 May 2025 12:33:05 -0400 Subject: [PATCH 20/29] minor optimization to diff-arrays --- packages/backend/src/misc/diff-arrays.ts | 6 ++++-- packages/backend/test/unit/misc/diff-arrays.ts | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/misc/diff-arrays.ts b/packages/backend/src/misc/diff-arrays.ts index 1f6820481b..b3879cc996 100644 --- a/packages/backend/src/misc/diff-arrays.ts +++ b/packages/backend/src/misc/diff-arrays.ts @@ -10,8 +10,9 @@ export interface DiffResult { /** * Calculates the difference between two snapshots of data. - * Null, undefined, and empty arrays are supported, and values do not have to be unique. + * Null, undefined, and empty arrays are supported, and duplicate values are ignored. * Result sets are de-duplicated, and will be empty if no data was added or removed (respectively). + * The inputs are treated as un-ordered, so a re-ordering of the same data will NOT be considered a change. * @param dataBefore Array containing data before the change * @param dataAfter Array containing data after the change */ @@ -26,7 +27,8 @@ export function diffArrays(dataBefore: T[] | null | undefined, dataAfter: T[] for (const host of before) { // before and NOT after => removed - if (!after.has(host)) { + // delete operation removes duplicates to speed up the "after" loop + if (!after.delete(host)) { removed.push(host); } } diff --git a/packages/backend/test/unit/misc/diff-arrays.ts b/packages/backend/test/unit/misc/diff-arrays.ts index 3eed5a8913..a2dee50652 100644 --- a/packages/backend/test/unit/misc/diff-arrays.ts +++ b/packages/backend/test/unit/misc/diff-arrays.ts @@ -18,19 +18,19 @@ describe(diffArrays, () => { expect(result.removed).toHaveLength(0); }); - it('should remove before when added is empty', () => { + it('should remove before when after is empty', () => { const result = diffArrays([1, 2, 3], []); expect(result.added).toHaveLength(0); expect(result.removed).toEqual([1, 2, 3]); }); - it('should deduplicate before when added is empty', () => { + it('should deduplicate before when after is empty', () => { const result = diffArrays([1, 1, 2, 2, 3], []); expect(result.added).toHaveLength(0); expect(result.removed).toEqual([1, 2, 3]); }); - it('should remove after when before is empty', () => { + it('should add after when before is empty', () => { const result = diffArrays([], [1, 2, 3]); expect(result.added).toEqual([1, 2, 3]); expect(result.removed).toHaveLength(0); From 35dfde838be2dd50ee37d420f87a0cac917e621c Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 25 May 2025 12:34:09 -0400 Subject: [PATCH 21/29] add function diffArraysSimple for more efficient change detection --- .../src/core/FederatedInstanceService.ts | 18 +++----- packages/backend/src/misc/diff-arrays.ts | 44 +++++++++++++++++++ .../backend/test/unit/misc/diff-arrays.ts | 40 ++++++++++++++++- 3 files changed, 89 insertions(+), 13 deletions(-) diff --git a/packages/backend/src/core/FederatedInstanceService.ts b/packages/backend/src/core/FederatedInstanceService.ts index eb634b1d68..34df10f0ff 100644 --- a/packages/backend/src/core/FederatedInstanceService.ts +++ b/packages/backend/src/core/FederatedInstanceService.ts @@ -14,7 +14,7 @@ import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; import { Serialized } from '@/types.js'; -import { diffArrays } from '@/misc/diff-arrays.js'; +import { diffArrays, diffArraysSimple } from '@/misc/diff-arrays.js'; @Injectable() export class FederatedInstanceService implements OnApplicationShutdown { @@ -99,11 +99,11 @@ export class FederatedInstanceService implements OnApplicationShutdown { private syncCache(before: Serialized, after: Serialized): void { const changed = - hasDiff(before?.blockedHosts, after.blockedHosts) || - hasDiff(before?.silencedHosts, after.silencedHosts) || - hasDiff(before?.mediaSilencedHosts, after.mediaSilencedHosts) || - hasDiff(before?.federationHosts, after.federationHosts) || - hasDiff(before?.bubbleInstances, after.bubbleInstances); + diffArraysSimple(before?.blockedHosts, after.blockedHosts) || + diffArraysSimple(before?.silencedHosts, after.silencedHosts) || + diffArraysSimple(before?.mediaSilencedHosts, after.mediaSilencedHosts) || + diffArraysSimple(before?.federationHosts, after.federationHosts) || + diffArraysSimple(before?.bubbleInstances, after.bubbleInstances); if (changed) { // We have to clear the whole thing, otherwise subdomains won't be synced. @@ -134,9 +134,3 @@ export class FederatedInstanceService implements OnApplicationShutdown { this.dispose(); } } - -function hasDiff(before: string[] | null | undefined, after: string[] | null | undefined): boolean { - const { added, removed } = diffArrays(before, after); - return added.length > 0 || removed.length > 0; -} - diff --git a/packages/backend/src/misc/diff-arrays.ts b/packages/backend/src/misc/diff-arrays.ts index b3879cc996..b50ca1d4f7 100644 --- a/packages/backend/src/misc/diff-arrays.ts +++ b/packages/backend/src/misc/diff-arrays.ts @@ -56,3 +56,47 @@ export function diffArrays(dataBefore: T[] | null | undefined, dataAfter: T[] // data NEITHER before nor after => no change return { added: [], removed: [] }; } + +/** + * Checks for any difference between two snapshots of data. + * Null, undefined, and empty arrays are supported, and duplicate values are ignored. + * The inputs are treated as un-ordered, so a re-ordering of the same data will NOT be considered a change. + * @param dataBefore Array containing data before the change + * @param dataAfter Array containing data after the change + */ +export function diffArraysSimple(dataBefore: T[] | null | undefined, dataAfter: T[] | null | undefined): boolean { + const before = dataBefore ? new Set(dataBefore) : null; + const after = dataAfter ? new Set(dataAfter) : null; + + if (before?.size && after?.size) { + // different size => changed + if (before.size !== after.size) return true; + + // removed => changed + for (const host of before) { + // delete operation removes duplicates to speed up the "after" loop + if (!after.delete(host)) { + return true; + } + } + + // added => changed + for (const host of after) { + if (!before.has(host)) { + return true; + } + } + + // identical values => no change + return false; + } + + // before and NOT after => change + if (before?.size) return true; + + // after and NOT before => change + if (after?.size) return true; + + // NEITHER before nor after => no change + return false; +} diff --git a/packages/backend/test/unit/misc/diff-arrays.ts b/packages/backend/test/unit/misc/diff-arrays.ts index a2dee50652..b6db5e2eca 100644 --- a/packages/backend/test/unit/misc/diff-arrays.ts +++ b/packages/backend/test/unit/misc/diff-arrays.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { diffArrays } from '@/misc/diff-arrays.js'; +import { diffArrays, diffArraysSimple } from '@/misc/diff-arrays.js'; describe(diffArrays, () => { it('should return empty result when both inputs are null', () => { @@ -51,3 +51,41 @@ describe(diffArrays, () => { expect(result.removed).toEqual(['b', 'd']); }); }); + +describe(diffArraysSimple, () => { + it('should return false when both inputs are null', () => { + const result = diffArraysSimple(null, null); + expect(result).toBe(false); + }); + + it('should return false when both inputs are empty', () => { + const result = diffArraysSimple([], []); + expect(result).toBe(false); + }); + + it('should return true when before is populated and after is empty', () => { + const result = diffArraysSimple([1, 2, 3], []); + expect(result).toBe(true); + }); + + it('should return true when before is empty and after is populated', () => { + const result = diffArraysSimple([], [1, 2, 3]); + expect(result).toBe(true); + }); + + it('should return true when values have changed', () => { + const result = diffArraysSimple( + ['a', 'a', 'b', 'c'], + ['a', 'b', 'c', 'd'], + ); + expect(result).toBe(true); + }); + + it('should return false when values have not changed', () => { + const result = diffArraysSimple( + ['a', 'a', 'b', 'c'], + ['a', 'b', 'c', 'c'], + ); + expect(result).toBe(false); + }); +}); From ab96402c2ac72e74110a0449976bd639b8c223ab Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 25 May 2025 12:43:25 -0400 Subject: [PATCH 22/29] add backend npm script to roll back a migration --- packages/backend/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend/package.json b/packages/backend/package.json index b9cb0002ab..7810db2079 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -10,6 +10,7 @@ "start": "node ./built/boot/entry.js", "start:test": "cross-env NODE_ENV=test node ./built/boot/entry.js", "migrate": "pnpm typeorm migration:run -d ormconfig.js", + "migrate:revert": "pnpm typeorm migration:revert -d ormconfig.js", "revert": "pnpm typeorm migration:revert -d ormconfig.js", "check:connect": "node ./scripts/check_connect.js", "build": "swc src -d built -D --strip-leading-paths", From 9e282b1d1086409e28c4a3798d8b1f5a9688d65e Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 25 May 2025 12:43:43 -0400 Subject: [PATCH 23/29] fix arrays in migration add_instance_block_columns --- ...748105111513-add_instance_block_columns.js | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/packages/backend/migration/1748105111513-add_instance_block_columns.js b/packages/backend/migration/1748105111513-add_instance_block_columns.js index c9087604e5..6e3d78d5e8 100644 --- a/packages/backend/migration/1748105111513-add_instance_block_columns.js +++ b/packages/backend/migration/1748105111513-add_instance_block_columns.js @@ -37,32 +37,32 @@ export class AddInstanceBlockColumns1748105111513 { // Blocked hosts if (meta.blockedHosts.length > 0) { - const pattern = buildPatterns(meta.blockedHosts); - await queryRunner.query(`UPDATE "instance" SET "isBlocked" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ${pattern}`); + const patterns = buildPatterns(meta.blockedHosts); + await queryRunner.query(`UPDATE "instance" SET "isBlocked" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]); } // Silenced hosts if (meta.silencedHosts.length > 0) { - const pattern = buildPatterns(meta.silencedHosts); - await queryRunner.query(`UPDATE "instance" SET "isSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ${pattern}`); + const patterns = buildPatterns(meta.silencedHosts); + await queryRunner.query(`UPDATE "instance" SET "isSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]); } // Media silenced hosts if (meta.mediaSilencedHosts.length > 0) { - const pattern = buildPatterns(meta.mediaSilencedHosts); - await queryRunner.query(`UPDATE "instance" SET "isMediaSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ${pattern}`); + const patterns = buildPatterns(meta.mediaSilencedHosts); + await queryRunner.query(`UPDATE "instance" SET "isMediaSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]); } // Allow-listed hosts if (meta.federationHosts.length > 0) { - const pattern = buildPatterns(meta.federationHosts); - await queryRunner.query(`UPDATE "instance" SET "isAllowListed" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ${pattern}`); + const patterns = buildPatterns(meta.federationHosts); + await queryRunner.query(`UPDATE "instance" SET "isAllowListed" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]); } // Bubbled hosts if (meta.bubbleInstances.length > 0) { - const pattern = buildPatterns(meta.bubbleInstances); - await queryRunner.query(`UPDATE "instance" SET "isBubbled" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ${pattern}`); + const patterns = buildPatterns(meta.bubbleInstances); + await queryRunner.query(`UPDATE "instance" SET "isBubbled" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]); } } } @@ -78,12 +78,8 @@ export class AddInstanceBlockColumns1748105111513 { /** * @param {string[]} input - * @returns {string} + * @returns {string[]} */ function buildPatterns(input) { - const strings = input - .map(i => i.toLowerCase().split('').reverse().join('') + '.%') - .map(i => `'${i}'`) - .join(', '); - return `(array[${strings}]::text[])`; + return input.map(i => i.toLowerCase().split('').reverse().join('') + '.%'); } From 12c2bbc7debb5d3b8e1de481c28ea26767ed4179 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 25 May 2025 12:50:47 -0400 Subject: [PATCH 24/29] add npm scripts to create or generate a migration --- packages/backend/package.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/backend/package.json b/packages/backend/package.json index 7810db2079..bad6990ba5 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -11,6 +11,8 @@ "start:test": "cross-env NODE_ENV=test node ./built/boot/entry.js", "migrate": "pnpm typeorm migration:run -d ormconfig.js", "migrate:revert": "pnpm typeorm migration:revert -d ormconfig.js", + "migrate:generate": "pnpm typeorm migration:generate -d ormconfig.js", + "migrate:create": "pnpm typeorm migration:create", "revert": "pnpm typeorm migration:revert -d ormconfig.js", "check:connect": "node ./scripts/check_connect.js", "build": "swc src -d built -D --strip-leading-paths", From 070084889fd71eb7b77aaf0831406b05282b0b97 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 25 May 2025 12:51:00 -0400 Subject: [PATCH 25/29] re-analyze all tables affected by new indexes --- ...48104955717-index_IDX_instance_host_key.js | 4 --- ...51-analyze_instance-user-note-following.js | 25 +++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 packages/backend/migration/1748191631151-analyze_instance-user-note-following.js diff --git a/packages/backend/migration/1748104955717-index_IDX_instance_host_key.js b/packages/backend/migration/1748104955717-index_IDX_instance_host_key.js index 3c3b779298..139eae740f 100644 --- a/packages/backend/migration/1748104955717-index_IDX_instance_host_key.js +++ b/packages/backend/migration/1748104955717-index_IDX_instance_host_key.js @@ -8,10 +8,6 @@ export class IndexIDXInstanceHostKey1748104955717 { async up(queryRunner) { await queryRunner.query(`CREATE UNIQUE INDEX "IDX_instance_host_key" ON "instance" (((lower(reverse("host")) || '.')::text) text_pattern_ops)`); - - // Flush all cached Linear Scan Plans and redo statistics for expression index - // this is important for Postgres to learn that even in highly complex queries, using this index first can reduce the result set significantly - await queryRunner.query(`ANALYZE "instance"`); } async down(queryRunner) { diff --git a/packages/backend/migration/1748191631151-analyze_instance-user-note-following.js b/packages/backend/migration/1748191631151-analyze_instance-user-note-following.js new file mode 100644 index 0000000000..f03a60980b --- /dev/null +++ b/packages/backend/migration/1748191631151-analyze_instance-user-note-following.js @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + */ + +/** + * @class + * @implements {MigrationInterface} + */ +export class AnalyzeInstanceUserNoteFollowing1748191631151 { + name = 'AnalyzeInstanceUserNoteFollowing1748191631151' + + async up(queryRunner) { + // Refresh statistics for tables impacted by new indexes. + // This helps the query planner to efficiently use them without waiting for the next full vacuum. + await queryRunner.query(`ANALYZE "instance", "user", "following", "note"`); + } + + async down(queryRunner) { + } +} From 4738b14d1ca76c29f71f3665c7a1b7658835404a Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 25 May 2025 18:42:47 -0400 Subject: [PATCH 26/29] fix TypeORM error from MetaService.fetch --- packages/backend/src/core/MetaService.ts | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/core/MetaService.ts b/packages/backend/src/core/MetaService.ts index b4ccfec4cc..07f82dc23e 100644 --- a/packages/backend/src/core/MetaService.ts +++ b/packages/backend/src/core/MetaService.ts @@ -74,11 +74,13 @@ export class MetaService implements OnApplicationShutdown { if (!noCache && this.cache) return this.cache; // 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する - let meta = await this.metasRepository.findOne({ - order: { + let meta = await this.metasRepository.createQueryBuilder('meta') + .select() + .orderBy({ id: 'DESC', - }, - }); + }) + .limit(1) + .getOne(); if (!meta) { await this.metasRepository.createQueryBuilder('meta') @@ -89,11 +91,13 @@ export class MetaService implements OnApplicationShutdown { .orIgnore() .execute(); - meta = await this.metasRepository.findOneOrFail({ - order: { + meta = await this.metasRepository.createQueryBuilder('meta') + .select() + .orderBy({ id: 'DESC', - }, - }); + }) + .limit(1) + .getOneOrFail(); } this.cache = meta; From 0912a8bb10fdb0275ce4ec65b7d6ebf3c86aee24 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 27 May 2025 20:22:22 -0400 Subject: [PATCH 27/29] fix tests --- packages/backend/test/unit/MetaService.ts | 11 ++++++----- packages/backend/test/unit/RoleService.ts | 16 ++++++++++++++++ .../backend/test/unit/UserSearchService.ts | 18 +++++++++++++++++- packages/backend/test/unit/activitypub.ts | 19 +++++++++++++++++++ 4 files changed, 58 insertions(+), 6 deletions(-) diff --git a/packages/backend/test/unit/MetaService.ts b/packages/backend/test/unit/MetaService.ts index 19c98eab3d..056838e180 100644 --- a/packages/backend/test/unit/MetaService.ts +++ b/packages/backend/test/unit/MetaService.ts @@ -11,6 +11,7 @@ import { GlobalModule } from '@/GlobalModule.js'; import { DI } from '@/di-symbols.js'; import { MetaService } from '@/core/MetaService.js'; import { CoreModule } from '@/core/CoreModule.js'; +import { MetasRepository } from '@/models/_.js'; import type { TestingModule } from '@nestjs/testing'; import type { DataSource } from 'typeorm'; @@ -39,8 +40,8 @@ describe('MetaService', () => { }); test('fetch (cache)', async () => { - const db = app.get(DI.db); - const spy = jest.spyOn(db, 'transaction'); + const metasRepository = app.get(DI.metasRepository); + const spy = jest.spyOn(metasRepository, 'createQueryBuilder'); const result = await metaService.fetch(); @@ -49,12 +50,12 @@ describe('MetaService', () => { }); test('fetch (force)', async () => { - const db = app.get(DI.db); - const spy = jest.spyOn(db, 'transaction'); + const metasRepository = app.get(DI.metasRepository); + const spy = jest.spyOn(metasRepository, 'createQueryBuilder'); const result = await metaService.fetch(true); expect(result.id).toBe('x'); - expect(spy).toHaveBeenCalledTimes(1); + expect(spy).toHaveBeenCalled(); }); }); diff --git a/packages/backend/test/unit/RoleService.ts b/packages/backend/test/unit/RoleService.ts index 553ff0982a..839402418e 100644 --- a/packages/backend/test/unit/RoleService.ts +++ b/packages/backend/test/unit/RoleService.ts @@ -15,6 +15,7 @@ import type { MockFunctionMetadata } from 'jest-mock'; import { GlobalModule } from '@/GlobalModule.js'; import { RoleService } from '@/core/RoleService.js'; import { + InstancesRepository, MiMeta, MiRole, MiRoleAssignment, @@ -39,6 +40,7 @@ const moduleMocker = new ModuleMocker(global); describe('RoleService', () => { let app: TestingModule; let roleService: RoleService; + let instancesRepository: InstancesRepository; let usersRepository: UsersRepository; let rolesRepository: RolesRepository; let roleAssignmentsRepository: RoleAssignmentsRepository; @@ -47,6 +49,19 @@ describe('RoleService', () => { let clock: lolex.InstalledClock; async function createUser(data: Partial = {}) { + if (data.host != null) { + await instancesRepository + .createQueryBuilder('instance') + .insert() + .values({ + id: genAidx(Date.now()), + firstRetrievedAt: new Date(), + host: data.host, + }) + .orIgnore() + .execute(); + } + const un = secureRndstr(16); const x = await usersRepository.insert({ id: genAidx(Date.now()), @@ -145,6 +160,7 @@ describe('RoleService', () => { app.enableShutdownHooks(); roleService = app.get(RoleService); + instancesRepository = app.get(DI.instancesRepository); usersRepository = app.get(DI.usersRepository); rolesRepository = app.get(DI.rolesRepository); roleAssignmentsRepository = app.get(DI.roleAssignmentsRepository); diff --git a/packages/backend/test/unit/UserSearchService.ts b/packages/backend/test/unit/UserSearchService.ts index 697425beb8..a6b331d1cb 100644 --- a/packages/backend/test/unit/UserSearchService.ts +++ b/packages/backend/test/unit/UserSearchService.ts @@ -7,16 +7,18 @@ import { Test, TestingModule } from '@nestjs/testing'; import { describe, jest, test } from '@jest/globals'; import { In } from 'typeorm'; import { UserSearchService } from '@/core/UserSearchService.js'; -import { FollowingsRepository, MiUser, UserProfilesRepository, UsersRepository } from '@/models/_.js'; +import { FollowingsRepository, InstancesRepository, MiUser, UserProfilesRepository, UsersRepository } from '@/models/_.js'; import { IdService } from '@/core/IdService.js'; import { GlobalModule } from '@/GlobalModule.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { genAidx } from '@/misc/id/aidx.js'; describe('UserSearchService', () => { let app: TestingModule; let service: UserSearchService; + let instancesRepository: InstancesRepository; let usersRepository: UsersRepository; let followingsRepository: FollowingsRepository; let idService: IdService; @@ -35,6 +37,19 @@ describe('UserSearchService', () => { let bobby: MiUser; async function createUser(data: Partial = {}) { + if (data.host != null) { + await instancesRepository + .createQueryBuilder('instance') + .insert() + .values({ + id: genAidx(Date.now()), + firstRetrievedAt: new Date(), + host: data.host, + }) + .orIgnore() + .execute(); + } + const user = await usersRepository .insert({ id: idService.gen(), @@ -104,6 +119,7 @@ describe('UserSearchService', () => { await app.init(); + instancesRepository = app.get(DI.instancesRepository); usersRepository = app.get(DI.usersRepository); userProfilesRepository = app.get(DI.userProfilesRepository); followingsRepository = app.get(DI.followingsRepository); diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index 6f6d4c4121..94dec16401 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -103,6 +103,25 @@ describe('ActivityPub', () => { let config: Config; const metaInitial = { + id: 'x', + name: 'Test Instance', + shortName: 'Test Instance', + description: 'Test Instance', + langs: [] as string[], + pinnedUsers: [] as string[], + hiddenTags: [] as string[], + prohibitedWordsForNameOfUser: [] as string[], + silencedHosts: [] as string[], + mediaSilencedHosts: [] as string[], + policies: {}, + serverRules: [] as string[], + bannedEmailDomains: [] as string[], + preservedUsernames: [] as string[], + bubbleInstances: [] as string[], + trustedLinkUrlPatterns: [] as string[], + federation: 'all', + federationHosts: [] as string[], + allowUnsignedFetch: 'always', cacheRemoteFiles: true, cacheRemoteSensitiveFiles: true, enableFanoutTimeline: true, From 53ec45482d6424964bdaf767e15b16623dc69661 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 30 May 2025 07:46:23 -0400 Subject: [PATCH 28/29] use more robust fixup in 1748128176881-add_instance_foreign_keys.js --- .../1748128176881-add_instance_foreign_keys.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/backend/migration/1748128176881-add_instance_foreign_keys.js b/packages/backend/migration/1748128176881-add_instance_foreign_keys.js index 9a970699a0..2c2383c50f 100644 --- a/packages/backend/migration/1748128176881-add_instance_foreign_keys.js +++ b/packages/backend/migration/1748128176881-add_instance_foreign_keys.js @@ -15,10 +15,19 @@ export class AddInstanceForeignKeys1748128176881 { name = 'AddInstanceForeignKeys1748128176881' async up(queryRunner) { - // Fix-up: Sharkey versions in early-mid 2025 could federate with HTTP URLs, which would produce a user with no matching instance. - // These users are fundamentally broken and can just be removed, which ensures that the FK can create without conflicts. - // But we must also *preserve* those with a matching registered instance, as FireFish allowed federation over HTTP and some older instances may have fully-populated users. - await queryRunner.query(`DELETE FROM "user" WHERE "uri" LIKE 'http:%' AND NOT EXISTS (select 1 from "instance" where "instance"."host" = "user"."host")`); + // Fix-up: Some older instances have users without a matching instance entry + await queryRunner.query(` + INSERT INTO "instance" ("id", "host", "firstRetrievedAt") + SELECT + MIN("id"), + "host", + COALESCE(MIN("lastFetchedAt"), CURRENT_TIMESTAMP) + FROM "user" + WHERE + "host" IS NOT NULL AND + NOT EXISTS (select 1 from "instance" where "instance"."host" = "user"."host") + GROUP BY "host" + `); await queryRunner.query(`ALTER TABLE "user" ADD CONSTRAINT "FK_user_host" FOREIGN KEY ("host") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_userHost" FOREIGN KEY ("userHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); From 9c4e3934d1c79c76448568730b765bcefde0669e Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 30 May 2025 14:39:37 -0400 Subject: [PATCH 29/29] exclude local notes from bubble timeline --- .../backend/src/server/api/endpoints/notes/bubble-timeline.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts b/packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts index be08f84b6b..7c375cb0f5 100644 --- a/packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts @@ -85,7 +85,8 @@ export default class extends Endpoint { // eslint- ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) .andWhere('note.visibility = \'public\'') .andWhere('note.channelId IS NULL') - .andWhere('(note.userHost IS NULL OR userInstance.isBubbled = true)') // This comes from generateVisibilityQuery below + .andWhere('note.userHost IS NULL') + .andWhere('userInstance.isBubbled = true') // This comes from generateVisibilityQuery below .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote')