diff --git a/.config/ci.yml b/.config/ci.yml index 4a6d21e1d5..5fcf78b737 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -115,8 +115,14 @@ db: user: postgres pass: ci - # Whether disable Caching queries - #disableCache: true + ## Log a warning to the server console if any query takes longer than this to complete. + ## Measured in milliseconds; set to 0 to disable. (default: 300) + #slowQueryThreshold: 300 + + # If false, then query results will be cached in redis. + # If true (default), then queries will not be cached. + # This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior. + #disableCache: false # Extra Connection options #extra: @@ -297,6 +303,10 @@ proxyBypassHosts: #proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4 #proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5 +# Path to the directory that uploaded media will be saved to +# Defaults to a folder called "files" in the Sharkey directory +#mediaDirectory: /var/lib/sharkey + # Media Proxy #mediaProxy: https://example.com/proxy diff --git a/.config/cypress-devcontainer.yml b/.config/cypress-devcontainer.yml index 356d583611..97263da68f 100644 --- a/.config/cypress-devcontainer.yml +++ b/.config/cypress-devcontainer.yml @@ -57,8 +57,14 @@ db: user: postgres pass: postgres - # Whether disable Caching queries - #disableCache: true + ## Log a warning to the server console if any query takes longer than this to complete. + ## Measured in milliseconds; set to 0 to disable. (default: 300) + #slowQueryThreshold: 300 + + # If false, then query results will be cached in redis. + # If true (default), then queries will not be cached. + # This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior. + #disableCache: false # Extra Connection options #extra: @@ -260,6 +266,10 @@ proxyBypassHosts: #proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4 #proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5 +# Path to the directory that uploaded media will be saved to +# Defaults to a folder called "files" in the Sharkey directory +#mediaDirectory: /var/lib/sharkey + # Media Proxy #mediaProxy: https://example.com/proxy diff --git a/.config/docker_example.yml b/.config/docker_example.yml index 68679f64ed..3aaa56e333 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -118,8 +118,14 @@ db: user: example-misskey-user pass: example-misskey-pass - # Whether disable Caching queries - #disableCache: true + ## Log a warning to the server console if any query takes longer than this to complete. + ## Measured in milliseconds; set to 0 to disable. (default: 300) + #slowQueryThreshold: 300 + + # If false, then query results will be cached in redis. + # If true (default), then queries will not be cached. + # This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior. + #disableCache: false # Extra Connection options #extra: @@ -351,6 +357,10 @@ proxyBypassHosts: #proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4 #proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5 +# Path to the directory that uploaded media will be saved to +# Defaults to a folder called "files" in the Sharkey directory +#mediaDirectory: /var/lib/sharkey + # Media Proxy # Reference Implementation: https://github.com/misskey-dev/media-proxy # * Deliver a common cache between instances diff --git a/.config/example.yml b/.config/example.yml index 9cb1e656c1..8cac42c050 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -121,8 +121,14 @@ db: user: sharkey pass: example-misskey-pass - # Whether disable Caching queries - #disableCache: true + ## Log a warning to the server console if any query takes longer than this to complete. + ## Measured in milliseconds; set to 0 to disable. (default: 300) + #slowQueryThreshold: 300 + + # If false, then query results will be cached in redis. + # If true (default), then queries will not be cached. + # This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior. + #disableCache: false # Extra Connection options #extra: @@ -354,6 +360,10 @@ proxyBypassHosts: #proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4 #proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5 +# Path to the directory that uploaded media will be saved to +# Defaults to a folder called "files" in the Sharkey directory +#mediaDirectory: /var/lib/sharkey + # Media Proxy # Reference Implementation: https://github.com/misskey-dev/media-proxy # * Deliver a common cache between instances diff --git a/.devcontainer/devcontainer.yml b/.devcontainer/devcontainer.yml index 6d904e87b9..1f8192101b 100644 --- a/.devcontainer/devcontainer.yml +++ b/.devcontainer/devcontainer.yml @@ -199,6 +199,10 @@ proxyBypassHosts: #proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4 #proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5 +# Path to the directory that uploaded media will be saved to +# Defaults to a folder called "files" in the Sharkey directory +#mediaDirectory: /var/lib/sharkey + # Media Proxy #mediaProxy: https://example.com/proxy diff --git a/.gitignore b/.gitignore index b07d195a3f..ea761882da 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,7 @@ vite.config.local-dev.ts.timestamp-* # Sharkey /packages/megalodon/lib + +# TypeScript +.tsbuildinfo +*.tsbuildinfo diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c8a58fadab..ef08d5275f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -690,7 +690,7 @@ seems to do a decent job) * re-generate locales (`pnpm run build-assets`) and commit * build the frontend: `rm -rf built/; NODE_ENV=development pnpm --filter=frontend --filter=frontend-embed --filter=frontend-shared build` (the `development` tells it to keep some of the original filenames in the built files) * make sure there aren't any new `ti-*` classes (Tabler Icons), and replace them with appropriate `ph-*` ones (Phosphor Icons) in [`vite.replaceicons.ts`](packages/frontend/vite.replaceIcons.ts). - * This command should show you want to change: `grep -ohrP '(?<=["'\'']ti )(ti-(?!fw)[\w\-]+)' --exclude \*.map -- built/ | sort -u`. + * This command should show you want to change: `grep -ohrP '(?<=["'\''](ti )?)(ti-(?!fw)[\w\-]+)' --exclude \*.map -- built/ | sort -u`. * NOTE: `ti-fw` is a special class that's defined by Misskey, leave it alone. * After every change, re-build the frontend and check again, until there are no more `ti-*` classes in the built files. * Commit! diff --git a/cypress/tsconfig.json b/cypress/tsconfig.json index 6fe7f32cc4..4435a4fda8 100644 --- a/cypress/tsconfig.json +++ b/cypress/tsconfig.json @@ -2,7 +2,8 @@ "compilerOptions": { "lib": ["dom", "es5"], "target": "es5", - "types": ["cypress", "node"] + "types": ["cypress", "node"], + "incremental": true }, "include": ["./**/*.ts"] } diff --git a/locales/index.d.ts b/locales/index.d.ts index f809613e7a..cee973a0a2 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -12492,6 +12492,14 @@ export interface Locale extends ILocale { * Displays content centered. */ "centerDescription": string; + /** + * Unix Time + */ + "unixtime": string; + /** + * Displays a timestamp in the viewer's current timezone. + */ + "unixtimeDescription": string; /** * Code (Inline) */ @@ -13069,6 +13077,26 @@ export interface Locale extends ILocale { * Users popular on {name} */ "popularUsersLocal": ParameterizedString<"name">; + /** + * Polls trending on {name} + */ + "pollsOnLocal": ParameterizedString<"name">; + /** + * Polls trending on the global network + */ + "pollsOnRemote": string; + /** + * Polls that have ended recently + */ + "pollsExpired": string; + /** + * Trending polls are disabled on this instance. + */ + "trendingPollsDisabled": string; + /** + * Please log in to view trending polls. + */ + "trendingPollsDisabledLogIn": string; /** * Silenced */ @@ -13141,6 +13169,54 @@ export interface Locale extends ILocale { * Written by {user} */ "writtenBy": ParameterizedString<"user">; + /** + * Following (Pub) + */ + "followingPub": string; + /** + * Followers (Sub) + */ + "followersSub": string; + /** + * Well-known resources + */ + "wellKnownResources": string; + /** + * Last posted: {at} + */ + "lastPosted": ParameterizedString<"at">; + /** + * NSFW + */ + "nsfw": string; + /** + * Raw + */ + "raw": string; + /** + * CW + */ + "cw": string; + /** + * Media Silenced + */ + "mediaSilenced": string; + /** + * Bubble + */ + "bubble": string; + /** + * Verified + */ + "verified": string; + /** + * Not Verified + */ + "notVerified": string; + /** + * Hibernated + */ + "hibernated": string; } declare const locales: { [lang: string]: Locale; diff --git a/package.json b/package.json index ffd07c5950..dfa7afda02 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharkey", - "version": "2025.4.2-rc", + "version": "2025.5.0-dev", "codename": "shonk", "repository": { "type": "git", diff --git a/packages/backend/migration/1747938628395-add-missing-indexes.js b/packages/backend/migration/1747938628395-add-missing-indexes.js index 745b39c855..0229a6c898 100644 --- a/packages/backend/migration/1747938628395-add-missing-indexes.js +++ b/packages/backend/migration/1747938628395-add-missing-indexes.js @@ -4,19 +4,35 @@ */ export class AddMissingIndexes1747938628395 { - name = 'AddMissingIndexes1747938628395' + name = 'AddMissingIndexes1747938628395' - async up(queryRunner) { - await queryRunner.query(`CREATE INDEX "IDX_58699f75b9cf904f5f007909cb" ON "user_profile" ("birthday") `); - await queryRunner.query(`CREATE INDEX "IDX_021015e6683570ae9f6b0c62be" ON "user_list_membership" ("userId") `); - await queryRunner.query(`CREATE INDEX "IDX_cddcaf418dc4d392ecfcca842a" ON "user_list_membership" ("userListId") `); - await queryRunner.query(`CREATE UNIQUE INDEX "IDX_e4f3094c43f2d665e6030b0337" ON "user_list_membership" ("userId", "userListId") `); - } + async up(queryRunner) { + // Some instances have duplicate list entries + await queryRunner.query(` + DELETE FROM "user_list_membership" + WHERE "id" NOT IN ( + SELECT MIN("id") + FROM "user_list_membership" + GROUP BY "userId", "userListId" + )`); - async down(queryRunner) { - await queryRunner.query(`DROP INDEX "public"."IDX_e4f3094c43f2d665e6030b0337"`); - await queryRunner.query(`DROP INDEX "public"."IDX_cddcaf418dc4d392ecfcca842a"`); - await queryRunner.query(`DROP INDEX "public"."IDX_021015e6683570ae9f6b0c62be"`); - await queryRunner.query(`DROP INDEX "public"."IDX_58699f75b9cf904f5f007909cb"`); - } + // Some instances already have these indexes, for an unknown reason + await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_e4f3094c43f2d665e6030b0337"`); + await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_cddcaf418dc4d392ecfcca842a"`); + await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_021015e6683570ae9f6b0c62be"`); + await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_58699f75b9cf904f5f007909cb"`); + + // Now the actual migration + await queryRunner.query(`CREATE INDEX "IDX_58699f75b9cf904f5f007909cb" ON "user_profile" ("birthday") `); + await queryRunner.query(`CREATE INDEX "IDX_021015e6683570ae9f6b0c62be" ON "user_list_membership" ("userId") `); + await queryRunner.query(`CREATE INDEX "IDX_cddcaf418dc4d392ecfcca842a" ON "user_list_membership" ("userListId") `); + await queryRunner.query(`CREATE UNIQUE INDEX "IDX_e4f3094c43f2d665e6030b0337" ON "user_list_membership" ("userId", "userListId") `); + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "public"."IDX_e4f3094c43f2d665e6030b0337"`); + await queryRunner.query(`DROP INDEX "public"."IDX_cddcaf418dc4d392ecfcca842a"`); + await queryRunner.query(`DROP INDEX "public"."IDX_021015e6683570ae9f6b0c62be"`); + await queryRunner.query(`DROP INDEX "public"."IDX_58699f75b9cf904f5f007909cb"`); + } } diff --git a/packages/backend/migration/1748104955717-index_IDX_instance_host_key.js b/packages/backend/migration/1748104955717-index_IDX_instance_host_key.js new file mode 100644 index 0000000000..139eae740f --- /dev/null +++ b/packages/backend/migration/1748104955717-index_IDX_instance_host_key.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class IndexIDXInstanceHostKey1748104955717 { + name = 'IndexIDXInstanceHostKey1748104955717' + + async up(queryRunner) { + await queryRunner.query(`CREATE UNIQUE INDEX "IDX_instance_host_key" ON "instance" (((lower(reverse("host")) || '.')::text) text_pattern_ops)`); + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "IDX_instance_host_key"`); + } +} diff --git a/packages/backend/migration/1748105111513-add_instance_block_columns.js b/packages/backend/migration/1748105111513-add_instance_block_columns.js new file mode 100644 index 0000000000..6e3d78d5e8 --- /dev/null +++ b/packages/backend/migration/1748105111513-add_instance_block_columns.js @@ -0,0 +1,85 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + * @typedef {{ blockedHosts: string[], silencedHosts: string[], mediaSilencedHosts: string[], federationHosts: string[], bubbleInstances: string[] }} Meta + */ + +/** + * @class + * @implements {MigrationInterface} + */ +export class AddInstanceBlockColumns1748105111513 { + name = 'AddInstanceBlockColumns1748105111513' + + async up(queryRunner) { + // Schema migration + await queryRunner.query(`ALTER TABLE "instance" ADD "isBlocked" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "instance"."isBlocked" IS 'True if this instance is blocked from federation.'`); + await queryRunner.query(`ALTER TABLE "instance" ADD "isAllowListed" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "instance"."isAllowListed" IS 'True if this instance is allow-listed.'`); + await queryRunner.query(`ALTER TABLE "instance" ADD "isBubbled" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "instance"."isBubbled" IS 'True if this instance is part of the local bubble.'`); + await queryRunner.query(`ALTER TABLE "instance" ADD "isSilenced" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "instance"."isSilenced" IS 'True if this instance is silenced.'`); + await queryRunner.query(`ALTER TABLE "instance" ADD "isMediaSilenced" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "instance"."isMediaSilenced" IS 'True if this instance is media-silenced.'`); + + // Data migration + /** @type {Meta[]} */ + const metas = await queryRunner.query(`SELECT "blockedHosts", "silencedHosts", "mediaSilencedHosts", "federationHosts", "bubbleInstances" FROM "meta"`); + if (metas.length > 0) { + /** @type {Meta} */ + const meta = metas[0]; + + // Blocked hosts + if (meta.blockedHosts.length > 0) { + const patterns = buildPatterns(meta.blockedHosts); + await queryRunner.query(`UPDATE "instance" SET "isBlocked" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]); + } + + // Silenced hosts + if (meta.silencedHosts.length > 0) { + const patterns = buildPatterns(meta.silencedHosts); + await queryRunner.query(`UPDATE "instance" SET "isSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]); + } + + // Media silenced hosts + if (meta.mediaSilencedHosts.length > 0) { + const patterns = buildPatterns(meta.mediaSilencedHosts); + await queryRunner.query(`UPDATE "instance" SET "isMediaSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]); + } + + // Allow-listed hosts + if (meta.federationHosts.length > 0) { + const patterns = buildPatterns(meta.federationHosts); + await queryRunner.query(`UPDATE "instance" SET "isAllowListed" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]); + } + + // Bubbled hosts + if (meta.bubbleInstances.length > 0) { + const patterns = buildPatterns(meta.bubbleInstances); + await queryRunner.query(`UPDATE "instance" SET "isBubbled" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]); + } + } + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isMediaSilenced"`); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isSilenced"`); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isBubbled"`); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isAllowListed"`); + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isBlocked"`); + } +} + +/** + * @param {string[]} input + * @returns {string[]} + */ +function buildPatterns(input) { + return input.map(i => i.toLowerCase().split('').reverse().join('') + '.%'); +} diff --git a/packages/backend/migration/1748128176881-add_instance_foreign_keys.js b/packages/backend/migration/1748128176881-add_instance_foreign_keys.js new file mode 100644 index 0000000000..2c2383c50f --- /dev/null +++ b/packages/backend/migration/1748128176881-add_instance_foreign_keys.js @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + */ + +/** + * @class + * @implements {MigrationInterface} + */ +export class AddInstanceForeignKeys1748128176881 { + name = 'AddInstanceForeignKeys1748128176881' + + async up(queryRunner) { + // Fix-up: Some older instances have users without a matching instance entry + await queryRunner.query(` + INSERT INTO "instance" ("id", "host", "firstRetrievedAt") + SELECT + MIN("id"), + "host", + COALESCE(MIN("lastFetchedAt"), CURRENT_TIMESTAMP) + FROM "user" + WHERE + "host" IS NOT NULL AND + NOT EXISTS (select 1 from "instance" where "instance"."host" = "user"."host") + GROUP BY "host" + `); + + await queryRunner.query(`ALTER TABLE "user" ADD CONSTRAINT "FK_user_host" FOREIGN KEY ("host") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_userHost" FOREIGN KEY ("userHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_replyUserHost" FOREIGN KEY ("replyUserHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_renoteUserHost" FOREIGN KEY ("renoteUserHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_renoteUserHost"`); + await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_replyUserHost"`); + await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_userHost"`); + await queryRunner.query(`ALTER TABLE "user" DROP CONSTRAINT "FK_user_host"`); + } +} diff --git a/packages/backend/migration/1748137683887-add_instance_foreign_keys_to_following.js b/packages/backend/migration/1748137683887-add_instance_foreign_keys_to_following.js new file mode 100644 index 0000000000..8f4a977ff5 --- /dev/null +++ b/packages/backend/migration/1748137683887-add_instance_foreign_keys_to_following.js @@ -0,0 +1,26 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + */ + +/** + * @class + * @implements {MigrationInterface} + */ +export class AddInstanceForeignKeysToFollowing1748137683887 { + name = 'AddInstanceForeignKeysToFollowing1748137683887' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "following" ADD CONSTRAINT "FK_following_followerHost" FOREIGN KEY ("followerHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "following" ADD CONSTRAINT "FK_following_followeeHost" FOREIGN KEY ("followeeHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "following" DROP CONSTRAINT "FK_following_followeeHost"`); + await queryRunner.query(`ALTER TABLE "following" DROP CONSTRAINT "FK_following_followerHost"`); + } +} diff --git a/packages/backend/migration/1748191631151-analyze_instance-user-note-following.js b/packages/backend/migration/1748191631151-analyze_instance-user-note-following.js new file mode 100644 index 0000000000..f03a60980b --- /dev/null +++ b/packages/backend/migration/1748191631151-analyze_instance-user-note-following.js @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + */ + +/** + * @class + * @implements {MigrationInterface} + */ +export class AnalyzeInstanceUserNoteFollowing1748191631151 { + name = 'AnalyzeInstanceUserNoteFollowing1748191631151' + + async up(queryRunner) { + // Refresh statistics for tables impacted by new indexes. + // This helps the query planner to efficiently use them without waiting for the next full vacuum. + await queryRunner.query(`ANALYZE "instance", "user", "following", "note"`); + } + + async down(queryRunner) { + } +} diff --git a/packages/backend/package.json b/packages/backend/package.json index b9cb0002ab..bad6990ba5 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -10,6 +10,9 @@ "start": "node ./built/boot/entry.js", "start:test": "cross-env NODE_ENV=test node ./built/boot/entry.js", "migrate": "pnpm typeorm migration:run -d ormconfig.js", + "migrate:revert": "pnpm typeorm migration:revert -d ormconfig.js", + "migrate:generate": "pnpm typeorm migration:generate -d ormconfig.js", + "migrate:create": "pnpm typeorm migration:create", "revert": "pnpm typeorm migration:revert -d ormconfig.js", "check:connect": "node ./scripts/check_connect.js", "build": "swc src -d built -D --strip-leading-paths", diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index e02f09bbe1..c2e7efd456 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -9,6 +9,7 @@ import { dirname, resolve } from 'node:path'; import * as yaml from 'js-yaml'; import { globSync } from 'glob'; import ipaddr from 'ipaddr.js'; +import Logger from './logger.js'; import type * as Sentry from '@sentry/node'; import type * as SentryVue from '@sentry/vue'; import type { RedisOptions } from 'ioredis'; @@ -40,6 +41,7 @@ type Source = { db?: string; user?: string; pass?: string; + slowQueryThreshold?: number; disableCache?: boolean; extra?: { [x: string]: string }; }; @@ -111,6 +113,7 @@ type Source = { deliverJobMaxAttempts?: number; inboxJobMaxAttempts?: number; + mediaDirectory?: string; mediaProxy?: string; proxyRemoteFiles?: boolean; videoThumbnailGenerator?: string; @@ -154,6 +157,8 @@ type Source = { } }; +const configLogger = new Logger('config'); + export type PrivateNetworkSource = string | { network?: string, ports?: number[] }; export type PrivateNetwork = { @@ -191,7 +196,7 @@ export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefine } } - console.warn('[config] Skipping invalid entry in allowedPrivateNetworks: ', e); + configLogger.warn('Skipping invalid entry in allowedPrivateNetworks: ', e); return null; }) .filter(p => p != null); @@ -221,6 +226,7 @@ export type Config = { db: string; user: string; pass: string; + slowQueryThreshold?: number; disableCache?: boolean; extra?: { [x: string]: string }; }; @@ -297,6 +303,7 @@ export type Config = { frontendManifestExists: boolean; frontendEmbedEntry: string; frontendEmbedManifestExists: boolean; + mediaDirectory: string; mediaProxy: string; externalMediaProxyEnabled: boolean; videoThumbnailGenerator: string | null; @@ -346,7 +353,7 @@ const _dirname = dirname(_filename); /** * Path of configuration directory */ -const dir = `${_dirname}/../../../.config`; +const dir = process.env.MISSKEY_CONFIG_DIR ?? `${_dirname}/../../../.config`; /** * Path of configuration file @@ -373,11 +380,14 @@ export function loadConfig(): Config { if (configFiles.length === 0 && !process.env['MK_WARNED_ABOUT_CONFIG']) { - console.log('No config files loaded, check if this is intentional'); + configLogger.warn('No config files loaded, check if this is intentional'); process.env['MK_WARNED_ABOUT_CONFIG'] = '1'; } - const config = configFiles.map(path => fs.readFileSync(path, 'utf-8')) + const config = configFiles.map(path => { + configLogger.info(`Reading configuration from ${path}`); + return fs.readFileSync(path, 'utf-8'); + }) .map(contents => yaml.load(contents) as Source) .reduce( (acc: Source, cur: Source) => Object.assign(acc, cur), @@ -403,6 +413,10 @@ export function loadConfig(): Config { const internalMediaProxy = `${scheme}://${host}/proxy`; const redis = convertRedisOptions(config.redis, host); + // nullish => 300 (default) + // 0 => undefined (disabled) + const slowQueryThreshold = (config.db.slowQueryThreshold ?? 300) || undefined; + return { version, publishTarballInsteadOfProvideRepositoryUrl: !!config.publishTarballInsteadOfProvideRepositoryUrl, @@ -421,7 +435,7 @@ export function loadConfig(): Config { apiUrl: `${scheme}://${host}/api`, authUrl: `${scheme}://${host}/auth`, driveUrl: `${scheme}://${host}/files`, - db: { ...config.db, db: dbDb, user: dbUser, pass: dbPass }, + db: { ...config.db, db: dbDb, user: dbUser, pass: dbPass, slowQueryThreshold }, dbReplications: config.dbReplications, dbSlaves: config.dbSlaves, fulltextSearch: config.fulltextSearch, @@ -463,6 +477,7 @@ export function loadConfig(): Config { signToActivityPubGet: config.signToActivityPubGet ?? true, attachLdSignatureForRelays: config.attachLdSignatureForRelays ?? true, checkActivityPubGetSignature: config.checkActivityPubGetSignature, + mediaDirectory: config.mediaDirectory ?? resolve(_dirname, '../../../files'), mediaProxy: externalMediaProxy ?? internalMediaProxy, externalMediaProxyEnabled: externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy, videoThumbnailGenerator: config.videoThumbnailGenerator ? @@ -493,6 +508,10 @@ export function loadConfig(): Config { } function tryCreateUrl(url: string) { + if (!url) { + throw new Error('Failed to load: no "url" property found in config. Please check the value of "MISSKEY_CONFIG_DIR" and "MISSKEY_CONFIG_YML", and verify that all configuration files are correct.'); + } + try { return new URL(url); } catch (e) { @@ -624,7 +643,7 @@ function applyEnvOverrides(config: Source) { // these are all the settings that can be overridden _apply_top([['url', 'port', 'address', 'socket', 'chmodSocket', 'disableHsts', 'id', 'dbReplications', 'websocketCompression']]); - _apply_top(['db', ['host', 'port', 'db', 'user', 'pass', 'disableCache']]); + _apply_top(['db', ['host', 'port', 'db', 'user', 'pass', 'slowQueryThreshold', 'disableCache']]); _apply_top(['dbSlaves', Array.from((config.dbSlaves ?? []).keys()), ['host', 'port', 'db', 'user', 'pass']]); _apply_top([ ['redis', 'redisForPubsub', 'redisForJobQueue', 'redisForTimelines', 'redisForReactions', 'redisForRateLimit'], @@ -638,7 +657,7 @@ function applyEnvOverrides(config: Source) { _apply_top(['sentryForFrontend', 'vueIntegration', 'tracingOptions', 'timeout']); _apply_top(['sentryForFrontend', 'browserTracingIntegration', 'routeLabel']); _apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]); - _apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'proxyRemoteFiles', 'videoThumbnailGenerator']]); + _apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaDirectory', 'mediaProxy', 'proxyRemoteFiles', 'videoThumbnailGenerator']]); _apply_top([['maxFileSize', 'maxNoteLength', 'maxRemoteNoteLength', 'maxAltTextLength', 'maxRemoteAltTextLength', 'pidFile', 'filePermissionBits']]); _apply_top(['import', ['downloadTimeout', 'maxFileSize']]); _apply_top([['signToActivityPubGet', 'checkActivityPubGetSignature', 'setupPassword', 'disallowExternalApRedirect']]); diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 82c447baaa..73125f36d7 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -159,6 +159,14 @@ export class DriveService { // thunbnail, webpublic を必要なら生成 const alts = await this.generateAlts(path, type, !file.uri); + if (type && type.startsWith('video/')) { + try { + await this.videoProcessingService.webOptimizeVideo(path, type); + } catch (err) { + this.registerLogger.warn(`Video optimization failed: ${err instanceof Error ? err.message : String(err)}`, { error: err }); + } + } + if (this.meta.useObjectStorage) { //#region ObjectStorage params let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) ?? ['']); diff --git a/packages/backend/src/core/FanoutTimelineEndpointService.ts b/packages/backend/src/core/FanoutTimelineEndpointService.ts index af2723e99d..f9cf41e854 100644 --- a/packages/backend/src/core/FanoutTimelineEndpointService.ts +++ b/packages/backend/src/core/FanoutTimelineEndpointService.ts @@ -136,10 +136,10 @@ export class FanoutTimelineEndpointService { const parentFilter = filter; filter = (note) => { if (!ps.ignoreAuthorFromInstanceBlock) { - if (this.utilityService.isBlockedHost(this.meta.blockedHosts, note.userHost)) return false; + if (note.userInstance?.isBlocked) return false; } - if (note.userId !== note.renoteUserId && this.utilityService.isBlockedHost(this.meta.blockedHosts, note.renoteUserHost)) return false; - if (note.userId !== note.replyUserId && this.utilityService.isBlockedHost(this.meta.blockedHosts, note.replyUserHost)) return false; + if (note.userId !== note.renoteUserId && note.renoteUserInstance?.isBlocked) return false; + if (note.userId !== note.replyUserId && note.replyUserInstance?.isBlocked) return false; return parentFilter(note); }; @@ -194,7 +194,10 @@ export class FanoutTimelineEndpointService { .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); + .leftJoinAndSelect('note.channel', 'channel') + .leftJoinAndSelect('note.userInstance', 'userInstance') + .leftJoinAndSelect('note.replyUserInstance', 'replyUserInstance') + .leftJoinAndSelect('note.renoteUserInstance', 'renoteUserInstance'); const notes = (await query.getMany()).filter(noteFilter); diff --git a/packages/backend/src/core/FederatedInstanceService.ts b/packages/backend/src/core/FederatedInstanceService.ts index 3f7ed99348..34df10f0ff 100644 --- a/packages/backend/src/core/FederatedInstanceService.ts +++ b/packages/backend/src/core/FederatedInstanceService.ts @@ -5,23 +5,24 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Redis from 'ioredis'; -import { QueryFailedError } from 'typeorm'; -import type { InstancesRepository } from '@/models/_.js'; +import type { InstancesRepository, MiMeta } from '@/models/_.js'; import type { MiInstance } from '@/models/Instance.js'; -import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; +import { MemoryKVCache } from '@/misc/cache.js'; import { IdService } from '@/core/IdService.js'; import { DI } from '@/di-symbols.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; -import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; +import type { GlobalEvents } from '@/core/GlobalEventService.js'; +import { Serialized } from '@/types.js'; +import { diffArrays, diffArraysSimple } from '@/misc/diff-arrays.js'; @Injectable() export class FederatedInstanceService implements OnApplicationShutdown { - public federatedInstanceCache: RedisKVCache; + private readonly federatedInstanceCache: MemoryKVCache; constructor( - @Inject(DI.redis) - private redisClient: Redis.Redis, + @Inject(DI.redisForSub) + private redisForSub: Redis.Redis, @Inject(DI.instancesRepository) private instancesRepository: InstancesRepository, @@ -29,67 +30,46 @@ export class FederatedInstanceService implements OnApplicationShutdown { private utilityService: UtilityService, private idService: IdService, ) { - this.federatedInstanceCache = new RedisKVCache(this.redisClient, 'federatedInstance', { - lifetime: 1000 * 60 * 30, // 30m - memoryCacheLifetime: 1000 * 60 * 3, // 3m - fetcher: (key) => this.instancesRepository.findOneBy({ host: key }), - toRedisConverter: (value) => JSON.stringify(value), - fromRedisConverter: (value) => { - const parsed = JSON.parse(value); - if (parsed == null) return null; - return { - ...parsed, - firstRetrievedAt: new Date(parsed.firstRetrievedAt), - latestRequestReceivedAt: parsed.latestRequestReceivedAt ? new Date(parsed.latestRequestReceivedAt) : null, - infoUpdatedAt: parsed.infoUpdatedAt ? new Date(parsed.infoUpdatedAt) : null, - notRespondingSince: parsed.notRespondingSince ? new Date(parsed.notRespondingSince) : null, - }; - }, - }); + this.federatedInstanceCache = new MemoryKVCache(1000 * 60 * 3); // 3m + this.redisForSub.on('message', this.onMessage); } @bindThis public async fetchOrRegister(host: string): Promise { host = this.utilityService.toPuny(host); - const cached = await this.federatedInstanceCache.get(host); + const cached = this.federatedInstanceCache.get(host); if (cached) return cached; - const index = await this.instancesRepository.findOneBy({ host }); - + let index = await this.instancesRepository.findOneBy({ host }); if (index == null) { - let i; - try { - i = await this.instancesRepository.insertOne({ + await this.instancesRepository.createQueryBuilder('instance') + .insert() + .values({ id: this.idService.gen(), host, firstRetrievedAt: new Date(), - }); - } catch (e: unknown) { - if (e instanceof QueryFailedError) { - if (isDuplicateKeyValueError(e)) { - i = await this.instancesRepository.findOneBy({ host }); - } - } + isBlocked: this.utilityService.isBlockedHost(host), + isSilenced: this.utilityService.isSilencedHost(host), + isMediaSilenced: this.utilityService.isMediaSilencedHost(host), + isAllowListed: this.utilityService.isAllowListedHost(host), + isBubbled: this.utilityService.isBubbledHost(host), + }) + .orIgnore() + .execute(); - if (i == null) { - throw e; - } - } - - this.federatedInstanceCache.set(host, i); - return i; - } else { - this.federatedInstanceCache.set(host, index); - return index; + index = await this.instancesRepository.findOneByOrFail({ host }); } + + this.federatedInstanceCache.set(host, index); + return index; } @bindThis public async fetch(host: string): Promise { host = this.utilityService.toPuny(host); - const cached = await this.federatedInstanceCache.get(host); + const cached = this.federatedInstanceCache.get(host); if (cached !== undefined) return cached; const index = await this.instancesRepository.findOneBy({ host }); @@ -117,8 +97,35 @@ export class FederatedInstanceService implements OnApplicationShutdown { this.federatedInstanceCache.set(result.host, result); } + private syncCache(before: Serialized, after: Serialized): void { + const changed = + diffArraysSimple(before?.blockedHosts, after.blockedHosts) || + diffArraysSimple(before?.silencedHosts, after.silencedHosts) || + diffArraysSimple(before?.mediaSilencedHosts, after.mediaSilencedHosts) || + diffArraysSimple(before?.federationHosts, after.federationHosts) || + diffArraysSimple(before?.bubbleInstances, after.bubbleInstances); + + if (changed) { + // We have to clear the whole thing, otherwise subdomains won't be synced. + this.federatedInstanceCache.clear(); + } + } + + @bindThis + private async onMessage(_: string, data: string): Promise { + const obj = JSON.parse(data); + + if (obj.channel === 'internal') { + const { type, body } = obj.message as GlobalEvents['internal']['payload']; + if (type === 'metaUpdated') { + this.syncCache(body.before, body.after); + } + } + } + @bindThis public dispose(): void { + this.redisForSub.off('message', this.onMessage); this.federatedInstanceCache.dispose(); } diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts index 2951691129..a0f2607ddc 100644 --- a/packages/backend/src/core/HttpRequestService.ts +++ b/packages/backend/src/core/HttpRequestService.ts @@ -235,7 +235,9 @@ export class HttpRequestService { } @bindThis - public async getActivityJson(url: string, isLocalAddressAllowed = false): Promise { + public async getActivityJson(url: string, isLocalAddressAllowed = false, allowAnonymous = false): Promise { + this.apUtilityService.assertApUrl(url); + const res = await this.send(url, { method: 'GET', headers: { @@ -253,7 +255,11 @@ export class HttpRequestService { // Make sure the object ID matches the final URL (which is where it actually exists). // The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match. - this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); + if (allowAnonymous && activity.id == null) { + activity.id = res.url; + } else { + this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); + } return activity as IObjectWithId; } diff --git a/packages/backend/src/core/InternalStorageService.ts b/packages/backend/src/core/InternalStorageService.ts index b00c5796d2..abdbbc61d3 100644 --- a/packages/backend/src/core/InternalStorageService.ts +++ b/packages/backend/src/core/InternalStorageService.ts @@ -6,18 +6,11 @@ import * as fs from 'node:fs'; import { copyFile, unlink, writeFile, chmod } from 'node:fs/promises'; import * as Path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { dirname } from 'node:path'; import { Inject, Injectable } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { bindThis } from '@/decorators.js'; -const _filename = fileURLToPath(import.meta.url); -const _dirname = dirname(_filename); - -const path = Path.resolve(_dirname, '../../../../files'); - @Injectable() export class InternalStorageService { constructor( @@ -25,12 +18,12 @@ export class InternalStorageService { private config: Config, ) { // No one should erase the working directory *while the server is running*. - fs.mkdirSync(path, { recursive: true }); + fs.mkdirSync(this.config.mediaDirectory, { recursive: true }); } @bindThis public resolvePath(key: string) { - return Path.resolve(path, key); + return Path.resolve(this.config.mediaDirectory, key); } @bindThis diff --git a/packages/backend/src/core/MetaService.ts b/packages/backend/src/core/MetaService.ts index 40e7439f5f..07f82dc23e 100644 --- a/packages/backend/src/core/MetaService.ts +++ b/packages/backend/src/core/MetaService.ts @@ -4,7 +4,7 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import { DataSource } from 'typeorm'; +import { DataSource, EntityManager } from 'typeorm'; import * as Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; import { MiMeta } from '@/models/Meta.js'; @@ -12,6 +12,9 @@ import { GlobalEventService } from '@/core/GlobalEventService.js'; import { bindThis } from '@/decorators.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; import { FeaturedService } from '@/core/FeaturedService.js'; +import { MiInstance } from '@/models/Instance.js'; +import { diffArrays } from '@/misc/diff-arrays.js'; +import type { MetasRepository } from '@/models/_.js'; import type { OnApplicationShutdown } from '@nestjs/common'; @Injectable() @@ -26,6 +29,9 @@ export class MetaService implements OnApplicationShutdown { @Inject(DI.db) private db: DataSource, + @Inject(DI.metasRepository) + private readonly metasRepository: MetasRepository, + private featuredService: FeaturedService, private globalEventService: GlobalEventService, ) { @@ -67,35 +73,35 @@ export class MetaService implements OnApplicationShutdown { public async fetch(noCache = false): Promise { if (!noCache && this.cache) return this.cache; - return await this.db.transaction(async transactionalEntityManager => { - // 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する - const metas = await transactionalEntityManager.find(MiMeta, { - order: { + // 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する + let meta = await this.metasRepository.createQueryBuilder('meta') + .select() + .orderBy({ + id: 'DESC', + }) + .limit(1) + .getOne(); + + if (!meta) { + await this.metasRepository.createQueryBuilder('meta') + .insert() + .values({ + id: 'x', + }) + .orIgnore() + .execute(); + + meta = await this.metasRepository.createQueryBuilder('meta') + .select() + .orderBy({ id: 'DESC', - }, - }); + }) + .limit(1) + .getOneOrFail(); + } - const meta = metas[0]; - - if (meta) { - this.cache = meta; - return meta; - } else { - // metaが空のときfetchMetaが同時に呼ばれるとここが同時に呼ばれてしまうことがあるのでフェイルセーフなupsertを使う - const saved = await transactionalEntityManager - .upsert( - MiMeta, - { - id: 'x', - }, - ['id'], - ) - .then((x) => transactionalEntityManager.findOneByOrFail(MiMeta, x.identifiers[0])); - - this.cache = saved; - return saved; - } - }); + this.cache = meta; + return meta; } @bindThis @@ -103,7 +109,7 @@ export class MetaService implements OnApplicationShutdown { let before: MiMeta | undefined; const updated = await this.db.transaction(async transactionalEntityManager => { - const metas = await transactionalEntityManager.find(MiMeta, { + const metas: (MiMeta | undefined)[] = await transactionalEntityManager.find(MiMeta, { order: { id: 'DESC', }, @@ -126,6 +132,10 @@ export class MetaService implements OnApplicationShutdown { }, }); + // Propagate changes to blockedHosts, silencedHosts, mediaSilencedHosts, federationInstances, and bubbleInstances to the relevant instance rows + // Do this inside the transaction to avoid potential race condition (when an instance gets registered while we're updating). + await this.persistBlocks(transactionalEntityManager, before ?? {}, afters[0]); + return afters[0]; }); @@ -159,4 +169,49 @@ export class MetaService implements OnApplicationShutdown { public onApplicationShutdown(signal?: string | undefined): void { this.dispose(); } + + private async persistBlocks(tem: EntityManager, before: Partial, after: Partial): Promise { + await this.persistBlock(tem, before.blockedHosts, after.blockedHosts, 'isBlocked'); + await this.persistBlock(tem, before.silencedHosts, after.silencedHosts, 'isSilenced'); + await this.persistBlock(tem, before.mediaSilencedHosts, after.mediaSilencedHosts, 'isMediaSilenced'); + await this.persistBlock(tem, before.federationHosts, after.federationHosts, 'isAllowListed'); + await this.persistBlock(tem, before.bubbleInstances, after.bubbleInstances, 'isBubbled'); + } + + private async persistBlock(tem: EntityManager, before: string[] | undefined, after: string[] | undefined, field: keyof MiInstance): Promise { + const { added, removed } = diffArrays(before, after); + + if (removed.length > 0) { + await this.updateInstancesByHost(tem, field, false, removed); + } + + if (added.length > 0) { + await this.updateInstancesByHost(tem, field, true, added); + } + } + + private async updateInstancesByHost(tem: EntityManager, field: keyof MiInstance, value: boolean, hosts: string[]): Promise { + // Use non-array queries when possible, as they are indexed and can be much faster. + if (hosts.length === 1) { + const pattern = genHostPattern(hosts[0]); + await tem + .createQueryBuilder(MiInstance, 'instance') + .update() + .set({ [field]: value }) + .where('(lower(reverse("host")) || \'.\') LIKE :pattern', { pattern }) + .execute(); + } else if (hosts.length > 1) { + const patterns = hosts.map(host => genHostPattern(host)); + await tem + .createQueryBuilder(MiInstance, 'instance') + .update() + .set({ [field]: value }) + .where('(lower(reverse("host")) || \'.\') LIKE ANY (:patterns)', { patterns }) + .execute(); + } + } +} + +function genHostPattern(host: string): string { + return host.toLowerCase().split('').reverse().join('') + '.%'; } diff --git a/packages/backend/src/core/QueryService.ts b/packages/backend/src/core/QueryService.ts index 50a72e8aa6..cf2419a9eb 100644 --- a/packages/backend/src/core/QueryService.ts +++ b/packages/backend/src/core/QueryService.ts @@ -243,47 +243,44 @@ export class QueryService { q.andWhere(new Brackets(qb => { qb - .where(new Brackets(qb => { - qb.where('note.renoteId IS NOT NULL'); - qb.andWhere('note.text IS NULL'); - qb.andWhere(`note.userId NOT IN (${ mutingQuery.getQuery() })`); - })) .orWhere('note.renoteId IS NULL') - .orWhere('note.text IS NOT NULL'); + .orWhere('note.text IS NOT NULL') + .orWhere('note.cw IS NOT NULL') + .orWhere('note.replyId IS NOT NULL') + .orWhere('note.hasPoll = false') + .orWhere('note.fileIds != \'{}\'') + .orWhere(`note.userId NOT IN (${ mutingQuery.getQuery() })`); })); q.setParameters(mutingQuery.getParameters()); } @bindThis - public generateBlockedHostQueryForNote(q: SelectQueryBuilder, excludeAuthor?: boolean): void { - let nonBlockedHostQuery: (part: string) => string; - if (this.meta.blockedHosts.length === 0) { - nonBlockedHostQuery = () => '1=1'; - } else { - nonBlockedHostQuery = (match: string) => `('.' || ${match}) NOT ILIKE ALL(select '%.' || x from (select unnest("blockedHosts") as x from "meta") t)`; + public generateBlockedHostQueryForNote(q: SelectQueryBuilder, excludeAuthor?: boolean, allowSilenced = true): void { + function checkFor(key: 'user' | 'replyUser' | 'renoteUser') { + q.leftJoin(`note.${key}Instance`, `${key}Instance`); + q.andWhere(new Brackets(qb => { + qb.orWhere(`note.${key}Id IS NULL`) // no corresponding user + .orWhere(`note.${key}Host IS NULL`); // local + + if (allowSilenced) { + qb.orWhere(`${key}Instance.isBlocked = false`); // not blocked + } else { + qb.orWhere(new Brackets(qbb => qbb + .andWhere(`${key}Instance.isBlocked = false`) // not blocked + .andWhere(`${key}Instance.isSilenced = false`))); // not silenced + } + + if (excludeAuthor) { + qb.orWhere(`note.userId = note.${key}Id`); // author + } + })); } - if (excludeAuthor) { - const instanceSuspension = (user: string) => new Brackets(qb => qb - .where(`note.${user}Id IS NULL`) // no corresponding user - .orWhere(`note.userId = note.${user}Id`) - .orWhere(`note.${user}Host IS NULL`) // local - .orWhere(nonBlockedHostQuery(`note.${user}Host`))); - - q - .andWhere(instanceSuspension('replyUser')) - .andWhere(instanceSuspension('renoteUser')); - } else { - const instanceSuspension = (user: string) => new Brackets(qb => qb - .where(`note.${user}Id IS NULL`) // no corresponding user - .orWhere(`note.${user}Host IS NULL`) // local - .orWhere(nonBlockedHostQuery(`note.${user}Host`))); - - q - .andWhere(instanceSuspension('user')) - .andWhere(instanceSuspension('replyUser')) - .andWhere(instanceSuspension('renoteUser')); + if (!excludeAuthor) { + checkFor('user'); } + checkFor('replyUser'); + checkFor('renoteUser'); } } diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index f05ee2ee73..86bf20067e 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -10,7 +10,7 @@ import { IdentifiableError } from '@/misc/identifiable-error.js'; import type { MiRemoteUser, MiUser } from '@/models/User.js'; import type { MiNote } from '@/models/Note.js'; import { IdService } from '@/core/IdService.js'; -import type { MiNoteReaction } from '@/models/NoteReaction.js'; +import { MiNoteReaction } from '@/models/NoteReaction.js'; import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { NotificationService } from '@/core/NotificationService.js'; @@ -31,6 +31,7 @@ import { isQuote, isRenote } from '@/misc/is-renote.js'; import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js'; import { PER_NOTE_REACTION_USER_PAIR_CACHE_MAX } from '@/const.js'; import { CacheService } from '@/core/CacheService.js'; +import type { DataSource } from 'typeorm'; const FALLBACK = '\u2764'; @@ -89,6 +90,9 @@ export class ReactionService { @Inject(DI.emojisRepository) private emojisRepository: EmojisRepository, + @Inject(DI.db) + private readonly db: DataSource, + private utilityService: UtilityService, private customEmojiService: CustomEmojiService, private roleService: RoleService, @@ -176,26 +180,28 @@ export class ReactionService { reaction, }; - try { - await this.noteReactionsRepository.insert(record); - } catch (e) { - if (isDuplicateKeyValueError(e)) { - const exists = await this.noteReactionsRepository.findOneByOrFail({ - noteId: note.id, - userId: user.id, - }); + const result = await this.db.transaction(async tem => { + await tem.createQueryBuilder(MiNoteReaction, 'noteReaction') + .insert() + .values(record) + .orIgnore() + .execute(); - if (exists.reaction !== reaction) { - // 別のリアクションがすでにされていたら置き換える - await this.delete(user, note); - await this.noteReactionsRepository.insert(record); - } else { - // 同じリアクションがすでにされていたらエラー - throw new IdentifiableError('51c42bb4-931a-456b-bff7-e5a8a70dd298'); - } - } else { - throw e; + return await tem.createQueryBuilder(MiNoteReaction, 'noteReaction') + .select() + .where({ noteId: note.id, userId: user.id }) + .getOneOrFail(); + }); + + if (result.id !== record.id) { + // Conflict with the same ID => nothing to do. + if (result.reaction === record.reaction) { + return; } + + // 別のリアクションがすでにされていたら置き換える + await this.delete(user, note); + await this.noteReactionsRepository.insert(record); } // Increment reactions count diff --git a/packages/backend/src/core/ReversiService.ts b/packages/backend/src/core/ReversiService.ts index 8c0a8f6cc7..e31d9e5b1a 100644 --- a/packages/backend/src/core/ReversiService.ts +++ b/packages/backend/src/core/ReversiService.ts @@ -587,6 +587,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { lastActiveDate: parsed.user1.lastActiveDate != null ? new Date(parsed.user1.lastActiveDate) : null, lastFetchedAt: parsed.user1.lastFetchedAt != null ? new Date(parsed.user1.lastFetchedAt) : null, movedAt: parsed.user1.movedAt != null ? new Date(parsed.user1.movedAt) : null, + instance: null, } : null, user2: parsed.user2 != null ? { ...parsed.user2, @@ -597,6 +598,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit { lastActiveDate: parsed.user2.lastActiveDate != null ? new Date(parsed.user2.lastActiveDate) : null, lastFetchedAt: parsed.user2.lastFetchedAt != null ? new Date(parsed.user2.lastFetchedAt) : null, movedAt: parsed.user2.movedAt != null ? new Date(parsed.user2.movedAt) : null, + instance: null, } : null, }; } else { diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index d3c458eec7..b250eeee21 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -86,10 +86,10 @@ export const DEFAULT_POLICIES: RolePolicies = { canManageCustomEmojis: false, canManageAvatarDecorations: false, canSearchNotes: false, - canUseTranslator: true, + canUseTranslator: false, canHideAds: false, driveCapacityMb: 100, - maxFileSizeMb: 10, + maxFileSizeMb: 25, alwaysMarkNsfw: false, canUpdateBioMedia: true, pinLimit: 5, diff --git a/packages/backend/src/core/UtilityService.ts b/packages/backend/src/core/UtilityService.ts index 170afc72dc..3098367392 100644 --- a/packages/backend/src/core/UtilityService.ts +++ b/packages/backend/src/core/UtilityService.ts @@ -49,22 +49,49 @@ export class UtilityService { return regexp.test(email); } + public isBlockedHost(host: string | null): boolean; + public isBlockedHost(blockedHosts: string[], host: string | null): boolean; @bindThis - public isBlockedHost(blockedHosts: string[], host: string | null): boolean { + public isBlockedHost(blockedHostsOrHost: string[] | string | null, host?: string | null): boolean { + const blockedHosts = Array.isArray(blockedHostsOrHost) ? blockedHostsOrHost : this.meta.blockedHosts; + host = Array.isArray(blockedHostsOrHost) ? host : blockedHostsOrHost; + if (host == null) return false; return blockedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); } + public isSilencedHost(host: string | null): boolean; + public isSilencedHost(silencedHosts: string[], host: string | null): boolean; @bindThis - public isSilencedHost(silencedHosts: string[] | undefined, host: string | null): boolean { - if (!silencedHosts || host == null) return false; + public isSilencedHost(silencedHostsOrHost: string[] | string | null, host?: string | null): boolean { + const silencedHosts = Array.isArray(silencedHostsOrHost) ? silencedHostsOrHost : this.meta.silencedHosts; + host = Array.isArray(silencedHostsOrHost) ? host : silencedHostsOrHost; + + if (host == null) return false; return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); } + public isMediaSilencedHost(host: string | null): boolean; + public isMediaSilencedHost(silencedHosts: string[], host: string | null): boolean; @bindThis - public isMediaSilencedHost(silencedHosts: string[] | undefined, host: string | null): boolean { - if (!silencedHosts || host == null) return false; - return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); + public isMediaSilencedHost(mediaSilencedHostsOrHost: string[] | string | null, host?: string | null): boolean { + const mediaSilencedHosts = Array.isArray(mediaSilencedHostsOrHost) ? mediaSilencedHostsOrHost : this.meta.mediaSilencedHosts; + host = Array.isArray(mediaSilencedHostsOrHost) ? host : mediaSilencedHostsOrHost; + + if (host == null) return false; + return mediaSilencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); + } + + @bindThis + public isAllowListedHost(host: string | null): boolean { + if (host == null) return false; + return this.meta.federationHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); + } + + @bindThis + public isBubbledHost(host: string | null): boolean { + if (host == null) return false; + return this.meta.bubbleInstances.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); } @bindThis diff --git a/packages/backend/src/core/VideoProcessingService.ts b/packages/backend/src/core/VideoProcessingService.ts index 747fe4fc7e..3e4fd6a4b0 100644 --- a/packages/backend/src/core/VideoProcessingService.ts +++ b/packages/backend/src/core/VideoProcessingService.ts @@ -3,24 +3,41 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +import fs from 'node:fs/promises'; import { Inject, Injectable } from '@nestjs/common'; import FFmpeg from 'fluent-ffmpeg'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { ImageProcessingService } from '@/core/ImageProcessingService.js'; import type { IImage } from '@/core/ImageProcessingService.js'; -import { createTempDir } from '@/misc/create-temp.js'; +import { createTemp, createTempDir } from '@/misc/create-temp.js'; import { bindThis } from '@/decorators.js'; import { appendQuery, query } from '@/misc/prelude/url.js'; +import { LoggerService } from '@/core/LoggerService.js'; +import type Logger from '@/logger.js'; + +// faststart is only supported for MP4, M4A, M4W and MOV files (the MOV family). +// WebM (and Matroska) files always support faststart-like behavior. +const supportedMimeTypes = new Map([ + ['video/mp4', 'mp4'], + ['video/m4a', 'mp4'], + ['video/m4v', 'mp4'], + ['video/quicktime', 'mov'], +]); @Injectable() export class VideoProcessingService { + private readonly logger: Logger; + constructor( @Inject(DI.config) private config: Config, private imageProcessingService: ImageProcessingService, + + private loggerService: LoggerService, ) { + this.logger = this.loggerService.getLogger('video-processing'); } @bindThis @@ -60,5 +77,50 @@ export class VideoProcessingService { }), ); } + + /** + * Optimize video for web playback by adding faststart flag. + * This allows the video to start playing before it is fully downloaded. + * The original file is modified in-place. + * @param source Path to the video file + * @param mimeType The MIME type of the video + * @returns Promise that resolves when optimization is complete + */ + @bindThis + public async webOptimizeVideo(source: string, mimeType: string): Promise { + const outputFormat = supportedMimeTypes.get(mimeType); + if (!outputFormat) { + this.logger.debug(`Skipping web optimization for unsupported MIME type: ${mimeType}`); + return; + } + + const [tempPath, cleanup] = await createTemp(); + + try { + await new Promise((resolve, reject) => { + FFmpeg(source) + .format(outputFormat) // Specify output format + .addOutputOptions('-c copy') // Copy streams without re-encoding + .addOutputOptions('-movflags +faststart') + .on('error', reject) + .on('end', async () => { + try { + // Replace original file with optimized version + await fs.copyFile(tempPath, source); + this.logger.info(`Web-optimized video: ${source}`); + resolve(); + } catch (copyError) { + reject(copyError); + } + }) + .save(tempPath); + }); + } catch (error) { + this.logger.warn(`Failed to web-optimize video: ${source}`, { error }); + throw error; + } finally { + cleanup(); + } + } } diff --git a/packages/backend/src/core/WebhookTestService.ts b/packages/backend/src/core/WebhookTestService.ts index 406998dbc7..8c1508df24 100644 --- a/packages/backend/src/core/WebhookTestService.ts +++ b/packages/backend/src/core/WebhookTestService.ts @@ -63,6 +63,7 @@ function generateDummyUser(override?: Partial): MiUser { emojis: [], score: 0, host: null, + instance: null, inbox: null, sharedInbox: null, featured: null, @@ -115,10 +116,13 @@ function generateDummyNote(override?: Partial): MiNote { channelId: null, channel: null, userHost: null, + userInstance: null, replyUserId: null, replyUserHost: null, + replyUserInstance: null, renoteUserId: null, renoteUserHost: null, + renoteUserInstance: null, updatedAt: null, processErrors: [], ...override, @@ -450,6 +454,7 @@ export class WebhookTestService { isAdmin: false, isModerator: false, isSystem: false, + instance: undefined, ...override, }; } diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index b8526a972c..c06939eae2 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -36,7 +36,7 @@ import InstanceChart from '@/core/chart/charts/instance.js'; import FederationChart from '@/core/chart/charts/federation.js'; import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; -import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js'; +import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js'; import { ApNoteService } from './models/ApNoteService.js'; import { ApLoggerService } from './ApLoggerService.js'; import { ApDbResolverService } from './ApDbResolverService.js'; @@ -106,22 +106,25 @@ export class ApInboxService { let result = undefined as string | void; if (isCollectionOrOrderedCollection(activity)) { const results = [] as [string, string | void][]; - // eslint-disable-next-line no-param-reassign resolver ??= this.apResolverService.createResolver(); - const items = toArray(isCollection(activity) ? activity.items : activity.orderedItems); - if (items.length >= resolver.getRecursionLimit()) { - throw new Error(`skipping activity: collection would surpass recursion limit: ${this.utilityService.extractDbHost(actor.uri)}`); - } - - for (const item of items) { - const act = await resolver.resolve(item); - if (act.id == null || this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) { - this.logger.debug('skipping activity: activity id is null or mismatching'); - continue; + const items = await resolver.resolveCollectionItems(activity); + for (let i = 0; i < items.length; i++) { + const act = items[i]; + if (act.id != null) { + if (this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) { + this.logger.warn('skipping activity: activity id mismatch'); + continue; + } + } else { + // Activity ID should only be string or undefined. + act.id = undefined; } + try { - results.push([getApId(item), await this.performOneActivity(actor, act, resolver)]); + const id = getNullableApId(act) ?? `${getNullableApId(activity)}#${i}`; + const result = await this.performOneActivity(actor, act, resolver); + results.push([id, result]); } catch (err) { if (err instanceof Error || typeof err === 'string') { this.logger.error(err); @@ -217,6 +220,10 @@ export class ApInboxService { const note = await this.apNoteService.resolveNote(object, { resolver }); if (!note) return `skip: target note not found ${targetUri}`; + if (note.userHost == null && note.localOnly) { + throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot react to local-only note'); + } + await this.apNoteService.extractEmojis(activity.tag ?? [], actor.host).catch(() => null); try { @@ -371,6 +378,10 @@ export class ApInboxService { return 'skip: invalid actor for this activity'; } + if (renote.userHost == null && renote.localOnly) { + throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot renote a local-only note'); + } + this.logger.info(`Creating the (Re)Note: ${uri}`); const activityAudience = await this.apAudienceService.parseAudience(actor, activity.to, activity.cc, resolver); diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 7118ce1e02..4c7cac2169 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -155,6 +155,8 @@ export class ApRequestService { @bindThis public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, digest?: string): Promise { + this.apUtilityService.assertApUrl(url); + const body = typeof object === 'string' ? object : JSON.stringify(object); const keypair = await this.userKeypairService.getUserKeypair(user.id); @@ -182,10 +184,13 @@ export class ApRequestService { * Get AP object with http-signature * @param user http-signature user * @param url URL to fetch - * @param followAlternate + * @param allowAnonymous If a fetched object lacks an ID, then it will be auto-generated from the final URL. (default: false) + * @param followAlternate Whether to resolve HTML responses to their referenced canonical AP endpoint. (default: true) */ @bindThis - public async signedGet(url: string, user: { id: MiUser['id'] }, followAlternate?: boolean): Promise { + public async signedGet(url: string, user: { id: MiUser['id'] }, allowAnonymous = false, followAlternate?: boolean): Promise { + this.apUtilityService.assertApUrl(url); + const _followAlternate = followAlternate ?? true; const keypair = await this.userKeypairService.getUserKeypair(user.id); @@ -254,7 +259,7 @@ export class ApRequestService { if (alternate) { const href = alternate.getAttribute('href'); if (href && this.apUtilityService.haveSameAuthority(url, href)) { - return await this.signedGet(href, user, false); + return await this.signedGet(href, user, allowAnonymous, false); } } } catch { @@ -271,7 +276,11 @@ export class ApRequestService { // Make sure the object ID matches the final URL (which is where it actually exists). // The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match. - this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); + if (allowAnonymous && activity.id == null) { + activity.id = res.url; + } else { + this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); + } return activity as IObjectWithId; } diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index 5e58f848c0..7997eccced 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -5,6 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { IsNull, Not } from 'typeorm'; +import promiseLimit from 'promise-limit'; import type { MiLocalUser, MiRemoteUser } from '@/models/User.js'; import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta, SkApFetchLog } from '@/models/_.js'; import type { Config } from '@/config.js'; @@ -19,11 +20,12 @@ import { ApLogService, calculateDurationSince, extractObjectContext } from '@/co import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; -import { getApId, getNullableApId, IObjectWithId, isCollectionOrOrderedCollection } from './type.js'; +import { toArray } from '@/misc/prelude/array.js'; +import { AnyCollection, getApId, getNullableApId, IObjectWithId, isCollection, isCollectionOrOrderedCollection, isCollectionPage, isOrderedCollection, isOrderedCollectionPage } from './type.js'; import { ApDbResolverService } from './ApDbResolverService.js'; import { ApRendererService } from './ApRendererService.js'; import { ApRequestService } from './ApRequestService.js'; -import type { IObject, ICollection, IOrderedCollection, ApObject } from './type.js'; +import type { IObject, ApObject, IAnonymousObject } from './type.js'; export class Resolver { private history: Set; @@ -63,11 +65,16 @@ export class Resolver { return this.recursionLimit; } + public async resolveCollection(value: string | IObjectWithId, allowAnonymous?: boolean, sentFromUri?: string): Promise; + public async resolveCollection(value: string | IObject, allowAnonymous: boolean | undefined, sentFromUri: string): Promise; + public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise; @bindThis - public async resolveCollection(value: string | IObject): Promise { + public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise { const collection = typeof value === 'string' - ? await this.resolve(value) - : value; + ? sentFromUri + ? await this.secureResolve(value, sentFromUri, allowAnonymous) + : await this.resolve(value, allowAnonymous) + : value; // TODO try and remove this eventually, as it's a major security foot-gun if (isCollectionOrOrderedCollection(collection)) { return collection; @@ -76,20 +83,110 @@ export class Resolver { } } + public async resolveCollectionItems(collection: IAnonymousObject, limit?: number | null, allowAnonymousItems?: true, concurrency?: number): Promise; + public async resolveCollectionItems(collection: string | IObjectWithId, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise; + public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise; + /** + * Recursively resolves items from a collection. + * Stops when reaching the resolution limit or an optional item limit - whichever is lower. + * This method supports Collection, OrderedCollection, and individual pages of either type. + * Malformed collections (mixing Ordered and un-Ordered types) are also supported. + * @param collection Collection to resolve from - can be a URL or object of any supported collection type. + * @param limit Maximum number of items to resolve. If null or undefined (default), then items will be resolved until reaching the recursion limit. + * @param allowAnonymousItems If true, collection items can be anonymous (lack an ID). If false (default), then an error is thrown when reaching an item without ID. + * @param concurrency Maximum number of items to resolve at once. (default: 4) + */ + @bindThis + public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency = 4): Promise { + const resolvedItems: IObject[] = []; + + // This is pulled up to avoid code duplication below + const iterate = async(items: ApObject, current: AnyCollection) => { + const sentFrom = current.id; + const itemArr = toArray(items); + const itemLimit = limit ?? Number.MAX_SAFE_INTEGER; + const allowAnonymous = allowAnonymousItems ?? false; + await this.resolveItemArray(itemArr, sentFrom, itemLimit, concurrency, allowAnonymous, resolvedItems); + }; + + let current: AnyCollection | null = await this.resolveCollection(collection); + do { + // Iterate all items in the current page + if (current.items) { + await iterate(current.items, current); + } + if (current.orderedItems) { + await iterate(current.orderedItems, current); + } + + if (this.history.size >= this.recursionLimit) { + // Stop when we reach the fetch limit + current = null; + } else if (limit != null && resolvedItems.length >= limit) { + // Stop when we reach the item limit + current = null; + } else if (isCollection(current) || isOrderedCollection(current)) { + // Continue to first page + current = current.first ? await this.resolveCollection(current.first, true, current.id) : null; + } else if (isCollectionPage(current) || isOrderedCollectionPage(current)) { + // Continue to next page + current = current.next ? await this.resolveCollection(current.next, true, current.id) : null; + } else { + // Stop in all other conditions + current = null; + } + } while (current != null); + + return resolvedItems; + } + + private async resolveItemArray(source: (string | IObject)[], sentFrom: undefined, itemLimit: number, concurrency: number, allowAnonymousItems: true, destination: IAnonymousObject[]): Promise; + private async resolveItemArray(source: (string | IObject)[], sentFrom: string, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObjectWithId[]): Promise; + private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise; + private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise { + const recursionLimit = this.recursionLimit - this.history.size; + const batchLimit = Math.min(source.length, recursionLimit, itemLimit); + + const limiter = promiseLimit(concurrency); + const batch = await Promise.all(source + .slice(0, batchLimit) + .map(item => limiter(async () => { + if (sentFrom) { + // Use secureResolve to avoid re-fetching items that were included inline. + return await this.secureResolve(item, sentFrom, allowAnonymousItems); + } else if (allowAnonymousItems) { + return await this.resolveAnonymous(item); + } else { + // ID is required if we have neither sentFrom not allowAnonymousItems + const id = getApId(item); + return await this.resolve(id); + } + }))); + + destination.push(...batch); + }; + /** * Securely resolves an AP object or URL that has been sent from another instance. * An input object is trusted if and only if its ID matches the authority of sentFromUri. * In all other cases, the object is re-fetched from remote by input string or object ID. + * @param input The input object or URL to resolve + * @param sentFromUri The URL where this object originated. This MUST be accurate - all security checks depend on this value! + * @param allowAnonymous If true, anonymous objects are allowed and will have their ID set to sentFromUri. If false (default) then anonymous objects will be rejected with an error. */ @bindThis - public async secureResolve(input: ApObject, sentFromUri: string): Promise { + public async secureResolve(input: string | IObject | [string | IObject], sentFromUri: string, allowAnonymous?: boolean): Promise { // Unpack arrays to get the value element. const value = fromTuple(input); - if (value == null) { - throw new IdentifiableError('20058164-9de1-4573-8715-425753a21c1d', 'Cannot resolve null input'); + + // If anonymous input is allowed, then any object is automatically valid if we set the ID. + // We can short-circuit here and avoid un-necessary checks. + if (allowAnonymous && typeof(value) === 'object' && value.id == null) { + value.id = sentFromUri; + return value as IObjectWithId; } - // This will throw if the input has no ID, which is good because we can't verify an anonymous object anyway. + // This ensures the input has a string ID, protecting against type confusion and rejecting anonymous objects. const id = getApId(value); // Check if we can use the provided object as-is. @@ -100,28 +197,52 @@ export class Resolver { } // If the checks didn't pass, then we must fetch the object and use that. - return await this.resolve(id); + return await this.resolve(id, allowAnonymous); } - public async resolve(value: string | [string]): Promise; - public async resolve(value: string | IObject | [string | IObject]): Promise; + /** + * Resolves an anonymous object. + * The returned value will not have any ID present. + * If one is provided in the response, it will be removed automatically. + */ @bindThis - public async resolve(value: string | IObject | [string | IObject]): Promise { + public async resolveAnonymous(value: string | IObject | [string | IObject]): Promise { value = fromTuple(value); + const object = await this.resolve(value); + object.id = undefined; + + return object as IAnonymousObject; + } + + public async resolve(value: string | [string], allowAnonymous?: boolean): Promise; + public async resolve(value: string | IObjectWithId | [string | IObjectWithId], allowAnonymous?: boolean): Promise; + public async resolve(value: string | IObject | [string | IObject], allowAnonymous?: boolean): Promise; + /** + * Resolves a URL or object to an AP object. + * Tuples are expanded to their first element before anything else, and non-string inputs are returned as-is. + * Otherwise, the string URL is fetched and validated to represent a valid ActivityPub object. + * @param value The input value to resolve + * @param allowAnonymous Determines what to do if a response object lacks an ID field. If false (default), then an exception is thrown. If true, then the ID is populated from the final response URL. + */ + @bindThis + public async resolve(value: string | IObject | [string | IObject], allowAnonymous = false): Promise { + value = fromTuple(value); + + // TODO try and remove this eventually, as it's a major security foot-gun if (typeof value !== 'string') { return value; } const host = this.utilityService.extractDbHost(value); if (this.config.activityLogging.enabled && !this.utilityService.isSelfHost(host)) { - return await this._resolveLogged(value, host); + return await this._resolveLogged(value, host, allowAnonymous); } else { - return await this._resolve(value, host); + return await this._resolve(value, host, allowAnonymous); } } - private async _resolveLogged(requestUri: string, host: string): Promise { + private async _resolveLogged(requestUri: string, host: string, allowAnonymous: boolean): Promise { const startTime = process.hrtime.bigint(); const log = await this.apLogService.createFetchLog({ @@ -130,7 +251,7 @@ export class Resolver { }); try { - const result = await this._resolve(requestUri, host, log); + const result = await this._resolve(requestUri, host, allowAnonymous, log); log.accepted = true; log.result = 'ok'; @@ -150,7 +271,7 @@ export class Resolver { } } - private async _resolve(value: string, host: string, log?: SkApFetchLog): Promise { + private async _resolve(value: string, host: string, allowAnonymous: boolean, log?: SkApFetchLog): Promise { if (value.includes('#')) { // URLs with fragment parts cannot be resolved correctly because // the fragment part does not get transmitted over HTTP(S). @@ -181,8 +302,8 @@ export class Resolver { } const object = (this.user - ? await this.apRequestService.signedGet(value, this.user) - : await this.httpRequestService.getActivityJson(value)); + ? await this.apRequestService.signedGet(value, this.user, allowAnonymous) + : await this.httpRequestService.getActivityJson(value, false, allowAnonymous)); if (log) { const { object: objectOnly, context, contextHash } = extractObjectContext(object); diff --git a/packages/backend/src/core/activitypub/ApUtilityService.ts b/packages/backend/src/core/activitypub/ApUtilityService.ts index ae6e4997e4..3c125c6cd9 100644 --- a/packages/backend/src/core/activitypub/ApUtilityService.ts +++ b/packages/backend/src/core/activitypub/ApUtilityService.ts @@ -77,16 +77,42 @@ export class ApUtilityService { return acceptableUrls[0]?.url ?? null; } + /** + * Verifies that a provided URL is in a format acceptable for federation. + * @throws {IdentifiableError} If URL cannot be parsed + * @throws {IdentifiableError} If URL is not HTTPS + */ + public assertApUrl(url: string | URL): void { + // If string, parse and validate + if (typeof(url) === 'string') { + try { + url = new URL(url); + } catch { + throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: not a valid URL`); + } + } + + // Must be HTTPS + if (!this.checkHttps(url)) { + throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: unsupported protocol ${url.protocol}`); + } + } + /** * Checks if the URL contains HTTPS. * Additionally, allows HTTP in non-production environments. * Based on check-https.ts. */ - private checkHttps(url: string): boolean { + private checkHttps(url: string | URL): boolean { const isNonProd = this.envService.env.NODE_ENV !== 'production'; - // noinspection HttpUrlsUsage - return url.startsWith('https://') || (url.startsWith('http://') && isNonProd); + try { + const proto = new URL(url).protocol; + return proto === 'https:' || (proto === 'http:' && isNonProd); + } catch { + // Invalid URLs don't "count" as HTTPS + return false; + } } } diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index f6152e3888..5b66031bee 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -95,6 +95,7 @@ export class ApNoteService { actor?: MiRemoteUser, user?: MiRemoteUser, ): Error | null { + this.apUtilityService.assertApUrl(uri); const expectHost = this.utilityService.extractDbHost(uri); const apType = getApType(object); @@ -284,6 +285,13 @@ export class ApNoteService { const quote = await this.getQuote(note, entryUri, resolver); const processErrors = quote === null ? ['quoteUnavailable'] : null; + if (reply && reply.userHost == null && reply.localOnly) { + throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot reply to local-only note'); + } + if (quote && quote.userHost == null && quote.localOnly) { + throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot quote a local-only note'); + } + // vote if (reply && reply.hasPoll) { const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id }); @@ -481,6 +489,10 @@ export class ApNoteService { const quote = await this.getQuote(note, entryUri, resolver); const processErrors = quote === null ? ['quoteUnavailable'] : null; + if (quote && quote.userHost == null && quote.localOnly) { + throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot quote a local-only note'); + } + // vote if (reply && reply.hasPoll) { const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id }); diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index bf5a5e01d5..ef8b58ff87 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -153,6 +153,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { */ @bindThis private validateActor(x: IObject, uri: string): IActor { + this.apUtilityService.assertApUrl(uri); const expectHost = this.utilityService.punyHostPSLDomain(uri); if (!isActor(x)) { @@ -167,6 +168,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox type`); } + this.apUtilityService.assertApUrl(x.inbox); const inboxHost = this.utilityService.punyHostPSLDomain(x.inbox); if (inboxHost !== expectHost) { throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox ${inboxHost}`); @@ -175,6 +177,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const sharedInboxObject = x.sharedInbox ?? (x.endpoints ? x.endpoints.sharedInbox : undefined); if (sharedInboxObject != null) { const sharedInbox = getApId(sharedInboxObject); + this.apUtilityService.assertApUrl(sharedInbox); if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHostPSLDomain(sharedInbox) === expectHost)) { throw new UnrecoverableError(`invalid Actor ${uri} - wrong shared inbox ${sharedInbox}`); } @@ -185,6 +188,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { if (xCollection != null) { const collectionUri = getApId(xCollection); if (typeof collectionUri === 'string' && collectionUri.length > 0) { + this.apUtilityService.assertApUrl(collectionUri); if (this.utilityService.punyHostPSLDomain(collectionUri) !== expectHost) { throw new UnrecoverableError(`invalid Actor ${uri} - wrong ${collection} ${collectionUri}`); } @@ -352,8 +356,8 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const [followingVisibility, followersVisibility] = await Promise.all( [ - this.isPublicCollection(person.following, resolver), - this.isPublicCollection(person.followers, resolver), + this.isPublicCollection(person.following, resolver, uri), + this.isPublicCollection(person.followers, resolver, uri), ].map((p): Promise<'public' | 'private'> => p .then(isPublic => isPublic ? 'public' : 'private') .catch(err => { @@ -389,10 +393,18 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { //#endregion //#region resolve counts - const _resolver = resolver ?? this.apResolverService.createResolver(); - const outboxcollection = await _resolver.resolveCollection(person.outbox).catch(() => { return null; }); - const followerscollection = await _resolver.resolveCollection(person.followers!).catch(() => { return null; }); - const followingcollection = await _resolver.resolveCollection(person.following!).catch(() => { return null; }); + const outboxCollection = person.outbox + ? await resolver.resolveCollection(person.outbox, true, uri).catch(() => { return null; }) + : null; + const followersCollection = person.followers + ? await resolver.resolveCollection(person.followers, true, uri).catch(() => { return null; }) + : null; + const followingCollection = person.following + ? await resolver.resolveCollection(person.following, true, uri).catch(() => { return null; }) + : null; + + // Register the instance first, to avoid FK errors + await this.federatedInstanceService.fetchOrRegister(host); try { // Start transaction @@ -419,9 +431,9 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { host, inbox: person.inbox, sharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null, - notesCount: outboxcollection?.totalItems ?? 0, - followersCount: followerscollection?.totalItems ?? 0, - followingCount: followingcollection?.totalItems ?? 0, + notesCount: outboxCollection?.totalItems ?? 0, + followersCount: followersCollection?.totalItems ?? 0, + followingCount: followingCollection?.totalItems ?? 0, followersUri: person.followers ? getApId(person.followers) : undefined, featured: person.featured ? getApId(person.featured) : undefined, uri: person.id, @@ -571,8 +583,8 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const [followingVisibility, followersVisibility] = await Promise.all( [ - this.isPublicCollection(person.following, resolver), - this.isPublicCollection(person.followers, resolver), + this.isPublicCollection(person.following, resolver, exist.uri), + this.isPublicCollection(person.followers, resolver, exist.uri), ].map((p): Promise<'public' | 'private' | undefined> => p .then(isPublic => isPublic ? 'public' : 'private') .catch(err => { @@ -797,13 +809,13 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { const _resolver = resolver ?? this.apResolverService.createResolver(); // Resolve to (Ordered)Collection Object - const collection = await _resolver.resolveCollection(user.featured).catch(err => { + const collection = user.featured ? await _resolver.resolveCollection(user.featured, true, user.uri).catch(err => { if (err instanceof AbortError || err instanceof StatusError) { this.logger.warn(`Failed to update featured notes: ${err.name}: ${err.message}`); } else { this.logger.error('Failed to update featured notes:', err); } - }); + }) : null; if (!collection) return; if (!isCollectionOrOrderedCollection(collection)) throw new UnrecoverableError(`featured ${user.featured} is not Collection or OrderedCollection in ${user.uri}`); @@ -889,11 +901,13 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown { } @bindThis - private async isPublicCollection(collection: string | ICollection | IOrderedCollection | undefined, resolver: Resolver): Promise { + private async isPublicCollection(collection: string | ICollection | IOrderedCollection | undefined, resolver: Resolver, sentFrom: string): Promise { if (collection) { - const resolved = await resolver.resolveCollection(collection); - if (resolved.first || (resolved as ICollection).items || (resolved as IOrderedCollection).orderedItems) { - return true; + const resolved = await resolver.resolveCollection(collection, true, sentFrom).catch(() => null); + if (resolved) { + if (resolved.first || (resolved as ICollection).items || (resolved as IOrderedCollection).orderedItems) { + return true; + } } } diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 0122697f2a..362c3af1e7 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -43,6 +43,18 @@ export interface IObjectWithId extends IObject { id: string; } +export function isObjectWithId(object: IObject): object is IObjectWithId { + return typeof(object.id) === 'string'; +} + +export interface IAnonymousObject extends IObject { + id: undefined; +} + +export function isAnonymousObject(object: IObject): object is IAnonymousObject { + return object.id === undefined; +} + /** * Get array of ActivityStreams Objects id */ @@ -125,48 +137,46 @@ export interface IActivity extends IObject { }; } -export interface ICollection extends IObject { +export interface CollectionBase extends IObject { + totalItems?: number; + first?: IObject | string; + last?: IObject | string; + current?: IObject | string; + partOf?: IObject | string; + next?: IObject | string; + prev?: IObject | string; + items?: ApObject; + orderedItems?: ApObject; +} + +export interface ICollection extends CollectionBase { type: 'Collection'; totalItems: number; - first?: IObject | string; - last?: IObject | string; - current?: IObject | string; items?: ApObject; + orderedItems?: undefined; } -export interface IOrderedCollection extends IObject { +export interface IOrderedCollection extends CollectionBase { type: 'OrderedCollection'; totalItems: number; - first?: IObject | string; - last?: IObject | string; - current?: IObject | string; + items?: undefined; orderedItems?: ApObject; } -export interface ICollectionPage extends IObject { +export interface ICollectionPage extends CollectionBase { type: 'CollectionPage'; - totalItems: number; - first?: IObject | string; - last?: IObject | string; - current?: IObject | string; - partOf?: IObject | string; - next?: IObject | string; - prev?: IObject | string; items?: ApObject; + orderedItems?: undefined; } -export interface IOrderedCollectionPage extends IObject { +export interface IOrderedCollectionPage extends CollectionBase { type: 'OrderedCollectionPage'; - totalItems: number; - first?: IObject | string; - last?: IObject | string; - current?: IObject | string; - partOf?: IObject | string; - next?: IObject | string; - prev?: IObject | string; + items?: undefined; orderedItems?: ApObject; } +export type AnyCollection = ICollection | IOrderedCollection | ICollectionPage | IOrderedCollectionPage; + export const validPost = ['Note', 'Question', 'Article', 'Audio', 'Document', 'Image', 'Page', 'Video', 'Event']; export const isPost = (object: IObject): object is IPost => { @@ -270,7 +280,7 @@ export const isCollectionPage = (object: IObject): object is ICollectionPage => export const isOrderedCollectionPage = (object: IObject): object is IOrderedCollectionPage => getApType(object) === 'OrderedCollectionPage'; -export const isCollectionOrOrderedCollection = (object: IObject): object is ICollection | IOrderedCollection => +export const isCollectionOrOrderedCollection = (object: IObject): object is AnyCollection => isCollection(object) || isOrderedCollection(object) || isCollectionPage(object) || isOrderedCollectionPage(object); export interface IApPropertyValue extends IObject { diff --git a/packages/backend/src/core/chart/charts/federation.ts b/packages/backend/src/core/chart/charts/federation.ts index bf702884ca..b6db6f5454 100644 --- a/packages/backend/src/core/chart/charts/federation.ts +++ b/packages/backend/src/core/chart/charts/federation.ts @@ -44,10 +44,6 @@ export default class FederationChart extends Chart { // eslint-di } protected async tickMinor(): Promise>> { - const suspendedInstancesQuery = this.instancesRepository.createQueryBuilder('instance') - .select('instance.host') - .where('instance.suspensionState != \'none\''); - const pubsubSubQuery = this.followingsRepository.createQueryBuilder('f') .select('f.followerHost') .where('f.followerHost IS NOT NULL'); @@ -64,22 +60,25 @@ export default class FederationChart extends Chart { // eslint-di this.followingsRepository.createQueryBuilder('following') .select('COUNT(DISTINCT following.followeeHost)') .where('following.followeeHost IS NOT NULL') - .andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followeeHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)') - .andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`) + .innerJoin('following.followeeInstance', 'followeeInstance') + .andWhere('followeeInstance.suspensionState = \'none\'') + .andWhere('followeeInstance.isBlocked = false') .getRawOne() .then(x => parseInt(x.count, 10)), this.followingsRepository.createQueryBuilder('following') .select('COUNT(DISTINCT following.followerHost)') .where('following.followerHost IS NOT NULL') - .andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followerHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)') - .andWhere(`following.followerHost NOT IN (${ suspendedInstancesQuery.getQuery() })`) + .innerJoin('following.followerInstance', 'followerInstance') + .andWhere('followerInstance.isBlocked = false') + .andWhere('followerInstance.suspensionState = \'none\'') .getRawOne() .then(x => parseInt(x.count, 10)), this.followingsRepository.createQueryBuilder('following') .select('COUNT(DISTINCT following.followeeHost)') .where('following.followeeHost IS NOT NULL') - .andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followeeHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)') - .andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`) + .innerJoin('following.followeeInstance', 'followeeInstance') + .andWhere('followeeInstance.isBlocked = false') + .andWhere('followeeInstance.suspensionState = \'none\'') .andWhere(`following.followeeHost IN (${ pubsubSubQuery.getQuery() })`) .setParameters(pubsubSubQuery.getParameters()) .getRawOne() @@ -87,7 +86,7 @@ export default class FederationChart extends Chart { // eslint-di this.instancesRepository.createQueryBuilder('instance') .select('COUNT(instance.id)') .where(`instance.host IN (${ subInstancesQuery.getQuery() })`) - .andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || instance.host) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)') + .andWhere('instance.isBlocked = false') .andWhere('instance.suspensionState = \'none\'') .andWhere('instance.isNotResponding = false') .getRawOne() @@ -95,7 +94,7 @@ export default class FederationChart extends Chart { // eslint-di this.instancesRepository.createQueryBuilder('instance') .select('COUNT(instance.id)') .where(`instance.host IN (${ pubInstancesQuery.getQuery() })`) - .andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || instance.host) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)') + .andWhere('instance.isBlocked = false') .andWhere('instance.suspensionState = \'none\'') .andWhere('instance.isNotResponding = false') .getRawOne() diff --git a/packages/backend/src/core/entities/InstanceEntityService.ts b/packages/backend/src/core/entities/InstanceEntityService.ts index fcc9bed3bd..a2ee4b0505 100644 --- a/packages/backend/src/core/entities/InstanceEntityService.ts +++ b/packages/backend/src/core/entities/InstanceEntityService.ts @@ -43,7 +43,7 @@ export class InstanceEntityService { isNotResponding: instance.isNotResponding, isSuspended: instance.suspensionState !== 'none', suspensionState: instance.suspensionState, - isBlocked: this.utilityService.isBlockedHost(this.meta.blockedHosts, instance.host), + isBlocked: instance.isBlocked, softwareName: instance.softwareName, softwareVersion: instance.softwareVersion, openRegistrations: instance.openRegistrations, @@ -51,8 +51,8 @@ export class InstanceEntityService { description: instance.description, maintainerName: instance.maintainerName, maintainerEmail: instance.maintainerEmail, - isSilenced: this.utilityService.isSilencedHost(this.meta.silencedHosts, instance.host), - isMediaSilenced: this.utilityService.isMediaSilencedHost(this.meta.mediaSilencedHosts, instance.host), + isSilenced: instance.isSilenced, + isMediaSilenced: instance.isMediaSilenced, iconUrl: instance.iconUrl, faviconUrl: instance.faviconUrl, themeColor: instance.themeColor, @@ -62,6 +62,7 @@ export class InstanceEntityService { rejectReports: instance.rejectReports, rejectQuotes: instance.rejectQuotes, moderationNote: iAmModerator ? instance.moderationNote : null, + isBubbled: this.utilityService.isBubbledHost(instance.host), }; } diff --git a/packages/backend/src/core/entities/NotificationEntityService.ts b/packages/backend/src/core/entities/NotificationEntityService.ts index 77e6a1c7e7..cc8edfc666 100644 --- a/packages/backend/src/core/entities/NotificationEntityService.ts +++ b/packages/backend/src/core/entities/NotificationEntityService.ts @@ -23,6 +23,13 @@ import type { NoteEntityService } from './NoteEntityService.js'; const NOTE_REQUIRED_NOTIFICATION_TYPES = new Set(['note', 'mention', 'reply', 'renote', 'renote:grouped', 'quote', 'reaction', 'reaction:grouped', 'pollEnded', 'edited', 'scheduledNotePosted'] as (typeof groupedNotificationTypes[number])[]); +function undefOnMissing(packPromise: Promise): Promise { + return packPromise.catch(err => { + if (err instanceof EntityNotFoundError) return undefined; + throw err; + }); +} + @Injectable() export class NotificationEntityService implements OnModuleInit { private userEntityService: UserEntityService; @@ -75,9 +82,9 @@ export class NotificationEntityService implements OnModuleInit { const noteIfNeed = needsNote ? ( hint?.packedNotes != null ? hint.packedNotes.get(notification.noteId) - : this.noteEntityService.pack(notification.noteId, { id: meId }, { + : undefOnMissing(this.noteEntityService.pack(notification.noteId, { id: meId }, { detail: true, - }) + })) ) : undefined; // if the note has been deleted, don't show this notification if (needsNote && !noteIfNeed) return null; @@ -86,7 +93,7 @@ export class NotificationEntityService implements OnModuleInit { const userIfNeed = needsUser ? ( hint?.packedUsers != null ? hint.packedUsers.get(notification.notifierId) - : this.userEntityService.pack(notification.notifierId, { id: meId }) + : undefOnMissing(this.userEntityService.pack(notification.notifierId, { id: meId })) ) : undefined; // if the user has been deleted, don't show this notification if (needsUser && !userIfNeed) return null; @@ -96,7 +103,7 @@ export class NotificationEntityService implements OnModuleInit { const reactions = (await Promise.all(notification.reactions.map(async reaction => { const user = hint?.packedUsers != null ? hint.packedUsers.get(reaction.userId)! - : await this.userEntityService.pack(reaction.userId, { id: meId }); + : await undefOnMissing(this.userEntityService.pack(reaction.userId, { id: meId })); return { user, reaction: reaction.reaction, @@ -121,7 +128,7 @@ export class NotificationEntityService implements OnModuleInit { return packedUser; } - return this.userEntityService.pack(userId, { id: meId }); + return undefOnMissing(this.userEntityService.pack(userId, { id: meId })); }))).filter(x => x != null); // if all users have been deleted, don't show this notification if (users.length === 0) { @@ -140,10 +147,7 @@ export class NotificationEntityService implements OnModuleInit { const needsRole = notification.type === 'roleAssigned'; const role = needsRole - ? await this.roleEntityService.pack(notification.roleId).catch(err => { - if (err instanceof EntityNotFoundError) return undefined; - throw err; - }) + ? await undefOnMissing(this.roleEntityService.pack(notification.roleId)) : undefined; // if the role has been deleted, don't show this notification if (needsRole && !role) { diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index f66a36336d..f6aeb0ef8b 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -610,7 +610,7 @@ export class UserEntityService implements OnModuleInit { requireSigninToViewContents: user.requireSigninToViewContents === false ? undefined : true, makeNotesFollowersOnlyBefore: user.makeNotesFollowersOnlyBefore ?? undefined, makeNotesHiddenBefore: user.makeNotesHiddenBefore ?? undefined, - instance: user.host ? this.federatedInstanceService.federatedInstanceCache.fetch(user.host).then(instance => instance ? { + instance: user.host ? this.federatedInstanceService.fetch(user.host).then(instance => instance ? { name: instance.name, softwareName: instance.softwareName, softwareVersion: instance.softwareVersion, diff --git a/packages/backend/src/env.ts b/packages/backend/src/env.ts index ba44cfa2e6..9a50eb8561 100644 --- a/packages/backend/src/env.ts +++ b/packages/backend/src/env.ts @@ -11,6 +11,7 @@ const envOption = { verbose: false, withLogTime: false, quiet: false, + hideWorkerId: false, }; for (const key of Object.keys(envOption) as (keyof typeof envOption)[]) { diff --git a/packages/backend/src/logger.ts b/packages/backend/src/logger.ts index b3735200eb..ca9b494ff2 100644 --- a/packages/backend/src/logger.ts +++ b/packages/backend/src/logger.ts @@ -71,7 +71,9 @@ export default class Logger { level === 'info' ? message : null; - let log = `${l} ${worker}\t[${contexts.join(' ')}]\t${m}`; + let log = envOption.hideWorkerId + ? `${l}\t[${contexts.join(' ')}]\t\t${m}` + : `${l} ${worker}\t[${contexts.join(' ')}]\t\t${m}`; if (envOption.withLogTime) log = chalk.gray(time) + ' ' + log; const args: unknown[] = [important ? chalk.bold(log) : log]; diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 48b8f43678..a6ab96c189 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -308,8 +308,17 @@ export class MemoryKVCache { } } + /** + * Removes all entries from the cache, but does not dispose it. + */ + @bindThis + public clear(): void { + this.cache.clear(); + } + @bindThis public dispose(): void { + this.clear(); clearInterval(this.gcIntervalHandle); } diff --git a/packages/backend/src/misc/diff-arrays.ts b/packages/backend/src/misc/diff-arrays.ts new file mode 100644 index 0000000000..b50ca1d4f7 --- /dev/null +++ b/packages/backend/src/misc/diff-arrays.ts @@ -0,0 +1,102 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export interface DiffResult { + added: T[]; + removed: T[]; +} + +/** + * Calculates the difference between two snapshots of data. + * Null, undefined, and empty arrays are supported, and duplicate values are ignored. + * Result sets are de-duplicated, and will be empty if no data was added or removed (respectively). + * The inputs are treated as un-ordered, so a re-ordering of the same data will NOT be considered a change. + * @param dataBefore Array containing data before the change + * @param dataAfter Array containing data after the change + */ +export function diffArrays(dataBefore: T[] | null | undefined, dataAfter: T[] | null | undefined): DiffResult { + const before = dataBefore ? new Set(dataBefore) : null; + const after = dataAfter ? new Set(dataAfter) : null; + + // data before AND after => changed + if (before?.size && after?.size) { + const added: T[] = []; + const removed: T[] = []; + + for (const host of before) { + // before and NOT after => removed + // delete operation removes duplicates to speed up the "after" loop + if (!after.delete(host)) { + removed.push(host); + } + } + + for (const host of after) { + // after and NOT before => added + if (!before.has(host)) { + added.push(host); + } + } + + return { added, removed }; + } + + // data ONLY before => all removed + if (before?.size) { + return { added: [], removed: Array.from(before) }; + } + + // data ONLY after => all added + if (after?.size) { + return { added: Array.from(after), removed: [] }; + } + + // data NEITHER before nor after => no change + return { added: [], removed: [] }; +} + +/** + * Checks for any difference between two snapshots of data. + * Null, undefined, and empty arrays are supported, and duplicate values are ignored. + * The inputs are treated as un-ordered, so a re-ordering of the same data will NOT be considered a change. + * @param dataBefore Array containing data before the change + * @param dataAfter Array containing data after the change + */ +export function diffArraysSimple(dataBefore: T[] | null | undefined, dataAfter: T[] | null | undefined): boolean { + const before = dataBefore ? new Set(dataBefore) : null; + const after = dataAfter ? new Set(dataAfter) : null; + + if (before?.size && after?.size) { + // different size => changed + if (before.size !== after.size) return true; + + // removed => changed + for (const host of before) { + // delete operation removes duplicates to speed up the "after" loop + if (!after.delete(host)) { + return true; + } + } + + // added => changed + for (const host of after) { + if (!before.has(host)) { + return true; + } + } + + // identical values => no change + return false; + } + + // before and NOT after => change + if (before?.size) return true; + + // after and NOT before => change + if (after?.size) return true; + + // NEITHER before nor after => no change + return false; +} diff --git a/packages/backend/src/models/Following.ts b/packages/backend/src/models/Following.ts index 62cbc29f26..0aa1b13976 100644 --- a/packages/backend/src/models/Following.ts +++ b/packages/backend/src/models/Following.ts @@ -4,6 +4,7 @@ */ import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm'; +import { MiInstance } from '@/models/Instance.js'; import { id } from './util/id.js'; import { MiUser } from './User.js'; @@ -66,6 +67,16 @@ export class MiFollowing { }) public followerHost: string | null; + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'followerHost', + foreignKeyConstraintName: 'FK_following_followerHost', + referencedColumnName: 'host', + }) + public followerInstance: MiInstance | null; + @Column('varchar', { length: 512, nullable: true, comment: '[Denormalized]', @@ -85,6 +96,16 @@ export class MiFollowing { }) public followeeHost: string | null; + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'followeeHost', + foreignKeyConstraintName: 'FK_following_followeeHost', + referencedColumnName: 'host', + }) + public followeeInstance: MiInstance | null; + @Column('varchar', { length: 512, nullable: true, comment: '[Denormalized]', diff --git a/packages/backend/src/models/Instance.ts b/packages/backend/src/models/Instance.ts index c64ebb1b3b..0022e58933 100644 --- a/packages/backend/src/models/Instance.ts +++ b/packages/backend/src/models/Instance.ts @@ -6,6 +6,7 @@ import { Entity, PrimaryColumn, Index, Column } from 'typeorm'; import { id } from './util/id.js'; +@Index('IDX_instance_host_key', { synchronize: false }) @Entity('instance') export class MiInstance { @PrimaryColumn(id()) @@ -98,6 +99,56 @@ export class MiInstance { }) public suspensionState: 'none' | 'manuallySuspended' | 'goneSuspended' | 'autoSuspendedForNotResponding'; + /** + * True if this instance is blocked from federation. + */ + @Column('boolean', { + nullable: false, + default: false, + comment: 'True if this instance is blocked from federation.', + }) + public isBlocked: boolean; + + /** + * True if this instance is allow-listed. + */ + @Column('boolean', { + nullable: false, + default: false, + comment: 'True if this instance is allow-listed.', + }) + public isAllowListed: boolean; + + /** + * True if this instance is part of the local bubble. + */ + @Column('boolean', { + nullable: false, + default: false, + comment: 'True if this instance is part of the local bubble.', + }) + public isBubbled: boolean; + + /** + * True if this instance is silenced. + */ + @Column('boolean', { + nullable: false, + default: false, + comment: 'True if this instance is silenced.', + }) + public isSilenced: boolean; + + /** + * True if this instance is media-silenced. + */ + @Column('boolean', { + nullable: false, + default: false, + comment: 'True if this instance is media-silenced.', + }) + public isMediaSilenced: boolean; + @Column('varchar', { length: 64, nullable: true, comment: 'The software of the Instance.', diff --git a/packages/backend/src/models/Note.ts b/packages/backend/src/models/Note.ts index ee2098216d..fa5839b6ec 100644 --- a/packages/backend/src/models/Note.ts +++ b/packages/backend/src/models/Note.ts @@ -5,6 +5,7 @@ import { Entity, Index, JoinColumn, Column, PrimaryColumn, ManyToOne } from 'typeorm'; import { noteVisibilities } from '@/types.js'; +import { MiInstance } from '@/models/Instance.js'; import { id } from './util/id.js'; import { MiUser } from './User.js'; import { MiChannel } from './Channel.js'; @@ -222,6 +223,16 @@ export class MiNote { }) public userHost: string | null; + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'userHost', + foreignKeyConstraintName: 'FK_note_userHost', + referencedColumnName: 'host', + }) + public userInstance: MiInstance | null; + @Column({ ...id(), nullable: true, @@ -235,6 +246,16 @@ export class MiNote { }) public replyUserHost: string | null; + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'replyUserHost', + foreignKeyConstraintName: 'FK_note_replyUserHost', + referencedColumnName: 'host', + }) + public replyUserInstance: MiInstance | null; + @Column({ ...id(), nullable: true, @@ -247,6 +268,16 @@ export class MiNote { comment: '[Denormalized]', }) public renoteUserHost: string | null; + + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'renoteUserHost', + foreignKeyConstraintName: 'FK_note_renoteUserHost', + referencedColumnName: 'host', + }) + public renoteUserInstance: MiInstance | null; //#endregion constructor(data: Partial) { diff --git a/packages/backend/src/models/User.ts b/packages/backend/src/models/User.ts index d5f572a879..3ef5817672 100644 --- a/packages/backend/src/models/User.ts +++ b/packages/backend/src/models/User.ts @@ -3,8 +3,9 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Entity, Column, Index, OneToOne, JoinColumn, PrimaryColumn } from 'typeorm'; +import { Entity, Column, Index, OneToOne, JoinColumn, PrimaryColumn, ManyToOne } from 'typeorm'; import { type UserUnsignedFetchOption, userUnsignedFetchOptions } from '@/const.js'; +import { MiInstance } from '@/models/Instance.js'; import { id } from './util/id.js'; import { MiDriveFile } from './DriveFile.js'; @@ -292,6 +293,16 @@ export class MiUser { }) public host: string | null; + @ManyToOne(() => MiInstance, { + onDelete: 'CASCADE', + }) + @JoinColumn({ + name: 'host', + foreignKeyConstraintName: 'FK_user_host', + referencedColumnName: 'host', + }) + public instance: MiInstance | null; + @Column('varchar', { length: 512, nullable: true, comment: 'The inbox URL of the User. It will be null if the origin of the user is local.', diff --git a/packages/backend/src/models/json-schema/federation-instance.ts b/packages/backend/src/models/json-schema/federation-instance.ts index 57d4466ffa..fd6eddf594 100644 --- a/packages/backend/src/models/json-schema/federation-instance.ts +++ b/packages/backend/src/models/json-schema/federation-instance.ts @@ -135,5 +135,9 @@ export const packedFederationInstanceSchema = { type: 'string', optional: true, nullable: true, }, + isBubbled: { + type: 'boolean', + optional: false, nullable: false, + }, }, } as const; diff --git a/packages/backend/src/postgres.ts b/packages/backend/src/postgres.ts index 632fd58927..b4bd934972 100644 --- a/packages/backend/src/postgres.ts +++ b/packages/backend/src/postgres.ts @@ -98,9 +98,12 @@ pg.types.setTypeParser(20, Number); export const dbLogger = new MisskeyLogger('db'); const sqlLogger = dbLogger.createSubLogger('sql', 'gray'); +const sqlMigrateLogger = sqlLogger.createSubLogger('migrate'); +const sqlSchemaLogger = sqlLogger.createSubLogger('schema'); export type LoggerProps = { disableQueryTruncation?: boolean; + enableQueryLogging?: boolean; enableQueryParamLogging?: boolean; printReplicationMode?: boolean, }; @@ -112,7 +115,7 @@ function highlightSql(sql: string) { } function truncateSql(sql: string) { - return sql.length > 100 ? `${sql.substring(0, 100)}...` : sql; + return sql.length > 100 ? `${sql.substring(0, 100)} [truncated]` : sql; } function stringifyParameter(param: any) { @@ -136,7 +139,7 @@ class MyCustomLogger implements Logger { modded = truncateSql(modded); } - return highlightSql(modded); + return this.props.enableQueryLogging ? highlightSql(modded) : modded; } @bindThis @@ -150,6 +153,8 @@ class MyCustomLogger implements Logger { @bindThis public logQuery(query: string, parameters?: any[], queryRunner?: QueryRunner) { + if (!this.props.enableQueryLogging) return; + const prefix = (this.props.printReplicationMode && queryRunner) ? `[${queryRunner.getReplicationMode()}] ` : undefined; @@ -161,7 +166,8 @@ class MyCustomLogger implements Logger { const prefix = (this.props.printReplicationMode && queryRunner) ? `[${queryRunner.getReplicationMode()}] ` : undefined; - sqlLogger.error(this.transformQueryLog(query, { prefix }), this.transformParameters(parameters)); + const transformed = this.transformQueryLog(query, { prefix }); + sqlLogger.error(`Query error (${error}): ${transformed}`, this.transformParameters(parameters)); } @bindThis @@ -169,22 +175,32 @@ class MyCustomLogger implements Logger { const prefix = (this.props.printReplicationMode && queryRunner) ? `[${queryRunner.getReplicationMode()}] ` : undefined; - sqlLogger.warn(this.transformQueryLog(query, { prefix }), this.transformParameters(parameters)); + const transformed = this.transformQueryLog(query, { prefix }); + sqlLogger.warn(`Query is slow (${time}ms): ${transformed}`, this.transformParameters(parameters)); } @bindThis public logSchemaBuild(message: string) { - sqlLogger.info(message); + sqlSchemaLogger.debug(message); } @bindThis - public log(message: string) { - sqlLogger.info(message); + public log(level: 'log' | 'info' | 'warn', message: string) { + switch (level) { + case 'log': + case 'info': { + sqlLogger.info(message); + break; + } + case 'warn': { + sqlLogger.warn(message); + } + } } @bindThis public logMigration(message: string) { - sqlLogger.info(message); + sqlMigrateLogger.debug(message); } } @@ -306,7 +322,7 @@ export function createPostgresDataSource(config: Config) { } : {}), synchronize: process.env.NODE_ENV === 'test', dropSchema: process.env.NODE_ENV === 'test', - cache: !config.db.disableCache && process.env.NODE_ENV !== 'test' ? { // dbをcloseしても何故かredisのコネクションが内部的に残り続けるようで、テストの際に支障が出るため無効にする(キャッシュも含めてテストしたいため本当は有効にしたいが...) + cache: config.db.disableCache === false && process.env.NODE_ENV !== 'test' ? { // dbをcloseしても何故かredisのコネクションが内部的に残り続けるようで、テストの際に支障が出るため無効にする(キャッシュも含めてテストしたいため本当は有効にしたいが...) type: 'ioredis', options: { ...config.redis, @@ -314,14 +330,13 @@ export function createPostgresDataSource(config: Config) { }, } : false, logging: log, - logger: log - ? new MyCustomLogger({ - disableQueryTruncation: config.logging?.sql?.disableQueryTruncation, - enableQueryParamLogging: config.logging?.sql?.enableQueryParamLogging, - printReplicationMode: !!config.dbReplications, - }) - : undefined, - maxQueryExecutionTime: 300, + logger: new MyCustomLogger({ + disableQueryTruncation: config.logging?.sql?.disableQueryTruncation, + enableQueryLogging: log, + enableQueryParamLogging: config.logging?.sql?.enableQueryParamLogging, + printReplicationMode: !!config.dbReplications, + }), + maxQueryExecutionTime: config.db.slowQueryThreshold, entities: entities, migrations: ['../../migration/*.js'], }); diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index 0d2dafd556..5c9e5717bb 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -344,14 +344,14 @@ export class ApiCallService implements OnApplicationShutdown { } if (ep.meta.requireCredential || ep.meta.requireModerator || ep.meta.requireAdmin) { - if (user == null) { + if (user == null && ep.meta.requireCredential !== 'optional') { throw new ApiError({ message: 'Credential required.', code: 'CREDENTIAL_REQUIRED', id: '1384574d-a912-4b81-8601-c7b1c4085df1', httpStatusCode: 401, }); - } else if (user!.isSuspended) { + } else if (user?.isSuspended) { throw new ApiError({ message: 'Your account has been suspended.', code: 'YOUR_ACCOUNT_SUSPENDED', @@ -372,8 +372,8 @@ export class ApiCallService implements OnApplicationShutdown { } } - if ((ep.meta.requireModerator || ep.meta.requireAdmin) && (this.meta.rootUserId !== user!.id)) { - const myRoles = await this.roleService.getUserRoles(user!.id); + if ((ep.meta.requireModerator || ep.meta.requireAdmin) && (this.meta.rootUserId !== user?.id)) { + const myRoles = user ? await this.roleService.getUserRoles(user) : []; if (ep.meta.requireModerator && !myRoles.some(r => r.isModerator || r.isAdministrator)) { throw new ApiError({ message: 'You are not assigned to a moderator role.', @@ -392,9 +392,9 @@ export class ApiCallService implements OnApplicationShutdown { } } - if (ep.meta.requiredRolePolicy != null && (this.meta.rootUserId !== user!.id)) { - const myRoles = await this.roleService.getUserRoles(user!.id); - const policies = await this.roleService.getUserPolicies(user!.id); + if (ep.meta.requiredRolePolicy != null && (this.meta.rootUserId !== user?.id)) { + const myRoles = user ? await this.roleService.getUserRoles(user) : []; + const policies = await this.roleService.getUserPolicies(user ?? null); if (!policies[ep.meta.requiredRolePolicy] && !myRoles.some(r => r.isAdministrator)) { throw new ApiError({ message: 'You are not assigned to a required role.', @@ -418,7 +418,7 @@ export class ApiCallService implements OnApplicationShutdown { // Cast non JSON input if ((ep.meta.requireFile || request.method === 'GET') && ep.params.properties) { for (const k of Object.keys(ep.params.properties)) { - const param = ep.params.properties![k]; + const param = ep.params.properties[k]; if (['boolean', 'number', 'integer'].includes(param.type ?? '') && typeof data[k] === 'string') { try { data[k] = JSON.parse(data[k]); diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index 0ba041c536..c7d884cce1 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -92,7 +92,7 @@ export type IEndpointMeta = (Omit & { secure: true, }) | (Omit & { - requireCredential: true, + requireCredential: true | 'optional', kind: (typeof permissions)[number], }) | (Omit & { requireModerator: true, diff --git a/packages/backend/src/server/api/endpoints/admin/show-user.ts b/packages/backend/src/server/api/endpoints/admin/show-user.ts index 1579719246..6a77fc177f 100644 --- a/packages/backend/src/server/api/endpoints/admin/show-user.ts +++ b/packages/backend/src/server/api/endpoints/admin/show-user.ts @@ -122,6 +122,10 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, + isAdministrator: { + type: 'boolean', + optional: false, nullable: false, + }, isSystem: { type: 'boolean', optional: false, nullable: false, @@ -257,6 +261,7 @@ export default class extends Endpoint { // eslint- } const isModerator = await this.roleService.isModerator(user); + const isAdministrator = await this.roleService.isAdministrator(user); const isSilenced = user.isSilenced || !(await this.roleService.getUserPolicies(user.id)).canPublicNote; const _me = await this.usersRepository.findOneByOrFail({ id: me.id }); @@ -289,6 +294,7 @@ export default class extends Endpoint { // eslint- mutedInstances: profile.mutedInstances, notificationRecieveConfig: profile.notificationRecieveConfig, isModerator: isModerator, + isAdministrator: isAdministrator, isSystem: isSystemAccount(user), isSilenced: isSilenced, isSuspended: user.isSuspended, diff --git a/packages/backend/src/server/api/endpoints/antennas/notes.ts b/packages/backend/src/server/api/endpoints/antennas/notes.ts index b90ba6aa0d..7e79f0dccc 100644 --- a/packages/backend/src/server/api/endpoints/antennas/notes.ts +++ b/packages/backend/src/server/api/endpoints/antennas/notes.ts @@ -121,6 +121,7 @@ export default class extends Endpoint { // eslint- this.queryService.generateVisibilityQuery(query, me); this.queryService.generateMutedUserQueryForNotes(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); const notes = await query.getMany(); if (sinceId != null && untilId == null) { diff --git a/packages/backend/src/server/api/endpoints/ap/get.ts b/packages/backend/src/server/api/endpoints/ap/get.ts index 14286bc23e..06dd37a140 100644 --- a/packages/backend/src/server/api/endpoints/ap/get.ts +++ b/packages/backend/src/server/api/endpoints/ap/get.ts @@ -7,6 +7,7 @@ import { Injectable } from '@nestjs/common'; import ms from 'ms'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { ApResolverService } from '@/core/activitypub/ApResolverService.js'; +import { isCollectionOrOrderedCollection, isOrderedCollection, isOrderedCollectionPage } from '@/core/activitypub/type.js'; export const meta = { tags: ['federation'], @@ -33,6 +34,9 @@ export const paramDef = { type: 'object', properties: { uri: { type: 'string' }, + expandCollectionItems: { type: 'boolean' }, + expandCollectionLimit: { type: 'integer', nullable: true }, + allowAnonymous: { type: 'boolean' }, }, required: ['uri'], } as const; @@ -44,7 +48,18 @@ export default class extends Endpoint { // eslint- ) { super(meta, paramDef, async (ps, me) => { const resolver = this.apResolverService.createResolver(); - const object = await resolver.resolve(ps.uri); + const object = await resolver.resolve(ps.uri, ps.allowAnonymous ?? false); + + if (ps.expandCollectionItems && isCollectionOrOrderedCollection(object)) { + const items = await resolver.resolveCollectionItems(object, ps.expandCollectionLimit, ps.allowAnonymous ?? false); + + if (isOrderedCollection(object) || isOrderedCollectionPage(object)) { + object.orderedItems = items; + } else { + object.items = items; + } + } + return object; }); } diff --git a/packages/backend/src/server/api/endpoints/channels/timeline.ts b/packages/backend/src/server/api/endpoints/channels/timeline.ts index 6336f43e9f..99ae1c2211 100644 --- a/packages/backend/src/server/api/endpoints/channels/timeline.ts +++ b/packages/backend/src/server/api/endpoints/channels/timeline.ts @@ -138,9 +138,11 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('note.channel', 'channel'); this.queryService.generateBlockedHostQueryForNote(query); + this.queryService.generateVisibilityQuery(query, me); if (me) { this.queryService.generateMutedUserQueryForNotes(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); } if (ps.withRenotes === false) { diff --git a/packages/backend/src/server/api/endpoints/charts/active-users.ts b/packages/backend/src/server/api/endpoints/charts/active-users.ts index dcdcf46d0b..9f5064fe83 100644 --- a/packages/backend/src/server/api/endpoints/charts/active-users.ts +++ b/packages/backend/src/server/api/endpoints/charts/active-users.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/ap-request.ts b/packages/backend/src/server/api/endpoints/charts/ap-request.ts index 28c64229e7..68dc87546e 100644 --- a/packages/backend/src/server/api/endpoints/charts/ap-request.ts +++ b/packages/backend/src/server/api/endpoints/charts/ap-request.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/drive.ts b/packages/backend/src/server/api/endpoints/charts/drive.ts index 69ff3c5d7a..c0bfb00608 100644 --- a/packages/backend/src/server/api/endpoints/charts/drive.ts +++ b/packages/backend/src/server/api/endpoints/charts/drive.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/federation.ts b/packages/backend/src/server/api/endpoints/charts/federation.ts index bd870cc3d9..bd15700670 100644 --- a/packages/backend/src/server/api/endpoints/charts/federation.ts +++ b/packages/backend/src/server/api/endpoints/charts/federation.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/instance.ts b/packages/backend/src/server/api/endpoints/charts/instance.ts index 765bf024ee..e1053d05d8 100644 --- a/packages/backend/src/server/api/endpoints/charts/instance.ts +++ b/packages/backend/src/server/api/endpoints/charts/instance.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/notes.ts b/packages/backend/src/server/api/endpoints/charts/notes.ts index ecac436311..4550e2f17e 100644 --- a/packages/backend/src/server/api/endpoints/charts/notes.ts +++ b/packages/backend/src/server/api/endpoints/charts/notes.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/user/drive.ts b/packages/backend/src/server/api/endpoints/charts/user/drive.ts index 98ec40ade2..9475a8ab0a 100644 --- a/packages/backend/src/server/api/endpoints/charts/user/drive.ts +++ b/packages/backend/src/server/api/endpoints/charts/user/drive.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/user/following.ts b/packages/backend/src/server/api/endpoints/charts/user/following.ts index cb3dd36bab..20d0ecb25d 100644 --- a/packages/backend/src/server/api/endpoints/charts/user/following.ts +++ b/packages/backend/src/server/api/endpoints/charts/user/following.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/user/notes.ts b/packages/backend/src/server/api/endpoints/charts/user/notes.ts index 0742a21210..1d24dc2b77 100644 --- a/packages/backend/src/server/api/endpoints/charts/user/notes.ts +++ b/packages/backend/src/server/api/endpoints/charts/user/notes.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/user/pv.ts b/packages/backend/src/server/api/endpoints/charts/user/pv.ts index a220381b00..e0026d5ff3 100644 --- a/packages/backend/src/server/api/endpoints/charts/user/pv.ts +++ b/packages/backend/src/server/api/endpoints/charts/user/pv.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/user/reactions.ts b/packages/backend/src/server/api/endpoints/charts/user/reactions.ts index 3bb33622c2..c15056466f 100644 --- a/packages/backend/src/server/api/endpoints/charts/user/reactions.ts +++ b/packages/backend/src/server/api/endpoints/charts/user/reactions.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/charts/users.ts b/packages/backend/src/server/api/endpoints/charts/users.ts index b5452517ab..0f96fae202 100644 --- a/packages/backend/src/server/api/endpoints/charts/users.ts +++ b/packages/backend/src/server/api/endpoints/charts/users.ts @@ -17,11 +17,11 @@ export const meta = { allowGet: true, cacheSec: 60 * 60, - // Burst up to 100, then 2/sec average + // Burst up to 200, then 5/sec average limit: { type: 'bucket', - size: 100, - dripRate: 500, + size: 200, + dripRate: 200, }, } as const; diff --git a/packages/backend/src/server/api/endpoints/clips/notes.ts b/packages/backend/src/server/api/endpoints/clips/notes.ts index 59513e530d..4758dbad00 100644 --- a/packages/backend/src/server/api/endpoints/clips/notes.ts +++ b/packages/backend/src/server/api/endpoints/clips/notes.ts @@ -92,10 +92,11 @@ export default class extends Endpoint { // eslint- .andWhere('clipNote.clipId = :clipId', { clipId: clip.id }); this.queryService.generateBlockedHostQueryForNote(query); + this.queryService.generateVisibilityQuery(query, me); if (me) { - this.queryService.generateVisibilityQuery(query, me); this.queryService.generateMutedUserQueryForNotes(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); } const notes = await query diff --git a/packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts b/packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts index df030d90aa..17c9b31c90 100644 --- a/packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts @@ -74,18 +74,15 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.btlDisabled); } - const [ - followings, - ] = me ? await Promise.all([ - this.cacheService.userFollowingsCache.fetch(me.id), - ]) : [undefined]; + const followings = me ? await this.cacheService.userFollowingsCache.fetch(me.id) : undefined; //#region Construct query const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) .andWhere('note.visibility = \'public\'') .andWhere('note.channelId IS NULL') - .andWhere('note.userHost IN (:...hosts)', { hosts: this.serverSettings.bubbleInstances }) + .andWhere('note.userHost IS NOT NULL') + .andWhere('userInstance.isBubbled = true') // This comes from generateBlockedHostQueryForNote below .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') @@ -97,6 +94,7 @@ export default class extends Endpoint { // eslint- if (me) this.queryService.generateMutedUserQueryForNotes(query, me); if (me) this.queryService.generateBlockedUserQueryForNotes(query, me); if (me) this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + if (!me) query.andWhere('user.requireSigninToViewContents = false'); if (ps.withFiles) { query.andWhere('note.fileIds != \'{}\''); @@ -104,21 +102,27 @@ export default class extends Endpoint { // eslint- if (!ps.withBots) query.andWhere('user.isBot = FALSE'); - if (ps.withRenotes === false) { - query.andWhere(new Brackets(qb => { - qb.where('note.renoteId IS NULL'); - qb.orWhere(new Brackets(qb => { - qb.where('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - })); - })); + if (!ps.withRenotes) { + query.andWhere(new Brackets(qb => qb + .orWhere('note.renoteId IS NULL') + .orWhere('note.text IS NOT NULL') + .orWhere('note.cw IS NOT NULL') + .orWhere('note.replyId IS NOT NULL') + .orWhere('note.hasPoll = false') + .orWhere('note.fileIds != \'{}\''))); } //#endregion let timeline = await query.limit(ps.limit).getMany(); timeline = timeline.filter(note => { - if (note.user?.isSilenced && me && followings && note.userId !== me.id && !followings[note.userId]) return false; + if (note.user?.isSilenced) { + if (!me) return false; + if (!followings) return false; + if (note.userId !== me.id) { + return followings[note.userId]; + } + } return true; }); diff --git a/packages/backend/src/server/api/endpoints/notes/following.ts b/packages/backend/src/server/api/endpoints/notes/following.ts index 5f6ee9f903..088b172ba4 100644 --- a/packages/backend/src/server/api/endpoints/notes/following.ts +++ b/packages/backend/src/server/api/endpoints/notes/following.ts @@ -4,7 +4,7 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import { ObjectLiteral, SelectQueryBuilder } from 'typeorm'; +import { IsNull, ObjectLiteral, SelectQueryBuilder } from 'typeorm'; import { SkLatestNote, MiFollowing } from '@/models/_.js'; import type { NotesRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; @@ -130,7 +130,9 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel') + + // Exclude channel notes + .andWhere({ channelId: IsNull() }) ; // Limit to files, if requested @@ -145,11 +147,10 @@ export default class extends Endpoint { // eslint- // Hide blocked users / instances query.andWhere('"user"."isSuspended" = false'); - query.andWhere('("replyUser" IS NULL OR "replyUser"."isSuspended" = false)'); - query.andWhere('("renoteUser" IS NULL OR "renoteUser"."isSuspended" = false)'); this.queryService.generateBlockedHostQueryForNote(query); - // Respect blocks and mutes + // Respect blocks, mutes, and privacy + this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); this.queryService.generateMutedUserQueryForNotes(query, me); @@ -161,7 +162,7 @@ export default class extends Endpoint { // eslint- // Query and return the next page const notes = await query.getMany(); - return await this.noteEntityService.packMany(notes, me); + return await this.noteEntityService.packMany(notes, me, { skipHide: true }); }); } } diff --git a/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts b/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts index 33a9c281b3..6f96821a63 100644 --- a/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts +++ b/packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts @@ -9,13 +9,13 @@ import type { NotesRepository, MutingsRepository, PollsRepository, PollVotesRepo import { Endpoint } from '@/server/api/endpoint-base.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; +import { QueryService } from '@/core/QueryService.js'; +import { RoleService } from '@/core/RoleService.js'; +import { ApiError } from '@/server/api/error.js'; export const meta = { tags: ['notes'], - requireCredential: true, - kind: 'read:account', - res: { type: 'array', optional: false, nullable: false, @@ -26,10 +26,24 @@ export const meta = { }, }, - // 2 calls per second + errors: { + ltlDisabled: { + message: 'Local timeline has been disabled.', + code: 'LTL_DISABLED', + id: '45a6eb02-7695-4393-b023-dd3be9aaaefd', + }, + gtlDisabled: { + message: 'Global timeline has been disabled.', + code: 'GTL_DISABLED', + id: '0332fc13-6ab2-4427-ae80-a9fadffd1a6b', + }, + }, + + // Up to 10 calls, then 2 per second limit: { - duration: 1000, - max: 2, + type: 'bucket', + size: 10, + dripRate: 500, }, } as const; @@ -39,6 +53,8 @@ export const paramDef = { limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, offset: { type: 'integer', default: 0 }, excludeChannels: { type: 'boolean', default: false }, + local: { type: 'boolean', nullable: true, default: null }, + expired: { type: 'boolean', default: false }, }, required: [], } as const; @@ -59,18 +75,54 @@ export default class extends Endpoint { // eslint- private mutingsRepository: MutingsRepository, private noteEntityService: NoteEntityService, + private readonly queryService: QueryService, + private readonly roleService: RoleService, ) { super(meta, paramDef, async (ps, me) => { const query = this.pollsRepository.createQueryBuilder('poll') - .where('poll.userHost IS NULL') - .andWhere('poll.userId != :meId', { meId: me.id }) - .andWhere('poll.noteVisibility = \'public\'') - .andWhere(new Brackets(qb => { + .innerJoinAndSelect('poll.note', 'note') + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('reply.user', 'replyUser') + .andWhere('user.isExplorable = TRUE') + ; + + if (me) { + query.andWhere('poll.userId != :meId', { meId: me.id }); + } + + if (ps.expired) { + query.andWhere('poll.expiresAt IS NOT NULL'); + query.andWhere('poll.expiresAt <= :expiresMax', { + expiresMax: new Date(), + }); + query.andWhere('poll.expiresAt >= :expiresMin', { + expiresMin: new Date(Date.now() - (1000 * 60 * 60 * 24 * 7)), + }); + } else { + query.andWhere(new Brackets(qb => { qb .where('poll.expiresAt IS NULL') .orWhere('poll.expiresAt > :now', { now: new Date() }); })); + } + const policies = await this.roleService.getUserPolicies(me?.id ?? null); + if (ps.local != null) { + if (ps.local) { + if (!policies.ltlAvailable) throw new ApiError(meta.errors.ltlDisabled); + query.andWhere('poll.userHost IS NULL'); + } else { + if (!policies.gtlAvailable) throw new ApiError(meta.errors.gtlDisabled); + query.andWhere('poll.userHost IS NOT NULL'); + } + } else { + if (!policies.gtlAvailable) throw new ApiError(meta.errors.gtlDisabled); + } + + /* //#region exclude arleady voted polls const votedQuery = this.pollVotesRepository.createQueryBuilder('vote') .select('vote.noteId') @@ -81,16 +133,15 @@ export default class extends Endpoint { // eslint- query.setParameters(votedQuery.getParameters()); //#endregion + */ - //#region mute - const mutingQuery = this.mutingsRepository.createQueryBuilder('muting') - .select('muting.muteeId') - .where('muting.muterId = :muterId', { muterId: me.id }); - - query - .andWhere(`poll.userId NOT IN (${ mutingQuery.getQuery() })`); - - query.setParameters(mutingQuery.getParameters()); + //#region block/mute/vis + this.queryService.generateVisibilityQuery(query, me); + this.queryService.generateBlockedHostQueryForNote(query); + if (me) { + this.queryService.generateBlockedUserQueryForNotes(query, me); + this.queryService.generateMutedUserQueryForNotes(query, me); + } //#endregion //#region exclude channels @@ -107,6 +158,7 @@ export default class extends Endpoint { // eslint- if (polls.length === 0) return []; + /* const notes = await this.notesRepository.find({ where: { id: In(polls.map(poll => poll.noteId)), @@ -115,6 +167,10 @@ export default class extends Endpoint { // eslint- id: 'DESC', }, }); + */ + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const notes = polls.map(poll => poll.note!); return await this.noteEntityService.packMany(notes, me, { detail: true, diff --git a/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts b/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts index 91874a8195..5c1ab0fb78 100644 --- a/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts +++ b/packages/backend/src/server/api/endpoints/notes/search-by-tag.ts @@ -96,10 +96,10 @@ export default class extends Endpoint { // eslint- if (!this.serverSettings.enableBotTrending) query.andWhere('user.isBot = FALSE'); - this.queryService.generateVisibilityQuery(query, me); - this.queryService.generateBlockedHostQueryForNote(query); + this.queryService.generateBlockedHostQueryForNote(query, undefined, false); if (me) this.queryService.generateMutedUserQueryForNotes(query, me); if (me) this.queryService.generateBlockedUserQueryForNotes(query, me); + if (me) this.queryService.generateMutedUserRenotesQueryForNotes(query, me); const followings = me ? await this.cacheService.userFollowingsCache.fetch(me.id) : {}; @@ -160,7 +160,6 @@ export default class extends Endpoint { // eslint- if (note.user?.isSuspended) return false; if (note.userHost) { if (!this.utilityService.isFederationAllowedHost(note.userHost)) return false; - if (this.utilityService.isSilencedHost(this.serverSettings.silencedHosts, note.userHost)) return false; } return true; }); diff --git a/packages/backend/src/server/api/endpoints/notes/translate.ts b/packages/backend/src/server/api/endpoints/notes/translate.ts index a97542c063..e55168e296 100644 --- a/packages/backend/src/server/api/endpoints/notes/translate.ts +++ b/packages/backend/src/server/api/endpoints/notes/translate.ts @@ -20,11 +20,9 @@ import { ApiError } from '../../error.js'; export const meta = { tags: ['notes'], - // TODO allow unauthenticated if default template allows? - // Maybe a value 'optional' that allows unauthenticated OR a token w/ appropriate role. - // This will allow unauthenticated requests without leaking post data to restricted clients. - requireCredential: true, + requireCredential: 'optional', kind: 'read:account', + requiredRolePolicy: 'canUseTranslator', res: { type: 'object', @@ -88,17 +86,12 @@ export default class extends Endpoint { // eslint- private readonly loggerService: ApiLoggerService, ) { super(meta, paramDef, async (ps, me) => { - const policies = await this.roleService.getUserPolicies(me.id); - if (!policies.canUseTranslator) { - throw new ApiError(meta.errors.unavailable); - } - const note = await this.getterService.getNote(ps.noteId).catch(err => { if (err.id === '9725d0ce-ba28-4dde-95a7-2cbb2c15de24') throw new ApiError(meta.errors.noSuchNote); throw err; }); - if (!(await this.noteEntityService.isVisibleForMe(note, me.id))) { + if (!(await this.noteEntityService.isVisibleForMe(note, me?.id ?? null))) { throw new ApiError(meta.errors.cannotTranslateInvisibleNote); } @@ -140,7 +133,7 @@ export default class extends Endpoint { // eslint- if (this.serverSettings.deeplAuthKey) params.append('auth_key', this.serverSettings.deeplAuthKey); params.append('text', note.text); params.append('target_lang', targetLang); - const endpoint = deeplFreeInstance ?? this.serverSettings.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate'; + const endpoint = deeplFreeInstance ?? ( this.serverSettings.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate' ); const res = await this.httpRequestService.send(endpoint, { method: 'POST', diff --git a/packages/backend/src/server/api/endpoints/roles/notes.ts b/packages/backend/src/server/api/endpoints/roles/notes.ts index d1c2e4b686..536384a381 100644 --- a/packages/backend/src/server/api/endpoints/roles/notes.ts +++ b/packages/backend/src/server/api/endpoints/roles/notes.ts @@ -107,10 +107,10 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('reply.user', 'replyUser') .leftJoinAndSelect('renote.user', 'renoteUser'); - this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedHostQueryForNote(query); this.queryService.generateMutedUserQueryForNotes(query, me); this.queryService.generateBlockedUserQueryForNotes(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); const notes = await query.getMany(); notes.sort((a, b) => a.id > b.id ? -1 : 1); diff --git a/packages/backend/src/server/api/endpoints/users/reactions.ts b/packages/backend/src/server/api/endpoints/users/reactions.ts index 56f59bd285..553787ad58 100644 --- a/packages/backend/src/server/api/endpoints/users/reactions.ts +++ b/packages/backend/src/server/api/endpoints/users/reactions.ts @@ -105,10 +105,15 @@ export default class extends Endpoint { // eslint- const query = this.queryService.makePaginationQuery(this.noteReactionsRepository.createQueryBuilder('reaction'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) .andWhere('reaction.userId = :userId', { userId: ps.userId }) - .leftJoinAndSelect('reaction.note', 'note'); + .innerJoinAndSelect('reaction.note', 'note'); this.queryService.generateVisibilityQuery(query, me); this.queryService.generateBlockedHostQueryForNote(query); + if (me) { + this.queryService.generateMutedUserQueryForNotes(query, me); + this.queryService.generateBlockedUserQueryForNotes(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + } const reactions = (await query .limit(ps.limit) diff --git a/packages/backend/src/server/api/mastodon/MastodonConverters.ts b/packages/backend/src/server/api/mastodon/MastodonConverters.ts index 375ea1ef08..02ce31c4f8 100644 --- a/packages/backend/src/server/api/mastodon/MastodonConverters.ts +++ b/packages/backend/src/server/api/mastodon/MastodonConverters.ts @@ -252,10 +252,10 @@ export class MastodonConverters { return await this.convertStatus(status, me); } - public async convertStatus(status: Entity.Status, me: MiLocalUser | null): Promise { + public async convertStatus(status: Entity.Status, me: MiLocalUser | null, hints?: { note?: MiNote, user?: MiUser }): Promise { const convertedAccount = this.convertAccount(status.account); - const note = await this.mastodonDataService.requireNote(status.id, me); - const noteUser = await this.getUser(status.account.id); + const note = hints?.note ?? await this.mastodonDataService.requireNote(status.id, me); + const noteUser = hints?.user ?? note.user ?? await this.getUser(status.account.id); const mentionedRemoteUsers = JSON.parse(note.mentionedRemoteUsers); const emojis = await this.customEmojiService.populateEmojis(note.emojis, noteUser.host ? noteUser.host : this.config.host); diff --git a/packages/backend/src/server/api/mastodon/MastodonDataService.ts b/packages/backend/src/server/api/mastodon/MastodonDataService.ts index db257756de..73cd553b9a 100644 --- a/packages/backend/src/server/api/mastodon/MastodonDataService.ts +++ b/packages/backend/src/server/api/mastodon/MastodonDataService.ts @@ -7,8 +7,8 @@ import { Inject, Injectable } from '@nestjs/common'; import { IsNull } from 'typeorm'; import { DI } from '@/di-symbols.js'; import { QueryService } from '@/core/QueryService.js'; -import type { MiNote, NotesRepository } from '@/models/_.js'; -import type { MiLocalUser } from '@/models/User.js'; +import type { MiChannel, MiNote, NotesRepository } from '@/models/_.js'; +import type { MiLocalUser, MiUser } from '@/models/User.js'; import { ApiError } from '../error.js'; /** @@ -27,8 +27,8 @@ export class MastodonDataService { /** * Fetches a note in the context of the current user, and throws an exception if not found. */ - public async requireNote(noteId: string, me?: MiLocalUser | null): Promise { - const note = await this.getNote(noteId, me); + public async requireNote(noteId: string, me: MiLocalUser | null | undefined, relations?: Rel): Promise> { + const note = await this.getNote(noteId, me, relations); if (!note) { throw new ApiError({ @@ -46,12 +46,39 @@ export class MastodonDataService { /** * Fetches a note in the context of the current user. */ - public async getNote(noteId: string, me?: MiLocalUser | null): Promise { + public async getNote(noteId: string, me: MiLocalUser | null | undefined, relations?: Rel): Promise | null> { // Root query: note + required dependencies const query = this.notesRepository .createQueryBuilder('note') - .where('note.id = :noteId', { noteId }) - .innerJoinAndSelect('note.user', 'user'); + .where('note.id = :noteId', { noteId }); + + // Load relations + if (relations) { + if (relations.reply) { + query.leftJoinAndSelect('note.reply', 'reply'); + if (typeof(relations.reply) === 'object') { + if (relations.reply.reply) query.leftJoinAndSelect('note.reply.reply', 'replyReply'); + if (relations.reply.renote) query.leftJoinAndSelect('note.reply.renote', 'replyRenote'); + if (relations.reply.user) query.innerJoinAndSelect('note.reply.user', 'replyUser'); + if (relations.reply.channel) query.leftJoinAndSelect('note.reply.channel', 'replyChannel'); + } + } + if (relations.renote) { + query.leftJoinAndSelect('note.renote', 'renote'); + if (typeof(relations.renote) === 'object') { + if (relations.renote.reply) query.leftJoinAndSelect('note.renote.reply', 'renoteReply'); + if (relations.renote.renote) query.leftJoinAndSelect('note.renote.renote', 'renoteRenote'); + if (relations.renote.user) query.innerJoinAndSelect('note.renote.user', 'renoteUser'); + if (relations.renote.channel) query.leftJoinAndSelect('note.renote.channel', 'renoteChannel'); + } + } + if (relations.user) { + query.innerJoinAndSelect('note.user', 'user'); + } + if (relations.channel) { + query.leftJoinAndSelect('note.channel', 'channel'); + } + } // Restrict visibility this.queryService.generateVisibilityQuery(query, me); @@ -59,7 +86,7 @@ export class MastodonDataService { this.queryService.generateBlockedUserQueryForNotes(query, me); } - return await query.getOne(); + return await query.getOne() as NoteWithRelations | null; } /** @@ -82,3 +109,41 @@ export class MastodonDataService { }); } } + +interface NoteRelations { + reply?: boolean | { + reply?: boolean; + renote?: boolean; + user?: boolean; + channel?: boolean; + }; + renote?: boolean | { + reply?: boolean; + renote?: boolean; + user?: boolean; + channel?: boolean; + }; + user?: boolean; + channel?: boolean; +} + +type NoteWithRelations = MiNote & { + reply: Rel extends { reply: false } + ? null + : null | (MiNote & { + reply: Rel['reply'] extends { reply: true } ? MiNote | null : null; + renote: Rel['reply'] extends { renote: true } ? MiNote | null : null; + user: Rel['reply'] extends { user: true } ? MiUser : null; + channel: Rel['reply'] extends { channel: true } ? MiChannel | null : null; + }); + renote: Rel extends { renote: false } + ? null + : null | (MiNote & { + reply: Rel['renote'] extends { reply: true } ? MiNote | null : null; + renote: Rel['renote'] extends { renote: true } ? MiNote | null : null; + user: Rel['renote'] extends { user: true } ? MiUser : null; + channel: Rel['renote'] extends { channel: true } ? MiChannel | null : null; + }); + user: Rel extends { user: true } ? MiUser : null; + channel: Rel extends { channel: true } ? MiChannel | null : null; +}; diff --git a/packages/backend/src/server/api/mastodon/endpoints/status.ts b/packages/backend/src/server/api/mastodon/endpoints/status.ts index 22b8a911ca..7a058a0ed9 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/status.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/status.ts @@ -8,6 +8,10 @@ import { Injectable } from '@nestjs/common'; import { emojiRegexAtStartToEnd } from '@/misc/emoji-regex.js'; import { parseTimelineArgs, TimelineArgs, toBoolean, toInt } from '@/server/api/mastodon/argsUtils.js'; import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js'; +import { MastodonDataService } from '@/server/api/mastodon/MastodonDataService.js'; +import { getNoteSummary } from '@/misc/get-note-summary.js'; +import type { Packed } from '@/misc/json-schema.js'; +import { isPureRenote } from '@/misc/is-renote.js'; import { convertAttachment, convertPoll, MastodonConverters } from '../MastodonConverters.js'; import type { Entity } from 'megalodon'; import type { FastifyInstance } from 'fastify'; @@ -22,6 +26,7 @@ export class ApiStatusMastodon { constructor( private readonly mastoConverters: MastodonConverters, private readonly clientService: MastodonClientService, + private readonly mastodonDataService: MastodonDataService, ) {} public register(fastify: FastifyInstance): void { @@ -29,13 +34,24 @@ export class ApiStatusMastodon { if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' }); const { client, me } = await this.clientService.getAuthClient(_request); - const data = await client.getStatus(_request.params.id); - const response = await this.mastoConverters.convertStatus(data.data, me); + const note = await this.mastodonDataService.requireNote(_request.params.id, me, { user: true, renote: { user: true } }); + + // Unpack renote for Discord, otherwise the preview breaks + const appearNote = (isPureRenote(note) && _request.headers['user-agent']?.match(/\bDiscordbot\//)) + ? note.renote as NonNullable + : note; + + const data = await client.getStatus(appearNote.id); + const response = await this.mastoConverters.convertStatus(data.data, me, { note: appearNote, user: appearNote.user }); // Fixup - Discord ignores CWs and renders the entire post. if (response.sensitive && _request.headers['user-agent']?.match(/\bDiscordbot\//)) { - response.content = '(preview disabled for sensitive content)'; + response.content = getNoteSummary(data.data satisfies Packed<'Note'>); response.media_attachments = []; + response.in_reply_to_id = null; + response.in_reply_to_account_id = null; + response.reblog = null; + response.quote = null; } return reply.send(response); @@ -170,7 +186,7 @@ export class ApiStatusMastodon { const data = await client.deleteEmojiReaction(id, react); return reply.send(data.data); } - if (!body.media_ids) body.media_ids = undefined; + body.media_ids ??= undefined; if (body.media_ids && !body.media_ids.length) body.media_ids = undefined; if (body.poll && !body.poll.options) { diff --git a/packages/backend/src/server/api/stream/channel.ts b/packages/backend/src/server/api/stream/channel.ts index 9af816dfbb..204ea9f705 100644 --- a/packages/backend/src/server/api/stream/channel.ts +++ b/packages/backend/src/server/api/stream/channel.ts @@ -65,6 +65,9 @@ export default abstract class Channel { * ミュートとブロックされてるを処理する */ protected isNoteMutedOrBlocked(note: Packed<'Note'>): boolean { + // Ignore notes that require sign-in + if (note.user.requireSigninToViewContents && !this.user) return true; + // 流れてきたNoteがインスタンスミュートしたインスタンスが関わる if (isInstanceMuted(note, new Set(this.userProfile?.mutedInstances ?? [])) && !this.following[note.userId]) return true; diff --git a/packages/backend/src/server/api/stream/channels/bubble-timeline.ts b/packages/backend/src/server/api/stream/channels/bubble-timeline.ts index d29101cbc5..88cb9937b3 100644 --- a/packages/backend/src/server/api/stream/channels/bubble-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/bubble-timeline.ts @@ -12,6 +12,7 @@ import { RoleService } from '@/core/RoleService.js'; import type { MiMeta } from '@/models/Meta.js'; import { isRenotePacked, isQuotePacked } from '@/misc/is-renote.js'; import type { JsonObject } from '@/misc/json-value.js'; +import { UtilityService } from '@/core/UtilityService.js'; import Channel, { MiChannelService } from '../channel.js'; class BubbleTimelineChannel extends Channel { @@ -26,6 +27,7 @@ class BubbleTimelineChannel extends Channel { constructor( private metaService: MetaService, private roleService: RoleService, + private readonly utilityService: UtilityService, noteEntityService: NoteEntityService, id: string, @@ -56,12 +58,15 @@ class BubbleTimelineChannel extends Channel { if (note.visibility !== 'public') return; if (note.channelId != null) return; if (note.user.host == null) return; - if (!this.instance.bubbleInstances.includes(note.user.host)) return; + if (!this.utilityService.isBubbledHost(note.user.host)) return; if (note.user.requireSigninToViewContents && this.user == null) return; if (isRenotePacked(note) && !isQuotePacked(note) && !this.withRenotes) return; - if (note.user.isSilenced && !this.following[note.userId] && note.userId !== this.user!.id) return; + if (note.user.isSilenced) { + if (!this.user) return; + if (note.userId !== this.user.id && !this.following[note.userId]) return; + } if (this.isNoteMutedOrBlocked(note)) return; @@ -88,6 +93,7 @@ export class BubbleTimelineChannelService implements MiChannelService { private metaService: MetaService, private roleService: RoleService, private noteEntityService: NoteEntityService, + private readonly utilityService: UtilityService, ) { } @@ -96,6 +102,7 @@ export class BubbleTimelineChannelService implements MiChannelService { return new BubbleTimelineChannel( this.metaService, this.roleService, + this.utilityService, this.noteEntityService, id, connection, diff --git a/packages/backend/test-federation/tsconfig.json b/packages/backend/test-federation/tsconfig.json index 3a1cb3b9f3..16b333f877 100644 --- a/packages/backend/test-federation/tsconfig.json +++ b/packages/backend/test-federation/tsconfig.json @@ -3,7 +3,7 @@ /* Visit https://aka.ms/tsconfig to read more about this file */ /* Projects */ - // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ diff --git a/packages/backend/test-server/tsconfig.json b/packages/backend/test-server/tsconfig.json index 10313699c2..cb394ecccd 100644 --- a/packages/backend/test-server/tsconfig.json +++ b/packages/backend/test-server/tsconfig.json @@ -23,6 +23,7 @@ "emitDecoratorMetadata": true, "resolveJsonModule": true, "isolatedModules": true, + "incremental": true, "rootDir": "../src", "baseUrl": "./", "paths": { diff --git a/packages/backend/test/tsconfig.json b/packages/backend/test/tsconfig.json index 2b562acda8..f3b6a5108d 100644 --- a/packages/backend/test/tsconfig.json +++ b/packages/backend/test/tsconfig.json @@ -23,6 +23,7 @@ "emitDecoratorMetadata": true, "resolveJsonModule": true, "isolatedModules": true, + "incremental": true, "baseUrl": "./", "paths": { "@/*": ["../src/*"] diff --git a/packages/backend/test/unit/MetaService.ts b/packages/backend/test/unit/MetaService.ts index 19c98eab3d..056838e180 100644 --- a/packages/backend/test/unit/MetaService.ts +++ b/packages/backend/test/unit/MetaService.ts @@ -11,6 +11,7 @@ import { GlobalModule } from '@/GlobalModule.js'; import { DI } from '@/di-symbols.js'; import { MetaService } from '@/core/MetaService.js'; import { CoreModule } from '@/core/CoreModule.js'; +import { MetasRepository } from '@/models/_.js'; import type { TestingModule } from '@nestjs/testing'; import type { DataSource } from 'typeorm'; @@ -39,8 +40,8 @@ describe('MetaService', () => { }); test('fetch (cache)', async () => { - const db = app.get(DI.db); - const spy = jest.spyOn(db, 'transaction'); + const metasRepository = app.get(DI.metasRepository); + const spy = jest.spyOn(metasRepository, 'createQueryBuilder'); const result = await metaService.fetch(); @@ -49,12 +50,12 @@ describe('MetaService', () => { }); test('fetch (force)', async () => { - const db = app.get(DI.db); - const spy = jest.spyOn(db, 'transaction'); + const metasRepository = app.get(DI.metasRepository); + const spy = jest.spyOn(metasRepository, 'createQueryBuilder'); const result = await metaService.fetch(true); expect(result.id).toBe('x'); - expect(spy).toHaveBeenCalledTimes(1); + expect(spy).toHaveBeenCalled(); }); }); diff --git a/packages/backend/test/unit/NoteCreateService.ts b/packages/backend/test/unit/NoteCreateService.ts index f4ecfef34d..63e3795a84 100644 --- a/packages/backend/test/unit/NoteCreateService.ts +++ b/packages/backend/test/unit/NoteCreateService.ts @@ -57,10 +57,13 @@ describe('NoteCreateService', () => { channelId: null, channel: null, userHost: null, + userInstance: null, replyUserId: null, replyUserHost: null, + replyUserInstance: null, renoteUserId: null, renoteUserHost: null, + renoteUserInstance: null, processErrors: [], }; diff --git a/packages/backend/test/unit/RoleService.ts b/packages/backend/test/unit/RoleService.ts index 553ff0982a..839402418e 100644 --- a/packages/backend/test/unit/RoleService.ts +++ b/packages/backend/test/unit/RoleService.ts @@ -15,6 +15,7 @@ import type { MockFunctionMetadata } from 'jest-mock'; import { GlobalModule } from '@/GlobalModule.js'; import { RoleService } from '@/core/RoleService.js'; import { + InstancesRepository, MiMeta, MiRole, MiRoleAssignment, @@ -39,6 +40,7 @@ const moduleMocker = new ModuleMocker(global); describe('RoleService', () => { let app: TestingModule; let roleService: RoleService; + let instancesRepository: InstancesRepository; let usersRepository: UsersRepository; let rolesRepository: RolesRepository; let roleAssignmentsRepository: RoleAssignmentsRepository; @@ -47,6 +49,19 @@ describe('RoleService', () => { let clock: lolex.InstalledClock; async function createUser(data: Partial = {}) { + if (data.host != null) { + await instancesRepository + .createQueryBuilder('instance') + .insert() + .values({ + id: genAidx(Date.now()), + firstRetrievedAt: new Date(), + host: data.host, + }) + .orIgnore() + .execute(); + } + const un = secureRndstr(16); const x = await usersRepository.insert({ id: genAidx(Date.now()), @@ -145,6 +160,7 @@ describe('RoleService', () => { app.enableShutdownHooks(); roleService = app.get(RoleService); + instancesRepository = app.get(DI.instancesRepository); usersRepository = app.get(DI.usersRepository); rolesRepository = app.get(DI.rolesRepository); roleAssignmentsRepository = app.get(DI.roleAssignmentsRepository); diff --git a/packages/backend/test/unit/UserSearchService.ts b/packages/backend/test/unit/UserSearchService.ts index 697425beb8..a6b331d1cb 100644 --- a/packages/backend/test/unit/UserSearchService.ts +++ b/packages/backend/test/unit/UserSearchService.ts @@ -7,16 +7,18 @@ import { Test, TestingModule } from '@nestjs/testing'; import { describe, jest, test } from '@jest/globals'; import { In } from 'typeorm'; import { UserSearchService } from '@/core/UserSearchService.js'; -import { FollowingsRepository, MiUser, UserProfilesRepository, UsersRepository } from '@/models/_.js'; +import { FollowingsRepository, InstancesRepository, MiUser, UserProfilesRepository, UsersRepository } from '@/models/_.js'; import { IdService } from '@/core/IdService.js'; import { GlobalModule } from '@/GlobalModule.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { genAidx } from '@/misc/id/aidx.js'; describe('UserSearchService', () => { let app: TestingModule; let service: UserSearchService; + let instancesRepository: InstancesRepository; let usersRepository: UsersRepository; let followingsRepository: FollowingsRepository; let idService: IdService; @@ -35,6 +37,19 @@ describe('UserSearchService', () => { let bobby: MiUser; async function createUser(data: Partial = {}) { + if (data.host != null) { + await instancesRepository + .createQueryBuilder('instance') + .insert() + .values({ + id: genAidx(Date.now()), + firstRetrievedAt: new Date(), + host: data.host, + }) + .orIgnore() + .execute(); + } + const user = await usersRepository .insert({ id: idService.gen(), @@ -104,6 +119,7 @@ describe('UserSearchService', () => { await app.init(); + instancesRepository = app.get(DI.instancesRepository); usersRepository = app.get(DI.usersRepository); userProfilesRepository = app.get(DI.userProfilesRepository); followingsRepository = app.get(DI.followingsRepository); diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index 6f6d4c4121..94dec16401 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -103,6 +103,25 @@ describe('ActivityPub', () => { let config: Config; const metaInitial = { + id: 'x', + name: 'Test Instance', + shortName: 'Test Instance', + description: 'Test Instance', + langs: [] as string[], + pinnedUsers: [] as string[], + hiddenTags: [] as string[], + prohibitedWordsForNameOfUser: [] as string[], + silencedHosts: [] as string[], + mediaSilencedHosts: [] as string[], + policies: {}, + serverRules: [] as string[], + bannedEmailDomains: [] as string[], + preservedUsernames: [] as string[], + bubbleInstances: [] as string[], + trustedLinkUrlPatterns: [] as string[], + federation: 'all', + federationHosts: [] as string[], + allowUnsignedFetch: 'always', cacheRemoteFiles: true, cacheRemoteSensitiveFiles: true, enableFanoutTimeline: true, diff --git a/packages/backend/test/unit/misc/diff-arrays.ts b/packages/backend/test/unit/misc/diff-arrays.ts new file mode 100644 index 0000000000..b6db5e2eca --- /dev/null +++ b/packages/backend/test/unit/misc/diff-arrays.ts @@ -0,0 +1,91 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { diffArrays, diffArraysSimple } from '@/misc/diff-arrays.js'; + +describe(diffArrays, () => { + it('should return empty result when both inputs are null', () => { + const result = diffArrays(null, null); + expect(result.added).toHaveLength(0); + expect(result.removed).toHaveLength(0); + }); + + it('should return empty result when both inputs are empty', () => { + const result = diffArrays([], []); + expect(result.added).toHaveLength(0); + expect(result.removed).toHaveLength(0); + }); + + it('should remove before when after is empty', () => { + const result = diffArrays([1, 2, 3], []); + expect(result.added).toHaveLength(0); + expect(result.removed).toEqual([1, 2, 3]); + }); + + it('should deduplicate before when after is empty', () => { + const result = diffArrays([1, 1, 2, 2, 3], []); + expect(result.added).toHaveLength(0); + expect(result.removed).toEqual([1, 2, 3]); + }); + + it('should add after when before is empty', () => { + const result = diffArrays([], [1, 2, 3]); + expect(result.added).toEqual([1, 2, 3]); + expect(result.removed).toHaveLength(0); + }); + + it('should deduplicate after when before is empty', () => { + const result = diffArrays([], [1, 1, 2, 2, 3]); + expect(result.added).toEqual([1, 2, 3]); + expect(result.removed).toHaveLength(0); + }); + + it('should return diff when both have values', () => { + const result = diffArrays( + ['a', 'b', 'c', 'd'], + ['a', 'c', 'e', 'f'], + ); + expect(result.added).toEqual(['e', 'f']); + expect(result.removed).toEqual(['b', 'd']); + }); +}); + +describe(diffArraysSimple, () => { + it('should return false when both inputs are null', () => { + const result = diffArraysSimple(null, null); + expect(result).toBe(false); + }); + + it('should return false when both inputs are empty', () => { + const result = diffArraysSimple([], []); + expect(result).toBe(false); + }); + + it('should return true when before is populated and after is empty', () => { + const result = diffArraysSimple([1, 2, 3], []); + expect(result).toBe(true); + }); + + it('should return true when before is empty and after is populated', () => { + const result = diffArraysSimple([], [1, 2, 3]); + expect(result).toBe(true); + }); + + it('should return true when values have changed', () => { + const result = diffArraysSimple( + ['a', 'a', 'b', 'c'], + ['a', 'b', 'c', 'd'], + ); + expect(result).toBe(true); + }); + + it('should return false when values have not changed', () => { + const result = diffArraysSimple( + ['a', 'a', 'b', 'c'], + ['a', 'b', 'c', 'c'], + ); + expect(result).toBe(false); + }); +}); diff --git a/packages/backend/test/unit/misc/is-renote.ts b/packages/backend/test/unit/misc/is-renote.ts index 24cd2236bb..b6cfa53466 100644 --- a/packages/backend/test/unit/misc/is-renote.ts +++ b/packages/backend/test/unit/misc/is-renote.ts @@ -40,10 +40,13 @@ const base: MiNote = { channelId: null, channel: null, userHost: null, + userInstance: null, replyUserId: null, replyUserHost: null, + replyUserInstance: null, renoteUserId: null, renoteUserHost: null, + renoteUserInstance: null, processErrors: [], }; diff --git a/packages/backend/tsconfig.json b/packages/backend/tsconfig.json index 392da169ad..afed1f186c 100644 --- a/packages/backend/tsconfig.json +++ b/packages/backend/tsconfig.json @@ -23,6 +23,7 @@ "emitDecoratorMetadata": true, "resolveJsonModule": true, "isolatedModules": true, + "incremental": true, "rootDir": "./src", "baseUrl": "./", "paths": { diff --git a/packages/frontend-embed/package.json b/packages/frontend-embed/package.json index 658a9f732d..1a851df49b 100644 --- a/packages/frontend-embed/package.json +++ b/packages/frontend-embed/package.json @@ -18,7 +18,7 @@ "@transfem-org/sfm-js": "0.24.5", "@twemoji/parser": "15.1.1", "@vitejs/plugin-vue": "5.2.3", - "@vue/compiler-sfc": "3.5.12", + "@vue/compiler-sfc": "3.5.14", "astring": "1.9.0", "buraha": "0.0.1", "estree-walker": "3.0.3", @@ -35,7 +35,7 @@ "typescript": "5.8.3", "uuid": "11.1.0", "vite": "6.3.3", - "vue": "3.5.12" + "vue": "3.5.14" }, "devDependencies": { "@misskey-dev/summaly": "5.2.1", @@ -49,7 +49,7 @@ "@typescript-eslint/eslint-plugin": "8.31.0", "@typescript-eslint/parser": "8.31.0", "@vitest/coverage-v8": "3.1.2", - "@vue/runtime-core": "3.5.12", + "@vue/runtime-core": "3.5.14", "acorn": "8.14.1", "cross-env": "7.0.3", "eslint-plugin-import": "2.31.0", diff --git a/packages/frontend-embed/src/style.scss b/packages/frontend-embed/src/style.scss index ba3238cd4c..529bb606be 100644 --- a/packages/frontend-embed/src/style.scss +++ b/packages/frontend-embed/src/style.scss @@ -101,7 +101,7 @@ rt { } } -.ti { +.ti, ph-lg { width: 1.28em; vertical-align: -12%; line-height: 1em; diff --git a/packages/frontend-embed/src/workers/tsconfig.json b/packages/frontend-embed/src/workers/tsconfig.json index 8ee8930465..39ba45ddbb 100644 --- a/packages/frontend-embed/src/workers/tsconfig.json +++ b/packages/frontend-embed/src/workers/tsconfig.json @@ -1,5 +1,6 @@ { "compilerOptions": { "lib": ["esnext", "webworker"], + "incremental": true } } diff --git a/packages/frontend-embed/tsconfig.json b/packages/frontend-embed/tsconfig.json index e0ee08188d..8db5776c91 100644 --- a/packages/frontend-embed/tsconfig.json +++ b/packages/frontend-embed/tsconfig.json @@ -23,6 +23,7 @@ "useDefineForClassFields": true, "verbatimModuleSyntax": true, "skipLibCheck": true, + "incremental": true, "baseUrl": ".", "paths": { "@/*": ["./src/*"], diff --git a/packages/frontend-shared/tsconfig.json b/packages/frontend-shared/tsconfig.json index 8f76763e10..0512b50caf 100644 --- a/packages/frontend-shared/tsconfig.json +++ b/packages/frontend-shared/tsconfig.json @@ -18,6 +18,7 @@ "esModuleInterop": true, "verbatimModuleSyntax": true, "skipLibCheck": true, + "incremental": true, "baseUrl": ".", "paths": { "@/*": ["./*"], diff --git a/packages/frontend/.storybook/tsconfig.json b/packages/frontend/.storybook/tsconfig.json index f325114522..18baf516ba 100644 --- a/packages/frontend/.storybook/tsconfig.json +++ b/packages/frontend/.storybook/tsconfig.json @@ -18,6 +18,7 @@ "esModuleInterop": true, "skipLibCheck": true, "forceConsistentCasingInFileNames": true, + "incremental": true, "jsx": "react", "jsxFactory": "h" }, diff --git a/packages/frontend/package.json b/packages/frontend/package.json index a20aba0201..f5c7bcf1b4 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -30,7 +30,7 @@ "@transfem-org/sfm-js": "0.24.6", "@twemoji/parser": "15.1.1", "@vitejs/plugin-vue": "5.2.3", - "@vue/compiler-sfc": "3.5.12", + "@vue/compiler-sfc": "3.5.14", "aiscript-vscode": "github:aiscript-dev/aiscript-vscode#v0.1.15", "astring": "1.9.0", "broadcast-channel": "7.1.0", @@ -76,7 +76,7 @@ "uuid": "11.1.0", "v-code-diff": "1.13.1", "vite": "6.3.3", - "vue": "3.5.12", + "vue": "3.5.14", "vuedraggable": "next", "wanakana": "5.3.1" }, @@ -119,8 +119,8 @@ "@typescript-eslint/eslint-plugin": "8.31.0", "@typescript-eslint/parser": "8.31.0", "@vitest/coverage-v8": "3.1.2", - "@vue/compiler-core": "3.5.12", - "@vue/runtime-core": "3.5.12", + "@vue/compiler-core": "3.5.14", + "@vue/runtime-core": "3.5.14", "acorn": "8.14.1", "cross-env": "7.0.3", "eslint-plugin-import": "2.31.0", diff --git a/packages/frontend/src/components/DynamicNote.vue b/packages/frontend/src/components/DynamicNote.vue index a5008e9ddf..67707bfda9 100644 --- a/packages/frontend/src/components/DynamicNote.vue +++ b/packages/frontend/src/components/DynamicNote.vue @@ -23,10 +23,10 @@ import type MkNote from '@/components/MkNote.vue'; import type SkNote from '@/components/SkNote.vue'; import { prefer } from '@/preferences'; -const XNote = computed(() => - prefer.r.noteDesign.value === 'misskey' - ? defineAsyncComponent(() => import('@/components/MkNote.vue')) - : defineAsyncComponent(() => import('@/components/SkNote.vue')), +const XNote = defineAsyncComponent(() => + prefer.s.noteDesign === 'misskey' + ? import('@/components/MkNote.vue') + : import('@/components/SkNote.vue') ); const rootEl = useTemplateRef>('rootEl'); diff --git a/packages/frontend/src/components/DynamicNoteDetailed.vue b/packages/frontend/src/components/DynamicNoteDetailed.vue index 21bf00ccbe..8594db2328 100644 --- a/packages/frontend/src/components/DynamicNoteDetailed.vue +++ b/packages/frontend/src/components/DynamicNoteDetailed.vue @@ -20,10 +20,10 @@ import type MkNoteDetailed from '@/components/MkNoteDetailed.vue'; import type SkNoteDetailed from '@/components/SkNoteDetailed.vue'; import { prefer } from '@/preferences'; -const XNoteDetailed = computed(() => - prefer.r.noteDesign.value === 'misskey' - ? defineAsyncComponent(() => import('@/components/MkNoteDetailed.vue')) - : defineAsyncComponent(() => import('@/components/SkNoteDetailed.vue')), +const XNoteDetailed = defineAsyncComponent(() => + prefer.s.noteDesign === 'misskey' + ? import('@/components/MkNoteDetailed.vue') + : import('@/components/SkNoteDetailed.vue'), ); const rootEl = useTemplateRef>('rootEl'); diff --git a/packages/frontend/src/components/DynamicNoteSimple.vue b/packages/frontend/src/components/DynamicNoteSimple.vue index 7ca345ccfa..5eaeaf6c23 100644 --- a/packages/frontend/src/components/DynamicNoteSimple.vue +++ b/packages/frontend/src/components/DynamicNoteSimple.vue @@ -21,10 +21,10 @@ import type MkNoteSimple from '@/components/MkNoteSimple.vue'; import type SkNoteSimple from '@/components/SkNoteSimple.vue'; import { prefer } from '@/preferences'; -const XNoteSimple = computed(() => - prefer.r.noteDesign.value === 'misskey' - ? defineAsyncComponent(() => import('@/components/MkNoteSimple.vue')) - : defineAsyncComponent(() => import('@/components/SkNoteSimple.vue')), +const XNoteSimple = defineAsyncComponent(() => + prefer.s.noteDesign === 'misskey' + ? import('@/components/MkNoteSimple.vue') + : import('@/components/SkNoteSimple.vue'), ); const rootEl = useTemplateRef>('rootEl'); diff --git a/packages/frontend/src/components/MkDateSeparatedList.vue b/packages/frontend/src/components/MkDateSeparatedList.vue index 63e6b74154..8cf4e5fa2d 100644 --- a/packages/frontend/src/components/MkDateSeparatedList.vue +++ b/packages/frontend/src/components/MkDateSeparatedList.vue @@ -16,6 +16,7 @@ import { instance } from '@/instance.js'; import { prefer } from '@/preferences.js'; import { getDateText } from '@/utility/timeline-date-separate.js'; import { $i } from '@/i.js'; +import SkTransitionGroup from '@/components/SkTransitionGroup.vue'; export default defineComponent({ props: { @@ -146,14 +147,12 @@ export default defineComponent({ [$style['direction-up']]: props.direction === 'up', }; - return () => prefer.s.animation ? h(TransitionGroup, { + return () => h(SkTransitionGroup, { class: classes, name: 'list', tag: 'div', onBeforeLeave, onLeaveCancelled, - }, { default: renderChildren }) : h('div', { - class: classes, }, { default: renderChildren }); }, }); diff --git a/packages/frontend/src/components/MkDonation.vue b/packages/frontend/src/components/MkDonation.vue index 43d2002204..dfdfc0a871 100644 --- a/packages/frontend/src/components/MkDonation.vue +++ b/packages/frontend/src/components/MkDonation.vue @@ -53,6 +53,7 @@ import { i18n } from '@/i18n.js'; import * as os from '@/os.js'; import { miLocalStorage } from '@/local-storage.js'; import { instance } from '@/instance.js'; +import { prefer } from '@/preferences.js'; const emit = defineEmits<{ (ev: 'closed'): void; @@ -66,6 +67,7 @@ function close() { } function neverShow() { + prefer.commit('neverShowDonationInfo', 'true'); miLocalStorage.setItem('neverShowDonationInfo', 'true'); close(); } diff --git a/packages/frontend/src/components/MkFolder.vue b/packages/frontend/src/components/MkFolder.vue index 2e5d0a3dea..4537bc9d82 100644 --- a/packages/frontend/src/components/MkFolder.vue +++ b/packages/frontend/src/components/MkFolder.vue @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only diff --git a/packages/frontend/src/components/global/MkStickyContainer.vue b/packages/frontend/src/components/global/MkStickyContainer.vue index 05245716c2..73ce393113 100644 --- a/packages/frontend/src/components/global/MkStickyContainer.vue +++ b/packages/frontend/src/components/global/MkStickyContainer.vue @@ -5,17 +5,17 @@ SPDX-License-Identifier: AGPL-3.0-only