mirrors/barkey
1
0
Fork 0
mirror of https://codeberg.org/yeentown/barkey.git synced 2025-07-06 03:56:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

merge: prepare release 2025.4.3 (!1125)

Browse source 
View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/1125

Approved-by: Marie <github@yuugi.dev>
Approved-by: Julia <julia@insertdomain.name>
This commit is contained in:
Julia 2025-06-19 21:35:18 +00:00
commit a77c32b17d
396 changed files with 10508 additions and 5098 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
.config/ci.yml
View file

@ -115,8 +115,14 @@ db:
user: postgres
pass: ci
# Whether disable Caching queries
#disableCache: true
## Log a warning to the server console if any query takes longer than this to complete.
## Measured in milliseconds; set to 0 to disable. (default: 300)
#slowQueryThreshold: 300
# If false, then query results will be cached in redis.
# If true (default), then queries will not be cached.
# This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior.
#disableCache: false
# Extra Connection options
#extra:

10
.config/cypress-devcontainer.yml
View file

@ -57,8 +57,14 @@ db:
user: postgres
pass: postgres
# Whether disable Caching queries
#disableCache: true
## Log a warning to the server console if any query takes longer than this to complete.
## Measured in milliseconds; set to 0 to disable. (default: 300)
#slowQueryThreshold: 300
# If false, then query results will be cached in redis.
# If true (default), then queries will not be cached.
# This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior.
#disableCache: false
# Extra Connection options
#extra:

10
.config/docker_example.yml
View file

@ -118,8 +118,14 @@ db:
user: example-misskey-user
pass: example-misskey-pass
# Whether disable Caching queries
#disableCache: true
## Log a warning to the server console if any query takes longer than this to complete.
## Measured in milliseconds; set to 0 to disable. (default: 300)
#slowQueryThreshold: 300
# If false, then query results will be cached in redis.
# If true (default), then queries will not be cached.
# This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior.
#disableCache: false
# Extra Connection options
#extra:

10
.config/example.yml
View file

@ -121,8 +121,14 @@ db:
user: sharkey
pass: example-misskey-pass
# Whether disable Caching queries
#disableCache: true
## Log a warning to the server console if any query takes longer than this to complete.
## Measured in milliseconds; set to 0 to disable. (default: 300)
#slowQueryThreshold: 300
# If false, then query results will be cached in redis.
# If true (default), then queries will not be cached.
# This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior.
#disableCache: false
# Extra Connection options
#extra:

4
.gitignore vendored
View file

@ -87,3 +87,7 @@ vite.config.local-dev.ts.timestamp-*
# Sharkey
/packages/megalodon/lib
# TypeScript
.tsbuildinfo
*.tsbuildinfo

31
CONTRIBUTING.md
View file

@ -622,6 +622,35 @@ marginはそのコンポーネントを使う側が設定する
### indexというファイル名を使うな
ESMではディレクトリインポートは廃止されているのと、ディレクトリインポートせずともファイル名が index だと何故か一部のライブラリ？でディレクトリインポートだと見做されてエラーになる
### Memory Caches
Sharkey offers multiple memory cache implementations, each meant for a different use case.
The following table compares the available options:
| Cache | Type | Consistency | Persistence | Data Source | Cardinality | Eviction | Description |
|---------------------|-----------|-------------|-------------|-------------|-------------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `MemoryKVCache` | Key-Value | None | None | Caller | Single | Lifetime | Implements a basic in-memory Key-Value store. The implementation is entirely synchronous, except for user-provided data sources. |
| `MemorySingleCache` | Single | None | None | Caller | Single | Lifetime | Implements a basic in-memory Single Value store. The implementation is entirely synchronous, except for user-provided data sources. |
| `RedisKVCache` | Key-Value | Eventual | Redis | Callback | Single | Lifetime | Extends `MemoryKVCache` with Redis-backed persistence and a pre-defined callback data source. This provides eventual consistency guarantees based on the memory cache lifetime. |
| `RedisSingleCache` | Single | Eventual | Redis | Callback | Single | Lifetime | Extends `MemorySingleCache` with Redis-backed persistence and a pre-defined callback data source. This provides eventual consistency guarantees based on the memory cache lifetime. |
| `QuantumKVCache` | Key-Value | Immediate | None | Callback | Multiple | Lifetime | Combines `MemoryKVCache` with a pre-defined callback data source and immediate consistency via Redis sync events. The implementation offers multi-item batch overloads for efficient bulk operations. **This is the recommended cache implementation for most use cases.** |
Key-Value caches store multiple entries per cache, while Single caches store a single value that can be accessed directly.
Consistency refers to the consistency of cached data between different processes in the instance cluster: "None" means no consistency guarantees, "Eventual" caches will gradually become consistent after some unknown time, and "Immediate" consistency ensures accurate data ASAP after the update.
Caches with persistence can retain their data after a reboot through an external service such as Redis.
If a data source is supported, then this allows the cache to directly load missing data in response to a fetch.
"Caller" data sources are passed into the fetch method(s) directly, while "Callback" sources are passed in as a function when the cache is first initialized.
The cardinality of a cache refers to the number of items that can be updated in a single operation, and eviction, finally, is the method that the cache uses to evict stale data.
#### Selecting a cache implementation
For most cache uses, `QuantumKVCache` should be considered first.
It offers strong consistency guarantees, multiple cardinality, and a cleaner API surface than the older caches.
An alternate cache implementation should be considered if any of the following apply:
* The data is particularly slow to calculate or difficult to access. In these cases, either `RedisKVCache` or `RedisSingleCache` should be considered.
* If stale data is acceptable, then consider `MemoryKVCache` or `MemorySingleCache`. These synchronous implementations have much less overhead than the other options.
* There is only one data item, or all data items must be fetched together. Using `MemorySingleCache` or `RedisSingleCache` could provide a cleaner implementation without resorting to hacks like a fixed key.
## CSS Recipe
### Lighten CSS vars
@ -690,7 +719,7 @@ seems to do a decent job)
* re-generate locales (`pnpm run build-assets`) and commit
* build the frontend: `rm -rf built/; NODE_ENV=development pnpm --filter=frontend --filter=frontend-embed --filter=frontend-shared build` (the `development` tells it to keep some of the original filenames in the built files)
* make sure there aren't any new `ti-*` classes (Tabler Icons), and replace them with appropriate `ph-*` ones (Phosphor Icons) in [`vite.replaceicons.ts`](packages/frontend/vite.replaceIcons.ts).
* This command should show you want to change: `grep -ohrP '(?<=["'\'']ti )(ti-(?!fw)[\w\-]+)' --exclude \*.map -- built/ | sort -u`.
* This command should show you want to change: `grep -ohrP '(?<=["'\''](ti )?)(ti-(?!fw)[\w\-]+)' --exclude \*.map -- built/ | sort -u`.
* NOTE: `ti-fw` is a special class that's defined by Misskey, leave it alone.
* After every change, re-build the frontend and check again, until there are no more `ti-*` classes in the built files.
* Commit!

2
Dockerfile
View file

@ -38,7 +38,7 @@ ARG UID="991"
ARG GID="991"
ENV COREPACK_DEFAULT_TO_LATEST=0
RUN apk add ffmpeg tini jemalloc pixman pango cairo libpng \
RUN apk add ffmpeg tini jemalloc pixman pango cairo libpng librsvg font-noto font-noto-cjk font-noto-thai \
&& corepack enable \
&& addgroup -g "${GID}" sharkey \
&& adduser -D -u "${UID}" -G sharkey -h /sharkey sharkey \

3
cypress/tsconfig.json
View file

@ -2,7 +2,8 @@
"compilerOptions": {
"lib": ["dom", "es5"],
"target": "es5",
"types": ["cypress", "node"]
"types": ["cypress", "node"],
"incremental": true
},
"include": ["./**/*.ts"]
}

132
locales/index.d.ts vendored
View file

@ -12492,6 +12492,14 @@ export interface Locale extends ILocale {
* Displays content centered.
*/
"centerDescription": string;
/**
* Unix Time
*/
"unixtime": string;
/**
* Displays a timestamp in the viewer's current timezone.
*/
"unixtimeDescription": string;
/**
* Code (Inline)
*/
@ -13069,6 +13077,26 @@ export interface Locale extends ILocale {
* Users popular on {name}
*/
"popularUsersLocal": ParameterizedString<"name">;
/**
* Polls trending on {name}
*/
"pollsOnLocal": ParameterizedString<"name">;
/**
* Polls trending on the global network
*/
"pollsOnRemote": string;
/**
* Polls that have ended recently
*/
"pollsExpired": string;
/**
* Trending polls are disabled on this instance.
*/
"trendingPollsDisabled": string;
/**
* Please log in to view trending polls.
*/
"trendingPollsDisabledLogIn": string;
/**
* Silenced
*/
@ -13129,6 +13157,110 @@ export interface Locale extends ILocale {
* Timeout in milliseconds for translation API requests.
*/
"translationTimeoutCaption": string;
/**
* Staff notes
*/
"staffNotes": string;
/**
* Icon of {name}
*/
"instanceIconAlt": ParameterizedString<"name">;
/**
* Attribution Domains
*/
"attributionDomains": string;
/**
* A list of domains whose content can be attributed to you on link previews, separated by new-line. Any subdomain will also be valid. The following needs to be on the webpage:
*/
"attributionDomainsDescription": string;
/**
* Written by {user}
*/
"writtenBy": ParameterizedString<"user">;
/**
* Following (Pub)
*/
"followingPub": string;
/**
* Followers (Sub)
*/
"followersSub": string;
/**
* Well-known resources
*/
"wellKnownResources": string;
/**
* Last posted: {at}
*/
"lastPosted": ParameterizedString<"at">;
/**
* NSFW
*/
"nsfw": string;
/**
* Raw
*/
"raw": string;
/**
* CW
*/
"cw": string;
/**
* Media Silenced
*/
"mediaSilenced": string;
/**
* Bubble
*/
"bubble": string;
/**
* Verified
*/
"verified": string;
/**
* Not Verified
*/
"notVerified": string;
/**
* Hibernated
*/
"hibernated": string;
/**
* When replying to a post with a Content Warning, automatically use the same CW for the reply.
*/
"keepCwDescription": string;
/**
* Disabled (do not copy CWs)
*/
"keepCwDisabled": string;
/**
* Enabled (copy CWs verbatim)
*/
"keepCwEnabled": string;
/**
* Enabled (copy CW and prepend "RE:")
*/
"keepCwPrependRe": string;
/**
* Note controls
*/
"noteFooterLabel": string;
/**
* Packed user data in its raw form. Most of these fields are public and visible to all users.
*/
"rawUserDescription": string;
/**
* Extended user data in its raw form. These fields are private and can only be accessed by moderators.
*/
"rawInfoDescription": string;
/**
* ActivityPub user data in its raw form. These fields are public and accessible to other instances.
*/
"rawApDescription": string;
/**
* Signup Reason
*/
"signupReason": string;
}
declare const locales: {
[lang: string]: Locale;

30
package.json
View file

@ -1,6 +1,6 @@
{
"name": "sharkey",
"version": "2025.4.2",
"version": "2025.4.3",
"codename": "shonk",
"repository": {
"type": "git",
@ -54,17 +54,7 @@
"lodash": "4.17.21"
},
"dependencies": {
"cssnano": "7.0.6",
"esbuild": "0.25.3",
"execa": "9.5.2",
"fast-glob": "3.3.3",
"glob": "11.0.2",
"ignore-walk": "7.0.0",
"js-yaml": "4.1.0",
"postcss": "8.5.3",
"tar": "7.4.3",
"terser": "5.39.0",
"typescript": "5.8.3"
"js-yaml": "4.1.0"
},
"optionalDependencies": {
"cypress": "14.3.2"
@ -75,10 +65,20 @@
"@typescript-eslint/eslint-plugin": "8.31.0",
"@typescript-eslint/parser": "8.31.0",
"cross-env": "7.0.3",
"cssnano": "7.0.6",
"esbuild": "0.25.3",
"eslint": "9.25.1",
"globals": "16.0.0",
"execa": "9.5.2",
"fast-glob": "3.3.3",
"glob": "11.0.2",
"globals": "16.1.0",
"ncp": "2.0.0",
"pnpm": "10.10.0",
"start-server-and-test": "2.0.11"
"pnpm": "9.6.0",
"ignore-walk": "7.0.0",
"postcss": "8.5.3",
"start-server-and-test": "2.0.11",
"tar": "7.4.3",
"terser": "5.39.0",
"typescript": "5.8.3"
}
}

42
packages/backend/migration/1747938628395-add-missing-indexes.js
View file

@ -4,19 +4,35 @@
*/
export class AddMissingIndexes1747938628395 {
name = 'AddMissingIndexes1747938628395'
name = 'AddMissingIndexes1747938628395'
async up(queryRunner) {
await queryRunner.query(`CREATE INDEX "IDX_58699f75b9cf904f5f007909cb" ON "user_profile" ("birthday") `);
await queryRunner.query(`CREATE INDEX "IDX_021015e6683570ae9f6b0c62be" ON "user_list_membership" ("userId") `);
await queryRunner.query(`CREATE INDEX "IDX_cddcaf418dc4d392ecfcca842a" ON "user_list_membership" ("userListId") `);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_e4f3094c43f2d665e6030b0337" ON "user_list_membership" ("userId", "userListId") `);
}
async up(queryRunner) {
// Some instances have duplicate list entries
await queryRunner.query(`
DELETE FROM "user_list_membership"
WHERE "id" NOT IN (
SELECT MIN("id")
FROM "user_list_membership"
GROUP BY "userId", "userListId"
)`);
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_e4f3094c43f2d665e6030b0337"`);
await queryRunner.query(`DROP INDEX "public"."IDX_cddcaf418dc4d392ecfcca842a"`);
await queryRunner.query(`DROP INDEX "public"."IDX_021015e6683570ae9f6b0c62be"`);
await queryRunner.query(`DROP INDEX "public"."IDX_58699f75b9cf904f5f007909cb"`);
}
// Some instances already have these indexes, for an unknown reason
await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_e4f3094c43f2d665e6030b0337"`);
await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_cddcaf418dc4d392ecfcca842a"`);
await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_021015e6683570ae9f6b0c62be"`);
await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_58699f75b9cf904f5f007909cb"`);
// Now the actual migration
await queryRunner.query(`CREATE INDEX "IDX_58699f75b9cf904f5f007909cb" ON "user_profile" ("birthday") `);
await queryRunner.query(`CREATE INDEX "IDX_021015e6683570ae9f6b0c62be" ON "user_list_membership" ("userId") `);
await queryRunner.query(`CREATE INDEX "IDX_cddcaf418dc4d392ecfcca842a" ON "user_list_membership" ("userListId") `);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_e4f3094c43f2d665e6030b0337" ON "user_list_membership" ("userId", "userListId") `);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_e4f3094c43f2d665e6030b0337"`);
await queryRunner.query(`DROP INDEX "public"."IDX_cddcaf418dc4d392ecfcca842a"`);
await queryRunner.query(`DROP INDEX "public"."IDX_021015e6683570ae9f6b0c62be"`);
await queryRunner.query(`DROP INDEX "public"."IDX_58699f75b9cf904f5f007909cb"`);
}
}

16
packages/backend/migration/1748096357260-AddAttributionDomains.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: piuvas and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class AddAttributionDomains1748096357260 {
name = 'AddAttributionDomains1748096357260'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "user" ADD "attributionDomains" text array NOT NULL DEFAULT '{}'`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "attributionDomains"`);
}
}

16
packages/backend/migration/1748104955717-index_IDX_instance_host_key.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class IndexIDXInstanceHostKey1748104955717 {
name = 'IndexIDXInstanceHostKey1748104955717'
async up(queryRunner) {
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_instance_host_key" ON "instance" (((lower(reverse("host")) || '.')::text) text_pattern_ops)`);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "IDX_instance_host_key"`);
}
}

85
packages/backend/migration/1748105111513-add_instance_block_columns.js Normal file
View file

@ -0,0 +1,85 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
* @typedef {{ blockedHosts: string[], silencedHosts: string[], mediaSilencedHosts: string[], federationHosts: string[], bubbleInstances: string[] }} Meta
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AddInstanceBlockColumns1748105111513 {
name = 'AddInstanceBlockColumns1748105111513'
async up(queryRunner) {
// Schema migration
await queryRunner.query(`ALTER TABLE "instance" ADD "isBlocked" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isBlocked" IS 'True if this instance is blocked from federation.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isAllowListed" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isAllowListed" IS 'True if this instance is allow-listed.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isBubbled" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isBubbled" IS 'True if this instance is part of the local bubble.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isSilenced" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isSilenced" IS 'True if this instance is silenced.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isMediaSilenced" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isMediaSilenced" IS 'True if this instance is media-silenced.'`);
// Data migration
/** @type {Meta[]} */
const metas = await queryRunner.query(`SELECT "blockedHosts", "silencedHosts", "mediaSilencedHosts", "federationHosts", "bubbleInstances" FROM "meta"`);
if (metas.length > 0) {
/** @type {Meta} */
const meta = metas[0];
// Blocked hosts
if (meta.blockedHosts.length > 0) {
const patterns = buildPatterns(meta.blockedHosts);
await queryRunner.query(`UPDATE "instance" SET "isBlocked" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Silenced hosts
if (meta.silencedHosts.length > 0) {
const patterns = buildPatterns(meta.silencedHosts);
await queryRunner.query(`UPDATE "instance" SET "isSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Media silenced hosts
if (meta.mediaSilencedHosts.length > 0) {
const patterns = buildPatterns(meta.mediaSilencedHosts);
await queryRunner.query(`UPDATE "instance" SET "isMediaSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Allow-listed hosts
if (meta.federationHosts.length > 0) {
const patterns = buildPatterns(meta.federationHosts);
await queryRunner.query(`UPDATE "instance" SET "isAllowListed" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Bubbled hosts
if (meta.bubbleInstances.length > 0) {
const patterns = buildPatterns(meta.bubbleInstances);
await queryRunner.query(`UPDATE "instance" SET "isBubbled" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
}
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isMediaSilenced"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isSilenced"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isBubbled"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isAllowListed"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isBlocked"`);
}
}
/**
* @param {string[]} input
* @returns {string[]}
*/
function buildPatterns(input) {
return input.map(i => i.toLowerCase().split('').reverse().join('') + '.%');
}

44
packages/backend/migration/1748128176881-add_instance_foreign_keys.js Normal file
View file

@ -0,0 +1,44 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AddInstanceForeignKeys1748128176881 {
name = 'AddInstanceForeignKeys1748128176881'
async up(queryRunner) {
// Fix-up: Some older instances have users without a matching instance entry
await queryRunner.query(`
INSERT INTO "instance" ("id", "host", "firstRetrievedAt")
SELECT
MIN("id"),
"host",
COALESCE(MIN("lastFetchedAt"), CURRENT_TIMESTAMP)
FROM "user"
WHERE
"host" IS NOT NULL AND
NOT EXISTS (select 1 from "instance" where "instance"."host" = "user"."host")
GROUP BY "host"
`);
await queryRunner.query(`ALTER TABLE "user" ADD CONSTRAINT "FK_user_host" FOREIGN KEY ("host") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_userHost" FOREIGN KEY ("userHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_replyUserHost" FOREIGN KEY ("replyUserHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_renoteUserHost" FOREIGN KEY ("renoteUserHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_renoteUserHost"`);
await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_replyUserHost"`);
await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_userHost"`);
await queryRunner.query(`ALTER TABLE "user" DROP CONSTRAINT "FK_user_host"`);
}
}

26
packages/backend/migration/1748137683887-add_instance_foreign_keys_to_following.js Normal file
View file

@ -0,0 +1,26 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AddInstanceForeignKeysToFollowing1748137683887 {
name = 'AddInstanceForeignKeysToFollowing1748137683887'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "following" ADD CONSTRAINT "FK_following_followerHost" FOREIGN KEY ("followerHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "following" ADD CONSTRAINT "FK_following_followeeHost" FOREIGN KEY ("followeeHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "following" DROP CONSTRAINT "FK_following_followeeHost"`);
await queryRunner.query(`ALTER TABLE "following" DROP CONSTRAINT "FK_following_followerHost"`);
}
}

25
packages/backend/migration/1748191631151-analyze_instance-user-note-following.js Normal file
View file

@ -0,0 +1,25 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AnalyzeInstanceUserNoteFollowing1748191631151 {
name = 'AnalyzeInstanceUserNoteFollowing1748191631151'
async up(queryRunner) {
// Refresh statistics for tables impacted by new indexes.
// This helps the query planner to efficiently use them without waiting for the next full vacuum.
await queryRunner.query(`ANALYZE "instance", "user", "following", "note"`);
}
async down(queryRunner) {
}
}

22
packages/backend/migration/1748990452958-replace_note-userHost_index.js Normal file
View file

@ -0,0 +1,22 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class ReplaceNoteUserHostIndex1748990452958 {
name = 'ReplaceNoteUserHostIndex1748990452958'
async up(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_7125a826ab192eb27e11d358a5"`);
await queryRunner.query(`
create index "IDX_note_userHost_id"
on "note" ("userHost", "id" desc)
nulls not distinct`);
await queryRunner.query(`comment on index "IDX_note_userHost_id" is 'User host with ID included'`);
}
async down(queryRunner) {
await queryRunner.query(`drop index if exists "IDX_note_userHost_id"`);
await queryRunner.query(`CREATE INDEX "IDX_7125a826ab192eb27e11d358a5" ON "note" ("userHost") `);
}
}

22
packages/backend/migration/1748990662839-fix-IDX_instance_host_key.js Normal file
View file

@ -0,0 +1,22 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class FixIDXInstanceHostKey1748990662839 {
async up(queryRunner) {
// must include host for index-only scans: https://www.postgresql.org/docs/current/indexes-index-only-scans.html
await queryRunner.query(`DROP INDEX "public"."IDX_instance_host_key"`);
await queryRunner.query(`
create index "IDX_instance_host_key"
on "instance" ((lower(reverse("host"::text)) || '.'::text) text_pattern_ops)
include ("host")
`);
await queryRunner.query(`comment on index "IDX_instance_host_key" is 'Expression index for finding instances by base domain'`);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_instance_host_key"`);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_instance_host_key" ON "instance" (((lower(reverse("host")) || '.')::text) text_pattern_ops)`);
}
}

19
packages/backend/migration/1748991828473-create-IDX_note_for_timelines.js Normal file
View file

@ -0,0 +1,19 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class CreateIDXNoteForTimelines1748991828473 {
async up(queryRunner) {
await queryRunner.query(`
create index "IDX_note_for_timelines"
on "note" ("id" desc, "channelId", "visibility", "userHost")
include ("userId", "userHost", "replyId", "replyUserId", "replyUserHost", "renoteId", "renoteUserId", "renoteUserHost")
NULLS NOT DISTINCT`);
await queryRunner.query(`comment on index "IDX_note_for_timelines" is 'Covering index for timeline queries'`);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "IDX_note_for_timelines"`);
}
}

17
packages/backend/migration/1748992017688-create-IDX_instance_host_filters.js Normal file
View file

@ -0,0 +1,17 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class CreateIDXInstanceHostFilters1748992017688 {
async up(queryRunner) {
await queryRunner.query(`
create index "IDX_instance_host_filters"
on "instance" ("host", "isBlocked", "isSilenced", "isMediaSilenced", "isAllowListed", "isBubbled", "suspensionState")`);
await queryRunner.query(`comment on index "IDX_instance_host_filters" is 'Covering index for host filter queries'`);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "IDX_instance_host_filters"`);
}
}

28
packages/backend/migration/1748992128683-create-statistics.js Normal file
View file

@ -0,0 +1,28 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class CreateStatistics1748992128683 {
async up(queryRunner) {
await queryRunner.query(`CREATE STATISTICS "STTS_instance_isBlocked_isBubbled" (mcv) ON "isBlocked", "isBubbled" FROM "instance"`);
await queryRunner.query(`CREATE STATISTICS "STTS_instance_isBlocked_isSilenced" (mcv) ON "isBlocked", "isSilenced" FROM "instance"`);
await queryRunner.query(`CREATE STATISTICS "STTS_note_replyId_replyUserId_replyUserHost" (dependencies) ON "replyId", "replyUserId", "replyUserHost" FROM "note"`)
await queryRunner.query(`CREATE STATISTICS "STTS_note_renoteId_renoteUserId_renoteUserHost" (dependencies) ON "renoteId", "renoteUserId", "renoteUserHost" FROM "note"`);
await queryRunner.query(`CREATE STATISTICS "STTS_note_userId_userHost" (mcv) ON "userId", "userHost" FROM "note"`);
await queryRunner.query(`CREATE STATISTICS "STTS_note_replyUserId_replyUserHost" (mcv) ON "replyUserId", "replyUserHost" FROM "note"`);
await queryRunner.query(`CREATE STATISTICS "STTS_note_renoteUserId_renoteUserHost" (mcv) ON "renoteUserId", "renoteUserHost" FROM "note"`);
await queryRunner.query(`ANALYZE "note", "instance"`);
}
async down(queryRunner) {
await queryRunner.query(`DROP STATISTICS "STTS_instance_isBlocked_isBubbled"`);
await queryRunner.query(`DROP STATISTICS "STTS_instance_isBlocked_isSilenced"`);
await queryRunner.query(`DROP STATISTICS "STTS_note_replyId_replyUserId_replyUserHost"`);
await queryRunner.query(`DROP STATISTICS "STTS_note_renoteId_renoteUserId_renoteUserHost"`);
await queryRunner.query(`DROP STATISTICS "STTS_note_userId_userHost"`);
await queryRunner.query(`DROP STATISTICS "STTS_note_replyUserId_replyUserHost"`);
await queryRunner.query(`DROP STATISTICS "STTS_note_renoteUserId_renoteUserHost"`);
await queryRunner.query(`ANALYZE "note", "instance"`);
}
}

28
packages/backend/migration/1749097536193-fix-IDX_note_for_timeline.js Normal file
View file

@ -0,0 +1,28 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class FixIDXNoteForTimeline1749097536193 {
async up(queryRunner) {
await queryRunner.query('drop index "IDX_note_for_timelines"');
await queryRunner.query(`
create index "IDX_note_for_timelines"
on "note" ("id" desc, "channelId", "visibility", "userHost")
include ("userId", "replyId", "replyUserId", "replyUserHost", "renoteId", "renoteUserId", "renoteUserHost", "threadId")
NULLS NOT DISTINCT
`);
await queryRunner.query(`comment on index "IDX_note_for_timelines" is 'Covering index for timeline queries'`);
}
async down(queryRunner) {
await queryRunner.query('drop index "IDX_note_for_timelines"');
await queryRunner.query(`
create index "IDX_note_for_timelines"
on "note" ("id" desc, "channelId", "visibility", "userHost")
include ("userId", "userHost", "replyId", "replyUserId", "replyUserHost", "renoteId", "renoteUserId", "renoteUserHost")
NULLS NOT DISTINCT
`);
await queryRunner.query(`comment on index "IDX_note_for_timelines" is 'Covering index for timeline queries'`);
}
}

16
packages/backend/migration/1749229288946-create-IDX_note_url.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class CreateIDXNoteUrl1749229288946 {
name = 'CreateIDXNoteUrl1749229288946'
async up(queryRunner) {
await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_note_url" ON "note" ("url") `);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_note_url"`);
}
}

17
packages/backend/migration/1749267016885-remove-IDX_instance_host_filters.js Normal file
View file

@ -0,0 +1,17 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class RemoveIDXInstanceHostFilters1749267016885 {
async up(queryRunner) {
await queryRunner.query(`DROP INDEX IF EXISTS "IDX_instance_host_filters"`);
}
async down(queryRunner) {
await queryRunner.query(`
create index "IDX_instance_host_filters"
on "instance" ("host", "isBlocked", "isSilenced", "isMediaSilenced", "isAllowListed", "isBubbled", "suspensionState")`);
await queryRunner.query(`comment on index "IDX_instance_host_filters" is 'Covering index for host filter queries'`);
}
}

55
packages/backend/package.json
View file

@ -10,6 +10,9 @@
"start": "node ./built/boot/entry.js",
"start:test": "cross-env NODE_ENV=test node ./built/boot/entry.js",
"migrate": "pnpm typeorm migration:run -d ormconfig.js",
"migrate:revert": "pnpm typeorm migration:revert -d ormconfig.js",
"migrate:generate": "pnpm typeorm migration:generate -d ormconfig.js",
"migrate:create": "pnpm typeorm migration:create",
"revert": "pnpm typeorm migration:revert -d ormconfig.js",
"check:connect": "node ./scripts/check_connect.js",
"build": "swc src -d built -D --strip-leading-paths",
@ -77,7 +80,7 @@
"@fastify/static": "8.1.1",
"@fastify/view": "10.0.2",
"@misskey-dev/sharp-read-bmp": "1.3.0",
"@misskey-dev/summaly": "5.2.1",
"@misskey-dev/summaly": "npm:@transfem-org/summaly@5.2.2",
"@nestjs/common": "11.1.0",
"@nestjs/core": "11.1.0",
"@nestjs/testing": "11.1.0",
@ -87,33 +90,30 @@
"@simplewebauthn/server": "12.0.0",
"@sinonjs/fake-timers": "11.3.1",
"@smithy/node-http-handler": "2.5.0",
"@swc/cli": "0.7.3",
"@swc/core": "1.11.24",
"@transfem-org/sfm-js": "0.24.6",
"mfm-js": "npm:@transfem-org/sfm-js@0.24.6",
"@twemoji/parser": "15.1.1",
"accepts": "1.3.8",
"ajv": "8.17.1",
"archiver": "7.0.1",
"argon2": "^0.40.1",
"argon2": "0.43.0",
"axios": "1.7.4",
"async-mutex": "0.5.0",
"bcryptjs": "2.4.3",
"blurhash": "2.0.5",
"body-parser": "1.20.3",
"bullmq": "5.51.1",
"cacheable-lookup": "7.0.0",
"canvas": "^3.1.0",
"canvas": "3.1.0",
"cbor": "9.0.2",
"chalk": "5.4.1",
"chalk-template": "1.1.0",
"cheerio": "1.0.0",
"chokidar": "3.6.0",
"cli-highlight": "2.1.11",
"cli-highlight": "npm:@transfem-org/cli-highlight@2.1.12",
"color-convert": "2.0.1",
"content-disposition": "0.5.4",
"date-fns": "2.30.0",
"deep-email-validator": "0.1.21",
"fast-xml-parser": "4.4.1",
"dom-serializer": "2.0.0",
"domhandler": "5.0.3",
"domutils": "3.2.2",
"fastify": "5.3.2",
"fastify-raw-body": "5.0.0",
"feed": "4.2.2",
@ -122,10 +122,9 @@
"form-data": "4.0.2",
"glob": "11.0.0",
"got": "14.4.7",
"happy-dom": "16.8.1",
"hpagent": "1.2.0",
"htmlescape": "1.1.1",
"http-link-header": "1.1.3",
"htmlparser2": "9.1.0",
"ioredis": "5.6.1",
"ip-cidr": "4.0.2",
"ipaddr.js": "2.2.0",
@ -133,49 +132,39 @@
"js-yaml": "4.1.0",
"json5": "2.2.3",
"jsonld": "8.3.3",
"jsrsasign": "11.1.0",
"juice": "11.0.1",
"megalodon": "workspace:*",
"meilisearch": "0.50.0",
"microformats-parser": "2.0.2",
"mime-types": "2.1.35",
"misskey-js": "workspace:*",
"misskey-reversi": "workspace:*",
"moment": "^2.30.1",
"moment": "2.30.1",
"ms": "3.0.0-canary.1",
"nanoid": "5.1.5",
"nested-property": "4.0.0",
"node-fetch": "3.3.2",
"nodemailer": "6.10.1",
"oauth": "0.10.2",
"oauth2orize": "1.12.0",
"oauth2orize-pkce": "0.1.2",
"os-utils": "0.0.14",
"otpauth": "9.4.0",
"parse5": "7.3.0",
"pg": "8.15.6",
"pkce-challenge": "4.1.0",
"probe-image-size": "7.2.3",
"promise-limit": "2.7.0",
"proxy-addr": "^2.0.7",
"psl": "^1.13.0",
"proxy-addr": "2.0.7",
"psl": "1.15.0",
"pug": "3.0.3",
"qrcode": "1.5.4",
"random-seed": "0.3.0",
"ratelimiter": "3.4.1",
"re2": "1.21.4",
"redis-info": "3.1.0",
"redis-lock": "0.1.4",
"reflect-metadata": "0.2.2",
"rename": "1.0.4",
"rss-parser": "3.13.0",
"rxjs": "7.8.2",
"sanitize-html": "2.16.0",
"secure-json-parse": "3.0.2",
"sharp": "0.34.1",
"slacc": "0.0.10",
"strict-event-emitter-types": "2.0.0",
"stringz": "2.1.0",
"systeminformation": "5.25.11",
"tinycolor2": "1.6.0",
"tmp": "0.2.3",
@ -184,7 +173,7 @@
"typeorm": "0.3.22",
"typescript": "5.8.3",
"ulid": "2.4.0",
"uuid": "^9.0.1",
"uuid": "11.1.0",
"vary": "1.1.2",
"web-push": "3.6.7",
"ws": "8.18.1",
@ -195,16 +184,16 @@
"@nestjs/platform-express": "11.1.0",
"@sentry/vue": "9.14.0",
"@simplewebauthn/types": "12.0.0",
"@swc/cli": "0.7.3",
"@swc/core": "1.11.24",
"@swc/jest": "0.2.38",
"@types/accepts": "1.3.7",
"@types/archiver": "6.0.3",
"@types/bcryptjs": "2.4.6",
"@types/body-parser": "1.19.5",
"@types/color-convert": "2.0.4",
"@types/content-disposition": "0.5.8",
"@types/fluent-ffmpeg": "2.1.27",
"@types/htmlescape": "1.1.3",
"@types/http-link-header": "1.0.7",
"@types/jest": "29.5.14",
"@types/js-yaml": "4.0.9",
"@types/jsonld": "1.5.15",
@ -217,12 +206,11 @@
"@types/oauth2orize": "1.11.5",
"@types/oauth2orize-pkce": "0.1.2",
"@types/pg": "8.11.14",
"@types/proxy-addr": "^2.0.3",
"@types/psl": "^1.1.3",
"@types/proxy-addr": "2.0.3",
"@types/psl": "1.1.3",
"@types/pug": "2.0.10",
"@types/qrcode": "1.5.5",
"@types/random-seed": "0.3.5",
"@types/ratelimiter": "3.4.6",
"@types/redis-info": "3.0.3",
"@types/rename": "1.0.7",
"@types/sanitize-html": "2.15.0",
@ -232,7 +220,6 @@
"@types/supertest": "6.0.3",
"@types/tinycolor2": "1.4.6",
"@types/tmp": "0.2.6",
"@types/uuid": "^9.0.4",
"@types/vary": "1.1.3",
"@types/web-push": "3.6.4",
"@types/ws": "8.18.1",
@ -241,7 +228,7 @@
"aws-sdk-client-mock": "4.1.0",
"cross-env": "7.0.3",
"eslint-plugin-import": "2.31.0",
"execa": "8.0.1",
"execa": "9.5.2",
"fkill": "9.0.0",
"jest": "29.7.0",
"jest-mock": "29.7.0",

40
packages/backend/src/boot/entry.ts
View file

@ -9,6 +9,7 @@
import cluster from 'node:cluster';
import { EventEmitter } from 'node:events';
import { inspect } from 'node:util';
import chalk from 'chalk';
import Xev from 'xev';
import Logger from '@/logger.js';
@ -53,20 +54,45 @@ async function main() {
// Display detail of unhandled promise rejection
if (!envOption.quiet) {
process.on('unhandledRejection', console.dir);
process.on('unhandledRejection', e => {
try {
logger.error('Unhandled rejection:', inspect(e));
} catch {
console.error('Unhandled rejection:', inspect(e));
}
});
}
// Display detail of uncaught exception
process.on('uncaughtException', err => {
process.on('uncaughtExceptionMonitor', ((err, origin) => {
try {
logger.error(err);
console.trace(err);
} catch { }
});
logger.error(`Uncaught exception (${origin}):`, err);
} catch {
console.error(`Uncaught exception (${origin}):`, err);
}
}));
// Dying away...
process.on('disconnect', () => {
try {
logger.warn('IPC channel disconnected! The process may soon die.');
} catch {
console.warn('IPC channel disconnected! The process may soon die.');
}
});
process.on('beforeExit', code => {
try {
logger.warn(`Event loop died! Process will exit with code ${code}.`);
} catch {
console.warn(`Event loop died! Process will exit with code ${code}.`);
}
});
process.on('exit', code => {
logger.info(`The process is going to exit with code ${code}`);
try {
logger.info(`The process is going to exit with code ${code}`);
} catch {
console.info(`The process is going to exit with code ${code}`);
}
});
//#endregion

14
packages/backend/src/boot/master.ts
View file

@ -74,7 +74,7 @@ export async function masterMain() {
process.exit(1);
}
bootLogger.succ('Sharkey initialized');
bootLogger.info('Sharkey initialized');
if (config.sentryForBackend) {
Sentry.init({
@ -140,10 +140,10 @@ export async function masterMain() {
}
if (envOption.onlyQueue) {
bootLogger.succ('Queue started', null, true);
bootLogger.info('Queue started', null, true);
} else {
const addressString = net.isIPv6(config.address) ? `[${config.address}]` : config.address;
bootLogger.succ(config.socket ? `Now listening on socket ${config.socket} on ${config.url}` : `Now listening on ${addressString}:${config.port} on ${config.url}`, null, true);
bootLogger.info(config.socket ? `Now listening on socket ${config.socket} on ${config.url}` : `Now listening on ${addressString}:${config.port} on ${config.url}`, null, true);
}
}
@ -172,7 +172,7 @@ function loadConfigBoot(): Config {
config = loadConfig();
} catch (exception) {
if (typeof exception === 'string') {
configLogger.error(exception);
configLogger.error('Exception loading config:', exception);
process.exit(1);
} else if ((exception as any).code === 'ENOENT') {
configLogger.error('Configuration file not found', null, true);
@ -181,7 +181,7 @@ function loadConfigBoot(): Config {
throw exception;
}
configLogger.succ('Loaded');
configLogger.info('Loaded');
return config;
}
@ -195,7 +195,7 @@ async function connectDb(): Promise<void> {
dbLogger.info('Connecting...');
await initDb();
const v = await db.query('SHOW server_version').then(x => x[0].server_version);
dbLogger.succ(`Connected: v${v}`);
dbLogger.info(`Connected: v${v}`);
} catch (err) {
dbLogger.error('Cannot connect', null, true);
dbLogger.error(err);
@ -211,7 +211,7 @@ async function spawnWorkers(limit = 1) {
bootLogger.info(`Starting ${workers} worker${workers === 1 ? '' : 's'}...`);
await Promise.all([...Array(workers)].map(spawnWorker));
bootLogger.succ('All workers started');
bootLogger.info('All workers started');
}
function spawnWorker(): Promise<void> {

26
packages/backend/src/config.ts
View file

@ -9,6 +9,7 @@ import { dirname, resolve } from 'node:path';
import * as yaml from 'js-yaml';
import { globSync } from 'glob';
import ipaddr from 'ipaddr.js';
import Logger from './logger.js';
import type * as Sentry from '@sentry/node';
import type * as SentryVue from '@sentry/vue';
import type { RedisOptions } from 'ioredis';
@ -40,6 +41,7 @@ type Source = {
db?: string;
user?: string;
pass?: string;
slowQueryThreshold?: number;
disableCache?: boolean;
extra?: { [x: string]: string };
};
@ -155,6 +157,8 @@ type Source = {
}
};
const configLogger = new Logger('config');
export type PrivateNetworkSource = string | { network?: string, ports?: number[] };
export type PrivateNetwork = {
@ -192,7 +196,7 @@ export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefine
}
}
console.warn('[config] Skipping invalid entry in allowedPrivateNetworks: ', e);
configLogger.warn('Skipping invalid entry in allowedPrivateNetworks: ', e);
return null;
})
.filter(p => p != null);
@ -222,6 +226,7 @@ export type Config = {
db: string;
user: string;
pass: string;
slowQueryThreshold?: number;
disableCache?: boolean;
extra?: { [x: string]: string };
};
@ -375,11 +380,14 @@ export function loadConfig(): Config {
if (configFiles.length === 0
&& !process.env['MK_WARNED_ABOUT_CONFIG']) {
console.log('No config files loaded, check if this is intentional');
configLogger.warn('No config files loaded, check if this is intentional');
process.env['MK_WARNED_ABOUT_CONFIG'] = '1';
}
const config = configFiles.map(path => fs.readFileSync(path, 'utf-8'))
const config = configFiles.map(path => {
configLogger.info(`Reading configuration from ${path}`);
return fs.readFileSync(path, 'utf-8');
})
.map(contents => yaml.load(contents) as Source)
.reduce(
(acc: Source, cur: Source) => Object.assign(acc, cur),
@ -405,6 +413,10 @@ export function loadConfig(): Config {
const internalMediaProxy = `${scheme}://${host}/proxy`;
const redis = convertRedisOptions(config.redis, host);
// nullish => 300 (default)
// 0 => undefined (disabled)
const slowQueryThreshold = (config.db.slowQueryThreshold ?? 300) || undefined;
return {
version,
publishTarballInsteadOfProvideRepositoryUrl: !!config.publishTarballInsteadOfProvideRepositoryUrl,
@ -423,7 +435,7 @@ export function loadConfig(): Config {
apiUrl: `${scheme}://${host}/api`,
authUrl: `${scheme}://${host}/auth`,
driveUrl: `${scheme}://${host}/files`,
db: { ...config.db, db: dbDb, user: dbUser, pass: dbPass },
db: { ...config.db, db: dbDb, user: dbUser, pass: dbPass, slowQueryThreshold },
dbReplications: config.dbReplications,
dbSlaves: config.dbSlaves,
fulltextSearch: config.fulltextSearch,
@ -496,6 +508,10 @@ export function loadConfig(): Config {
}
function tryCreateUrl(url: string) {
if (!url) {
throw new Error('Failed to load: no "url" property found in config. Please check the value of "MISSKEY_CONFIG_DIR" and "MISSKEY_CONFIG_YML", and verify that all configuration files are correct.');
}
try {
return new URL(url);
} catch (e) {
@ -627,7 +643,7 @@ function applyEnvOverrides(config: Source) {
// these are all the settings that can be overridden
_apply_top([['url', 'port', 'address', 'socket', 'chmodSocket', 'disableHsts', 'id', 'dbReplications', 'websocketCompression']]);
_apply_top(['db', ['host', 'port', 'db', 'user', 'pass', 'disableCache']]);
_apply_top(['db', ['host', 'port', 'db', 'user', 'pass', 'slowQueryThreshold', 'disableCache']]);
_apply_top(['dbSlaves', Array.from((config.dbSlaves ?? []).keys()), ['host', 'port', 'db', 'user', 'pass']]);
_apply_top([
['redis', 'redisForPubsub', 'redisForJobQueue', 'redisForTimelines', 'redisForReactions', 'redisForRateLimit'],

32
packages/backend/src/core/AbuseReportNotificationService.ts
View file

@ -82,6 +82,28 @@ export class AbuseReportNotificationService implements OnApplicationShutdown {
}
}
/**
* Collects all email addresses that a abuse report should be sent to.
*/
@bindThis
public async getRecipientEMailAddresses(): Promise<string[]> {
const recipientEMailAddresses = await this.fetchEMailRecipients().then(it => it
.filter(it => it.isActive && it.userProfile?.emailVerified)
.map(it => it.userProfile?.email)
.filter(x => x != null),
);
if (this.meta.email) {
recipientEMailAddresses.push(this.meta.email);
}
if (this.meta.maintainerEmail) {
recipientEMailAddresses.push(this.meta.maintainerEmail);
}
return recipientEMailAddresses;
}
/**
* Mailを用いて{@link abuseReports}の内容を管理者各位に通知する.
* メールアドレスの送信先は以下の通り.
@ -96,15 +118,7 @@ export class AbuseReportNotificationService implements OnApplicationShutdown {
return;
}
const recipientEMailAddresses = await this.fetchEMailRecipients().then(it => it
.filter(it => it.isActive && it.userProfile?.emailVerified)
.map(it => it.userProfile?.email)
.filter(x => x != null),
);
recipientEMailAddresses.push(
...(this.meta.email ? [this.meta.email] : []),
);
const recipientEMailAddresses = await this.getRecipientEMailAddresses();
if (recipientEMailAddresses.length <= 0) {
return;

5
packages/backend/src/core/AbuseReportService.ts
View file

@ -13,6 +13,7 @@ import { QueueService } from '@/core/QueueService.js';
import { ApRendererService } from '@/core/activitypub/ApRendererService.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { IdService } from './IdService.js';
@Injectable()
@ -125,11 +126,11 @@ export class AbuseReportService {
const report = await this.abuseUserReportsRepository.findOneByOrFail({ id: reportId });
if (report.targetUserHost == null) {
throw new Error('The target user host is null.');
throw new IdentifiableError('0b1ce202-b2c1-4ee4-8af4-2742a51b383d', 'The target user host is null.');
}
if (report.forwarded) {
throw new Error('The report has already been forwarded.');
throw new IdentifiableError('5c008bdf-f0e8-4154-9f34-804e114516d7', 'The report has already been forwarded.');
}
await this.abuseUserReportsRepository.update(report.id, {

24
packages/backend/src/core/AccountMoveService.ts
View file

@ -26,6 +26,7 @@ import PerUserFollowingChart from '@/core/chart/charts/per-user-following.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { RoleService } from '@/core/RoleService.js';
import { AntennaService } from '@/core/AntennaService.js';
import { CacheService } from '@/core/CacheService.js';
@Injectable()
export class AccountMoveService {
@ -68,6 +69,7 @@ export class AccountMoveService {
private systemAccountService: SystemAccountService,
private roleService: RoleService,
private antennaService: AntennaService,
private readonly cacheService: CacheService,
) {
}
@ -107,12 +109,10 @@ export class AccountMoveService {
this.globalEventService.publishMainStream(src.id, 'meUpdated', iObj);
// Unfollow after 24 hours
const followings = await this.followingsRepository.findBy({
followerId: src.id,
});
this.queueService.createDelayedUnfollowJob(followings.map(following => ({
const followings = await this.cacheService.userFollowingsCache.fetch(src.id);
this.queueService.createDelayedUnfollowJob(Array.from(followings.keys()).map(followeeId => ({
from: { id: src.id },
to: { id: following.followeeId },
to: { id: followeeId },
})), process.env.NODE_ENV === 'test' ? 10000 : 1000 * 60 * 60 * 24);
await this.postMoveProcess(src, dst);
@ -138,11 +138,9 @@ export class AccountMoveService {
// follow the new account
const proxy = await this.systemAccountService.fetch('proxy');
const followings = await this.followingsRepository.findBy({
followeeId: src.id,
followerHost: IsNull(), // follower is local
followerId: Not(proxy.id),
});
const followings = await this.cacheService.userFollowersCache.fetch(src.id)
.then(fs => Array.from(fs.values())
.filter(f => f.followerHost == null && f.followerId !== proxy.id));
const followJobs = followings.map(following => ({
from: { id: following.followerId },
to: { id: dst.id },
@ -318,9 +316,9 @@ export class AccountMoveService {
await this.usersRepository.decrement({ id: In(localFollowerIds) }, 'followingCount', 1);
// Decrease follower counts of local followees by 1.
const oldFollowings = await this.followingsRepository.findBy({ followerId: oldAccount.id });
if (oldFollowings.length > 0) {
await this.usersRepository.decrement({ id: In(oldFollowings.map(following => following.followeeId)) }, 'followersCount', 1);
const oldFollowings = await this.cacheService.userFollowingsCache.fetch(oldAccount.id);
if (oldFollowings.size > 0) {
await this.usersRepository.decrement({ id: In(Array.from(oldFollowings.keys())) }, 'followersCount', 1);
}
// Update instance stats by decreasing remote followers count by the number of local followers who were following the old account.

3
packages/backend/src/core/AntennaService.ts
View file

@ -130,7 +130,8 @@ export class AntennaService implements OnApplicationShutdown {
}
if (note.visibility === 'followers') {
const isFollowing = Object.hasOwn(await this.cacheService.userFollowingsCache.fetch(antenna.userId), note.userId);
const followings = await this.cacheService.userFollowingsCache.fetch(antenna.userId);
const isFollowing = followings.has(note.userId);
if (!isFollowing && antenna.userId !== note.userId) return false;
}

6
packages/backend/src/core/BunnyService.ts
View file

@ -80,15 +80,15 @@ export class BunnyService {
});
req.on('error', (error) => {
this.bunnyCdnLogger.error(error);
this.bunnyCdnLogger.error('Unhandled error', error);
data.destroy();
throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140bf91', 'An error has occured during the connectiong to BunnyCDN');
throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140bf91', 'An error has occurred while connecting to BunnyCDN', true, error);
});
data.pipe(req).on('finish', () => {
data.destroy();
});
// wait till stream gets destroyed upon finish of piping to prevent the UI from showing the upload as success way too early
await finished(data);
}

354
packages/backend/src/core/CacheService.ts
View file

@ -5,14 +5,16 @@
import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import { IsNull } from 'typeorm';
import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiFollowing, MiNote } from '@/models/_.js';
import { In, IsNull } from 'typeorm';
import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiNote, MiFollowing } from '@/models/_.js';
import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js';
import { QuantumKVCache } from '@/misc/QuantumKVCache.js';
import type { MiLocalUser, MiUser } from '@/models/User.js';
import { DI } from '@/di-symbols.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { bindThis } from '@/decorators.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import type { InternalEventTypes } from '@/core/GlobalEventService.js';
import { InternalEventService } from '@/core/InternalEventService.js';
import type { OnApplicationShutdown } from '@nestjs/common';
export interface FollowStats {
@ -27,7 +29,7 @@ export interface CachedTranslation {
text: string | undefined;
}
interface CachedTranslationEntity {
export interface CachedTranslationEntity {
l?: string;
t?: string;
u?: number;
@ -39,14 +41,16 @@ export class CacheService implements OnApplicationShutdown {
public localUserByNativeTokenCache: MemoryKVCache<MiLocalUser | null>;
public localUserByIdCache: MemoryKVCache<MiLocalUser>;
public uriPersonCache: MemoryKVCache<MiUser | null>;
public userProfileCache: RedisKVCache<MiUserProfile>;
public userMutingsCache: RedisKVCache<Set<string>>;
public userBlockingCache: RedisKVCache<Set<string>>;
public userBlockedCache: RedisKVCache<Set<string>>; // NOTE: 「被」Blockキャッシュ
public renoteMutingsCache: RedisKVCache<Set<string>>;
public userFollowingsCache: RedisKVCache<Record<string, Pick<MiFollowing, 'withReplies'> | undefined>>;
private readonly userFollowStatsCache = new MemoryKVCache<FollowStats>(1000 * 60 * 10); // 10 minutes
private readonly translationsCache: RedisKVCache<CachedTranslationEntity>;
public userProfileCache: QuantumKVCache<MiUserProfile>;
public userMutingsCache: QuantumKVCache<Set<string>>;
public userBlockingCache: QuantumKVCache<Set<string>>;
public userBlockedCache: QuantumKVCache<Set<string>>; // NOTE: 「被」Blockキャッシュ
public renoteMutingsCache: QuantumKVCache<Set<string>>;
public userFollowingsCache: QuantumKVCache<Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>;
public userFollowersCache: QuantumKVCache<Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>;
public hibernatedUserCache: QuantumKVCache<boolean>;
protected userFollowStatsCache = new MemoryKVCache<FollowStats>(1000 * 60 * 10); // 10 minutes
protected translationsCache: RedisKVCache<CachedTranslationEntity>;
constructor(
@Inject(DI.redis)
@ -74,6 +78,7 @@ export class CacheService implements OnApplicationShutdown {
private followingsRepository: FollowingsRepository,
private userEntityService: UserEntityService,
private readonly internalEventService: InternalEventService,
) {
//this.onMessage = this.onMessage.bind(this);
@ -82,58 +87,148 @@ export class CacheService implements OnApplicationShutdown {
this.localUserByIdCache = new MemoryKVCache<MiLocalUser>(1000 * 60 * 5); // 5m
this.uriPersonCache = new MemoryKVCache<MiUser | null>(1000 * 60 * 5); // 5m
this.userProfileCache = new RedisKVCache<MiUserProfile>(this.redisClient, 'userProfile', {
this.userProfileCache = new QuantumKVCache(this.internalEventService, 'userProfile', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.userProfilesRepository.findOneByOrFail({ userId: key }),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => JSON.parse(value), // TODO: date型の考慮
bulkFetcher: userIds => this.userProfilesRepository.findBy({ userId: In(userIds) }).then(ps => ps.map(p => [p.userId, p])),
});
this.userMutingsCache = new RedisKVCache<Set<string>>(this.redisClient, 'userMutings', {
this.userMutingsCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'userMutings', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.mutingsRepository.find({ where: { muterId: key }, select: ['muteeId'] }).then(xs => new Set(xs.map(x => x.muteeId))),
toRedisConverter: (value) => JSON.stringify(Array.from(value)),
fromRedisConverter: (value) => new Set(JSON.parse(value)),
bulkFetcher: muterIds => this.mutingsRepository
.createQueryBuilder('muting')
.select('"muting"."muterId"', 'muterId')
.addSelect('array_agg("muting"."muteeId")', 'muteeIds')
.where({ muterId: In(muterIds) })
.groupBy('muting.muterId')
.getRawMany<{ muterId: string, muteeIds: string[] }>()
.then(ms => ms.map(m => [m.muterId, new Set(m.muteeIds)])),
});
this.userBlockingCache = new RedisKVCache<Set<string>>(this.redisClient, 'userBlocking', {
this.userBlockingCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'userBlocking', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.blockingsRepository.find({ where: { blockerId: key }, select: ['blockeeId'] }).then(xs => new Set(xs.map(x => x.blockeeId))),
toRedisConverter: (value) => JSON.stringify(Array.from(value)),
fromRedisConverter: (value) => new Set(JSON.parse(value)),
bulkFetcher: blockerIds => this.blockingsRepository
.createQueryBuilder('blocking')
.select('"blocking"."blockerId"', 'blockerId')
.addSelect('array_agg("blocking"."blockeeId")', 'blockeeIds')
.where({ blockerId: In(blockerIds) })
.groupBy('blocking.blockerId')
.getRawMany<{ blockerId: string, blockeeIds: string[] }>()
.then(ms => ms.map(m => [m.blockerId, new Set(m.blockeeIds)])),
});
this.userBlockedCache = new RedisKVCache<Set<string>>(this.redisClient, 'userBlocked', {
this.userBlockedCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'userBlocked', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.blockingsRepository.find({ where: { blockeeId: key }, select: ['blockerId'] }).then(xs => new Set(xs.map(x => x.blockerId))),
toRedisConverter: (value) => JSON.stringify(Array.from(value)),
fromRedisConverter: (value) => new Set(JSON.parse(value)),
bulkFetcher: blockeeIds => this.blockingsRepository
.createQueryBuilder('blocking')
.select('"blocking"."blockeeId"', 'blockeeId')
.addSelect('array_agg("blocking"."blockeeId")', 'blockeeIds')
.where({ blockeeId: In(blockeeIds) })
.groupBy('blocking.blockeeId')
.getRawMany<{ blockeeId: string, blockerIds: string[] }>()
.then(ms => ms.map(m => [m.blockeeId, new Set(m.blockerIds)])),
});
this.renoteMutingsCache = new RedisKVCache<Set<string>>(this.redisClient, 'renoteMutings', {
this.renoteMutingsCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'renoteMutings', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.renoteMutingsRepository.find({ where: { muterId: key }, select: ['muteeId'] }).then(xs => new Set(xs.map(x => x.muteeId))),
toRedisConverter: (value) => JSON.stringify(Array.from(value)),
fromRedisConverter: (value) => new Set(JSON.parse(value)),
bulkFetcher: muterIds => this.renoteMutingsRepository
.createQueryBuilder('muting')
.select('"muting"."muterId"', 'muterId')
.addSelect('array_agg("muting"."muteeId")', 'muteeIds')
.where({ muterId: In(muterIds) })
.groupBy('muting.muterId')
.getRawMany<{ muterId: string, muteeIds: string[] }>()
.then(ms => ms.map(m => [m.muterId, new Set(m.muteeIds)])),
});
this.userFollowingsCache = new RedisKVCache<Record<string, Pick<MiFollowing, 'withReplies'> | undefined>>(this.redisClient, 'userFollowings', {
this.userFollowingsCache = new QuantumKVCache<Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>(this.internalEventService, 'userFollowings', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.followingsRepository.find({ where: { followerId: key }, select: ['followeeId', 'withReplies'] }).then(xs => {
const obj: Record<string, Pick<MiFollowing, 'withReplies'> | undefined> = {};
for (const x of xs) {
obj[x.followeeId] = { withReplies: x.withReplies };
fetcher: (key) => this.followingsRepository.findBy({ followerId: key }).then(xs => new Map(xs.map(f => [f.followeeId, f]))),
bulkFetcher: followerIds => this.followingsRepository
.findBy({ followerId: In(followerIds) })
.then(fs => fs
.reduce((groups, f) => {
let group = groups.get(f.followerId);
if (!group) {
group = new Map();
groups.set(f.followerId, group);
}
group.set(f.followeeId, f);
return groups;
}, new Map<string, Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>)),
});
this.userFollowersCache = new QuantumKVCache<Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>(this.internalEventService, 'userFollowers', {
lifetime: 1000 * 60 * 30, // 30m
fetcher: followeeId => this.followingsRepository.findBy({ followeeId: followeeId }).then(xs => new Map(xs.map(x => [x.followerId, x]))),
bulkFetcher: followeeIds => this.followingsRepository
.findBy({ followeeId: In(followeeIds) })
.then(fs => fs
.reduce((groups, f) => {
let group = groups.get(f.followeeId);
if (!group) {
group = new Map();
groups.set(f.followeeId, group);
}
group.set(f.followerId, f);
return groups;
}, new Map<string, Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>)),
});
this.hibernatedUserCache = new QuantumKVCache<boolean>(this.internalEventService, 'hibernatedUsers', {
lifetime: 1000 * 60 * 30, // 30m
fetcher: async userId => {
const { isHibernated } = await this.usersRepository.findOneOrFail({
where: { id: userId },
select: { isHibernated: true },
});
return isHibernated;
},
bulkFetcher: async userIds => {
const results = await this.usersRepository.find({
where: { id: In(userIds) },
select: { id: true, isHibernated: true },
});
return results.map(({ id, isHibernated }) => [id, isHibernated]);
},
onChanged: async userIds => {
// We only update local copies since each process will get this event, but we can have user objects in multiple different caches.
// Before doing anything else we must "find" all the objects to update.
const userObjects = new Map<string, MiUser[]>();
const toUpdate: string[] = [];
for (const uid of userIds) {
const toAdd: MiUser[] = [];
const localUserById = this.localUserByIdCache.get(uid);
if (localUserById) toAdd.push(localUserById);
const userById = this.userByIdCache.get(uid);
if (userById) toAdd.push(userById);
if (toAdd.length > 0) {
toUpdate.push(uid);
userObjects.set(uid, toAdd);
}
}
return obj;
}),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => JSON.parse(value),
// In many cases, we won't have to do anything.
// Skipping the DB fetch ensures that this remains a single-step synchronous process.
if (toUpdate.length > 0) {
const hibernations = await this.usersRepository.find({ where: { id: In(toUpdate) }, select: { id: true, isHibernated: true } });
for (const { id, isHibernated } of hibernations) {
const users = userObjects.get(id);
if (users) {
for (const u of users) {
u.isHibernated = isHibernated;
}
}
}
}
},
});
this.translationsCache = new RedisKVCache<CachedTranslationEntity>(this.redisClient, 'translations', {
@ -143,20 +238,21 @@ export class CacheService implements OnApplicationShutdown {
// NOTE: チャンネルのフォロー状況キャッシュはChannelFollowingServiceで行っている
this.redisForSub.on('message', this.onMessage);
this.internalEventService.on('userChangeSuspendedState', this.onUserEvent);
this.internalEventService.on('userChangeDeletedState', this.onUserEvent);
this.internalEventService.on('remoteUserUpdated', this.onUserEvent);
this.internalEventService.on('localUserUpdated', this.onUserEvent);
this.internalEventService.on('userChangeSuspendedState', this.onUserEvent);
this.internalEventService.on('userTokenRegenerated', this.onTokenEvent);
this.internalEventService.on('follow', this.onFollowEvent);
this.internalEventService.on('unfollow', this.onFollowEvent);
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
switch (type) {
case 'userChangeSuspendedState':
case 'userChangeDeletedState':
case 'remoteUserUpdated':
case 'localUserUpdated': {
private async onUserEvent<E extends 'userChangeSuspendedState' | 'userChangeDeletedState' | 'remoteUserUpdated' | 'localUserUpdated'>(body: InternalEventTypes[E], _: E, isLocal: boolean): Promise<void> {
{
{
{
const user = await this.usersRepository.findOneBy({ id: body.id });
if (user == null) {
this.userByIdCache.delete(body.id);
@ -166,6 +262,18 @@ export class CacheService implements OnApplicationShutdown {
this.uriPersonCache.delete(k);
}
}
if (isLocal) {
await Promise.all([
this.userProfileCache.delete(body.id),
this.userMutingsCache.delete(body.id),
this.userBlockingCache.delete(body.id),
this.userBlockedCache.delete(body.id),
this.renoteMutingsCache.delete(body.id),
this.userFollowingsCache.delete(body.id),
this.userFollowersCache.delete(body.id),
this.hibernatedUserCache.delete(body.id),
]);
}
} else {
this.userByIdCache.set(user.id, user);
for (const [k, v] of this.uriPersonCache.entries) {
@ -178,20 +286,37 @@ export class CacheService implements OnApplicationShutdown {
this.localUserByIdCache.set(user.id, user);
}
}
break;
}
case 'userTokenRegenerated': {
}
}
}
@bindThis
private async onTokenEvent<E extends 'userTokenRegenerated'>(body: InternalEventTypes[E]): Promise<void> {
{
{
{
const user = await this.usersRepository.findOneByOrFail({ id: body.id }) as MiLocalUser;
this.localUserByNativeTokenCache.delete(body.oldToken);
this.localUserByNativeTokenCache.set(body.newToken, user);
break;
}
}
}
}
@bindThis
private async onFollowEvent<E extends 'follow' | 'unfollow'>(body: InternalEventTypes[E], type: E): Promise<void> {
{
switch (type) {
case 'follow': {
const follower = this.userByIdCache.get(body.followerId);
if (follower) follower.followingCount++;
const followee = this.userByIdCache.get(body.followeeId);
if (followee) followee.followersCount++;
this.userFollowingsCache.delete(body.followerId);
await Promise.all([
this.userFollowingsCache.delete(body.followerId),
this.userFollowersCache.delete(body.followeeId),
]);
this.userFollowStatsCache.delete(body.followerId);
this.userFollowStatsCache.delete(body.followeeId);
break;
@ -201,13 +326,14 @@ export class CacheService implements OnApplicationShutdown {
if (follower) follower.followingCount--;
const followee = this.userByIdCache.get(body.followeeId);
if (followee) followee.followersCount--;
this.userFollowingsCache.delete(body.followerId);
await Promise.all([
this.userFollowingsCache.delete(body.followerId),
this.userFollowersCache.delete(body.followeeId),
]);
this.userFollowStatsCache.delete(body.followerId);
this.userFollowStatsCache.delete(body.followeeId);
break;
}
default:
break;
}
}
}
@ -298,9 +424,115 @@ export class CacheService implements OnApplicationShutdown {
});
}
@bindThis
public async getUsers(userIds: Iterable<string>): Promise<Map<string, MiUser>> {
const users = new Map<string, MiUser>;
const toFetch: string[] = [];
for (const userId of userIds) {
const fromCache = this.userByIdCache.get(userId);
if (fromCache) {
users.set(userId, fromCache);
} else {
toFetch.push(userId);
}
}
if (toFetch.length > 0) {
const fetched = await this.usersRepository.findBy({
id: In(toFetch),
});
for (const user of fetched) {
users.set(user.id, user);
this.userByIdCache.set(user.id, user);
}
}
return users;
}
@bindThis
public async isFollowing(follower: string | { id: string }, followee: string | { id: string }): Promise<boolean> {
const followerId = typeof(follower) === 'string' ? follower : follower.id;
const followeeId = typeof(followee) === 'string' ? followee : followee.id;
// This lets us use whichever one is in memory, falling back to DB fetch via userFollowingsCache.
return this.userFollowersCache.get(followeeId)?.has(followerId)
?? (await this.userFollowingsCache.fetch(followerId)).has(followeeId);
}
/**
* Returns all hibernated followers.
*/
@bindThis
public async getHibernatedFollowers(followeeId: string): Promise<MiFollowing[]> {
const followers = await this.getFollowersWithHibernation(followeeId);
return followers.filter(f => f.isFollowerHibernated);
}
/**
* Returns all non-hibernated followers.
*/
@bindThis
public async getNonHibernatedFollowers(followeeId: string): Promise<MiFollowing[]> {
const followers = await this.getFollowersWithHibernation(followeeId);
return followers.filter(f => !f.isFollowerHibernated);
}
/**
* Returns follower relations with populated isFollowerHibernated.
* If you don't need this field, then please use userFollowersCache directly for reduced overhead.
*/
@bindThis
public async getFollowersWithHibernation(followeeId: string): Promise<MiFollowing[]> {
const followers = await this.userFollowersCache.fetch(followeeId);
const hibernations = await this.hibernatedUserCache.fetchMany(followers.keys()).then(fs => fs.reduce((map, f) => {
map.set(f[0], f[1]);
return map;
}, new Map<string, boolean>));
return Array.from(followers.values()).map(following => ({
...following,
isFollowerHibernated: hibernations.get(following.followerId) ?? false,
}));
}
/**
* Refreshes follower and following relations for the given user.
*/
@bindThis
public async refreshFollowRelationsFor(userId: string): Promise<void> {
const followings = await this.userFollowingsCache.refresh(userId);
const followees = Array.from(followings.values()).map(f => f.followeeId);
await this.userFollowersCache.deleteMany(followees);
}
@bindThis
public clear(): void {
this.userByIdCache.clear();
this.localUserByNativeTokenCache.clear();
this.localUserByIdCache.clear();
this.uriPersonCache.clear();
this.userProfileCache.clear();
this.userMutingsCache.clear();
this.userBlockingCache.clear();
this.userBlockedCache.clear();
this.renoteMutingsCache.clear();
this.userFollowingsCache.clear();
this.userFollowStatsCache.clear();
this.translationsCache.clear();
}
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.internalEventService.off('userChangeSuspendedState', this.onUserEvent);
this.internalEventService.off('userChangeDeletedState', this.onUserEvent);
this.internalEventService.off('remoteUserUpdated', this.onUserEvent);
this.internalEventService.off('localUserUpdated', this.onUserEvent);
this.internalEventService.off('userChangeSuspendedState', this.onUserEvent);
this.internalEventService.off('userTokenRegenerated', this.onTokenEvent);
this.internalEventService.off('follow', this.onFollowEvent);
this.internalEventService.off('unfollow', this.onFollowEvent);
this.userByIdCache.dispose();
this.localUserByNativeTokenCache.dispose();
this.localUserByIdCache.dispose();

10
packages/backend/src/core/CaptchaService.ts
View file

@ -54,7 +54,7 @@ export class CaptchaError extends Error {
public readonly cause?: unknown;
constructor(code: CaptchaErrorCode, message: string, cause?: unknown) {
super(message);
super(message, cause ? { cause } : undefined);
this.code = code;
this.cause = cause;
this.name = 'CaptchaError';
@ -117,7 +117,7 @@ export class CaptchaService {
}
const result = await this.getCaptchaResponse('https://www.recaptcha.net/recaptcha/api/siteverify', secret, response).catch(err => {
throw new CaptchaError(captchaErrorCodes.requestFailed, `recaptcha-request-failed: ${err}`);
throw new CaptchaError(captchaErrorCodes.requestFailed, `recaptcha-request-failed: ${err}`, err);
});
if (result.success !== true) {
@ -133,7 +133,7 @@ export class CaptchaService {
}
const result = await this.getCaptchaResponse('https://hcaptcha.com/siteverify', secret, response).catch(err => {
throw new CaptchaError(captchaErrorCodes.requestFailed, `hcaptcha-request-failed: ${err}`);
throw new CaptchaError(captchaErrorCodes.requestFailed, `hcaptcha-request-failed: ${err}`, err);
});
if (result.success !== true) {
@ -209,7 +209,7 @@ export class CaptchaService {
}
const result = await this.getCaptchaResponse('https://challenges.cloudflare.com/turnstile/v0/siteverify', secret, response).catch(err => {
throw new CaptchaError(captchaErrorCodes.requestFailed, `turnstile-request-failed: ${err}`);
throw new CaptchaError(captchaErrorCodes.requestFailed, `turnstile-request-failed: ${err}`, err);
});
if (result.success !== true) {
@ -386,7 +386,7 @@ export class CaptchaService {
this.logger.info(err);
const error = err instanceof CaptchaError
? err
: new CaptchaError(captchaErrorCodes.unknown, `unknown error: ${err}`);
: new CaptchaError(captchaErrorCodes.unknown, `unknown error: ${err}`, err);
return {
success: false,
error,

29
packages/backend/src/core/ChannelFollowingService.ts
View file

@ -9,14 +9,15 @@ import { DI } from '@/di-symbols.js';
import type { ChannelFollowingsRepository } from '@/models/_.js';
import { MiChannel } from '@/models/_.js';
import { IdService } from '@/core/IdService.js';
import { GlobalEvents, GlobalEventService } from '@/core/GlobalEventService.js';
import { GlobalEvents, GlobalEventService, InternalEventTypes } from '@/core/GlobalEventService.js';
import { bindThis } from '@/decorators.js';
import type { MiLocalUser } from '@/models/User.js';
import { RedisKVCache } from '@/misc/cache.js';
import { QuantumKVCache } from '@/misc/QuantumKVCache.js';
import { InternalEventService } from './InternalEventService.js';
@Injectable()
export class ChannelFollowingService implements OnModuleInit {
public userFollowingChannelsCache: RedisKVCache<Set<string>>;
public userFollowingChannelsCache: QuantumKVCache<Set<string>>;
constructor(
@Inject(DI.redis)
@ -27,19 +28,18 @@ export class ChannelFollowingService implements OnModuleInit {
private channelFollowingsRepository: ChannelFollowingsRepository,
private idService: IdService,
private globalEventService: GlobalEventService,
private readonly internalEventService: InternalEventService,
) {
this.userFollowingChannelsCache = new RedisKVCache<Set<string>>(this.redisClient, 'userFollowingChannels', {
this.userFollowingChannelsCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'userFollowingChannels', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.channelFollowingsRepository.find({
where: { followerId: key },
select: ['followeeId'],
}).then(xs => new Set(xs.map(x => x.followeeId))),
toRedisConverter: (value) => JSON.stringify(Array.from(value)),
fromRedisConverter: (value) => new Set(JSON.parse(value)),
});
this.redisForSub.on('message', this.onMessage);
this.internalEventService.on('followChannel', this.onMessage);
this.internalEventService.on('unfollowChannel', this.onMessage);
}
onModuleInit() {
@ -79,18 +79,15 @@ export class ChannelFollowingService implements OnModuleInit {
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
private async onMessage<E extends 'followChannel' | 'unfollowChannel'>(body: InternalEventTypes[E], type: E): Promise<void> {
{
switch (type) {
case 'followChannel': {
this.userFollowingChannelsCache.refresh(body.userId);
await this.userFollowingChannelsCache.delete(body.userId);
break;
}
case 'unfollowChannel': {
this.userFollowingChannelsCache.delete(body.userId);
await this.userFollowingChannelsCache.delete(body.userId);
break;
}
}
@ -99,6 +96,8 @@ export class ChannelFollowingService implements OnModuleInit {
@bindThis
public dispose(): void {
this.internalEventService.off('followChannel', this.onMessage);
this.internalEventService.off('unfollowChannel', this.onMessage);
this.userFollowingChannelsCache.dispose();
}

6
packages/backend/src/core/CoreModule.ts
View file

@ -41,6 +41,7 @@ import { HttpRequestService } from './HttpRequestService.js';
import { IdService } from './IdService.js';
import { ImageProcessingService } from './ImageProcessingService.js';
import { SystemAccountService } from './SystemAccountService.js';
import { InternalEventService } from './InternalEventService.js';
import { InternalStorageService } from './InternalStorageService.js';
import { MetaService } from './MetaService.js';
import { MfmService } from './MfmService.js';
@ -186,6 +187,7 @@ const $HashtagService: Provider = { provide: 'HashtagService', useExisting: Hash
const $HttpRequestService: Provider = { provide: 'HttpRequestService', useExisting: HttpRequestService };
const $IdService: Provider = { provide: 'IdService', useExisting: IdService };
const $ImageProcessingService: Provider = { provide: 'ImageProcessingService', useExisting: ImageProcessingService };
const $InternalEventService: Provider = { provide: 'InternalEventService', useExisting: InternalEventService };
const $InternalStorageService: Provider = { provide: 'InternalStorageService', useExisting: InternalStorageService };
const $MetaService: Provider = { provide: 'MetaService', useExisting: MetaService };
const $MfmService: Provider = { provide: 'MfmService', useExisting: MfmService };
@ -345,6 +347,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
HttpRequestService,
IdService,
ImageProcessingService,
InternalEventService,
InternalStorageService,
MetaService,
MfmService,
@ -500,6 +503,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$HttpRequestService,
$IdService,
$ImageProcessingService,
$InternalEventService,
$InternalStorageService,
$MetaService,
$MfmService,
@ -656,6 +660,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
HttpRequestService,
IdService,
ImageProcessingService,
InternalEventService,
InternalStorageService,
MetaService,
MfmService,
@ -810,6 +815,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
$HttpRequestService,
$IdService,
$ImageProcessingService,
$InternalEventService,
$InternalStorageService,
$MetaService,
$MfmService,

17
packages/backend/src/core/DownloadService.ts
View file

@ -18,6 +18,7 @@ import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
import { bindThis } from '@/decorators.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
@Injectable()
export class DownloadService {
@ -37,7 +38,7 @@ export class DownloadService {
public async downloadUrl(url: string, path: string, options: { timeout?: number, operationTimeout?: number, maxSize?: number } = {} ): Promise<{
filename: string;
}> {
this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
this.logger.debug(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
const timeout = options.timeout ?? 30 * 1000;
const operationTimeout = options.operationTimeout ?? 60 * 1000;
@ -86,7 +87,7 @@ export class DownloadService {
filename = parsed.parameters.filename;
}
} catch (e) {
this.logger.warn(`Failed to parse content-disposition: ${contentDisposition}`, { stack: e });
this.logger.warn(`Failed to parse content-disposition ${contentDisposition}: ${renderInlineError(e)}`);
}
}
}).on('downloadProgress', (progress: Got.Progress) => {
@ -100,13 +101,17 @@ export class DownloadService {
await stream.pipeline(req, fs.createWriteStream(path));
} catch (e) {
if (e instanceof Got.HTTPError) {
throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
} else {
throw new StatusError(`download error from ${url}`, e.response.statusCode, e.response.statusMessage, e);
} else if (e instanceof Got.RequestError || e instanceof Got.AbortError) {
throw new Error(String(e), { cause: e });
} else if (e instanceof Error) {
throw e;
} else {
throw new Error(String(e), { cause: e });
}
}
this.logger.succ(`Download finished: ${chalk.cyan(url)}`);
this.logger.info(`Download finished: ${chalk.cyan(url)}`);
return {
filename,
@ -118,7 +123,7 @@ export class DownloadService {
// Create temp file
const [path, cleanup] = await createTemp();
this.logger.info(`text file: Temp file is ${path}`);
this.logger.debug(`text file: Temp file is ${path}`);
try {
// write content at URL to temp file

91
packages/backend/src/core/DriveService.ts
View file

@ -45,6 +45,7 @@ import { isMimeImage } from '@/misc/is-mime-image.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { BunnyService } from '@/core/BunnyService.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import { LoggerService } from './LoggerService.js';
type AddFileArgs = {
@ -159,6 +160,14 @@ export class DriveService {
// thunbnail, webpublic を必要なら生成
const alts = await this.generateAlts(path, type, !file.uri);
if (type && type.startsWith('video/')) {
try {
await this.videoProcessingService.webOptimizeVideo(path, type);
} catch (err) {
this.registerLogger.warn(`Video optimization failed: ${renderInlineError(err)}`);
}
}
if (this.meta.useObjectStorage) {
//#region ObjectStorage params
let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) ?? ['']);
@ -194,7 +203,7 @@ export class DriveService {
//#endregion
//#region Uploads
this.registerLogger.info(`uploading original: ${key}`);
this.registerLogger.debug(`uploading original: ${key}`);
const uploads = [
this.upload(key, fs.createReadStream(path), type, null, name),
];
@ -203,7 +212,7 @@ export class DriveService {
webpublicKey = `${prefix}webpublic-${randomUUID()}.${alts.webpublic.ext}`;
webpublicUrl = `${ baseUrl }/${ webpublicKey }`;
this.registerLogger.info(`uploading webpublic: ${webpublicKey}`);
this.registerLogger.debug(`uploading webpublic: ${webpublicKey}`);
uploads.push(this.upload(webpublicKey, alts.webpublic.data, alts.webpublic.type, alts.webpublic.ext, name));
}
@ -211,7 +220,7 @@ export class DriveService {
thumbnailKey = `${prefix}thumbnail-${randomUUID()}.${alts.thumbnail.ext}`;
thumbnailUrl = `${ baseUrl }/${ thumbnailKey }`;
this.registerLogger.info(`uploading thumbnail: ${thumbnailKey}`);
this.registerLogger.debug(`uploading thumbnail: ${thumbnailKey}`);
uploads.push(this.upload(thumbnailKey, alts.thumbnail.data, alts.thumbnail.type, alts.thumbnail.ext, `${name}.thumbnail`));
}
@ -255,11 +264,11 @@ export class DriveService {
const [url, thumbnailUrl, webpublicUrl] = await Promise.all(promises);
if (thumbnailUrl) {
this.registerLogger.info(`thumbnail stored: ${thumbnailAccessKey}`);
this.registerLogger.debug(`thumbnail stored: ${thumbnailAccessKey}`);
}
if (webpublicUrl) {
this.registerLogger.info(`web stored: ${webpublicAccessKey}`);
this.registerLogger.debug(`web stored: ${webpublicAccessKey}`);
}
file.storedInternal = true;
@ -303,7 +312,7 @@ export class DriveService {
thumbnail,
};
} catch (err) {
this.registerLogger.warn(`GenerateVideoThumbnail failed: ${err}`);
this.registerLogger.warn(`GenerateVideoThumbnail failed: ${renderInlineError(err)}`);
return {
webpublic: null,
thumbnail: null,
@ -336,7 +345,7 @@ export class DriveService {
metadata.height && metadata.height <= 2048
);
} catch (err) {
this.registerLogger.warn(`sharp failed: ${err}`);
this.registerLogger.warn(`sharp failed: ${renderInlineError(err)}`);
return {
webpublic: null,
thumbnail: null,
@ -347,7 +356,7 @@ export class DriveService {
let webpublic: IImage | null = null;
if (generateWeb && !satisfyWebpublic && !isAnimated) {
this.registerLogger.info('creating web image');
this.registerLogger.debug('creating web image');
try {
if (['image/jpeg', 'image/webp', 'image/avif'].includes(type)) {
@ -358,12 +367,12 @@ export class DriveService {
this.registerLogger.debug('web image not created (not an required image)');
}
} catch (err) {
this.registerLogger.warn('web image not created (an error occurred)', err as Error);
this.registerLogger.warn(`web image not created: ${renderInlineError(err)}`);
}
} else {
if (satisfyWebpublic) this.registerLogger.info('web image not created (original satisfies webpublic)');
else if (isAnimated) this.registerLogger.info('web image not created (animated image)');
else this.registerLogger.info('web image not created (from remote)');
if (satisfyWebpublic) this.registerLogger.debug('web image not created (original satisfies webpublic)');
else if (isAnimated) this.registerLogger.debug('web image not created (animated image)');
else this.registerLogger.debug('web image not created (from remote)');
}
// #endregion webpublic
@ -377,7 +386,7 @@ export class DriveService {
thumbnail = await this.imageProcessingService.convertSharpToWebp(img, 498, 422);
}
} catch (err) {
this.registerLogger.warn('thumbnail not created (an error occurred)', err as Error);
this.registerLogger.warn(`Error creating thumbnail: ${renderInlineError(err)}`);
}
// #endregion thumbnail
@ -411,27 +420,21 @@ export class DriveService {
);
if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read';
if (this.bunnyService.usingBunnyCDN(this.meta)) {
await this.bunnyService.upload(this.meta, key, stream).catch(
err => {
this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err);
},
);
} else {
await this.s3Service.upload(this.meta, params)
.then(
result => {
if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput
this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
} else { // AbortMultipartUploadCommandOutput
this.registerLogger.error(`Upload Result Aborted: key = ${key}, filename = ${filename}`);
}
})
.catch(
err => {
this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err);
},
);
try {
if (this.bunnyService.usingBunnyCDN(this.meta)) {
await this.bunnyService.upload(this.meta, key, stream);
} else {
const result = await this.s3Service.upload(this.meta, params);
if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput
this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
} else { // AbortMultipartUploadCommandOutput
this.registerLogger.error(`Upload Result Aborted: key = ${key}, filename = ${filename}`);
throw new Error('S3 upload aborted');
}
}
} catch (err) {
this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}: ${renderInlineError(err)}`);
throw err;
}
}
@ -490,7 +493,6 @@ export class DriveService {
}: AddFileArgs): Promise<MiDriveFile> {
const userRoleNSFW = user && (await this.roleService.getUserPolicies(user.id)).alwaysMarkNsfw;
const info = await this.fileInfoService.getFileInfo(path);
this.registerLogger.info(`${JSON.stringify(info)}`);
// detect name
const detectedName = correctFilename(
@ -500,6 +502,8 @@ export class DriveService {
ext ?? info.type.ext,
);
this.registerLogger.debug(`Detected file info: ${JSON.stringify(info)}`);
if (user && !force) {
// Check if there is a file with the same hash
const matched = await this.driveFilesRepository.findOneBy({
@ -508,7 +512,7 @@ export class DriveService {
});
if (matched) {
this.registerLogger.info(`file with same hash is found: ${matched.id}`);
this.registerLogger.debug(`file with same hash is found: ${matched.id}`);
if (sensitive && !matched.isSensitive) {
// The file is federated as sensitive for this time, but was federated as non-sensitive before.
// Therefore, update the file to sensitive.
@ -636,14 +640,14 @@ export class DriveService {
} catch (err) {
// duplicate key error (when already registered)
if (isDuplicateKeyValueError(err)) {
this.registerLogger.info(`already registered ${file.uri}`);
this.registerLogger.debug(`already registered ${file.uri}`);
file = await this.driveFilesRepository.findOneBy({
uri: file.uri!,
userId: user ? user.id : IsNull(),
}) as MiDriveFile;
} else {
this.registerLogger.error(err as Error);
this.registerLogger.error('Error in drive register', err as Error);
throw err;
}
}
@ -651,7 +655,7 @@ export class DriveService {
file = await (this.save(file, path, detectedName, info));
}
this.registerLogger.succ(`drive file has been created ${file.id}`);
this.registerLogger.info(`Created file ${file.id} (${detectedName}) of type ${info.type.mime} for user ${user?.id ?? '<none>'}`);
if (user) {
this.driveFileEntityService.pack(file, { self: true }).then(packedFile => {
@ -847,7 +851,7 @@ export class DriveService {
}
} catch (err: any) {
if (err.name === 'NoSuchKey') {
this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error);
this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`);
return;
} else {
throw new Error(`Failed to delete the file from the object storage with the given key: ${key}`, {
@ -884,13 +888,10 @@ export class DriveService {
}
const driveFile = await this.addFile({ user, path, name, comment, folderId, force, isLink, url, uri, sensitive, requestIp, requestHeaders });
this.downloaderLogger.succ(`Got: ${driveFile.id}`);
this.downloaderLogger.debug(`Upload succeeded: created file ${driveFile.id}`);
return driveFile!;
} catch (err) {
this.downloaderLogger.error(`Failed to create drive file: ${err}`, {
url: url,
e: err,
});
this.downloaderLogger.error(`Failed to create drive file from ${url}: ${renderInlineError(err)}`);
throw err;
} finally {
cleanup();

11
packages/backend/src/core/FanoutTimelineEndpointService.ts
View file

@ -136,10 +136,10 @@ export class FanoutTimelineEndpointService {
const parentFilter = filter;
filter = (note) => {
if (!ps.ignoreAuthorFromInstanceBlock) {
if (this.utilityService.isBlockedHost(this.meta.blockedHosts, note.userHost)) return false;
if (note.userInstance?.isBlocked) return false;
}
if (note.userId !== note.renoteUserId && this.utilityService.isBlockedHost(this.meta.blockedHosts, note.renoteUserHost)) return false;
if (note.userId !== note.replyUserId && this.utilityService.isBlockedHost(this.meta.blockedHosts, note.replyUserHost)) return false;
if (note.userId !== note.renoteUserId && note.renoteUserInstance?.isBlocked) return false;
if (note.userId !== note.replyUserId && note.replyUserInstance?.isBlocked) return false;
return parentFilter(note);
};
@ -194,7 +194,10 @@ export class FanoutTimelineEndpointService {
.leftJoinAndSelect('note.renote', 'renote')
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser')
.leftJoinAndSelect('note.channel', 'channel');
.leftJoinAndSelect('note.channel', 'channel')
.leftJoinAndSelect('note.userInstance', 'userInstance')
.leftJoinAndSelect('note.replyUserInstance', 'replyUserInstance')
.leftJoinAndSelect('note.renoteUserInstance', 'renoteUserInstance');
const notes = (await query.getMany()).filter(noteFilter);

103
packages/backend/src/core/FederatedInstanceService.ts
View file

@ -5,23 +5,24 @@
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import * as Redis from 'ioredis';
import { QueryFailedError } from 'typeorm';
import type { InstancesRepository } from '@/models/_.js';
import type { InstancesRepository, MiMeta } from '@/models/_.js';
import type { MiInstance } from '@/models/Instance.js';
import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js';
import { MemoryKVCache } from '@/misc/cache.js';
import { IdService } from '@/core/IdService.js';
import { DI } from '@/di-symbols.js';
import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { Serialized } from '@/types.js';
import { diffArrays, diffArraysSimple } from '@/misc/diff-arrays.js';
@Injectable()
export class FederatedInstanceService implements OnApplicationShutdown {
public federatedInstanceCache: RedisKVCache<MiInstance | null>;
private readonly federatedInstanceCache: MemoryKVCache<MiInstance | null>;
constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis,
@Inject(DI.redisForSub)
private redisForSub: Redis.Redis,
@Inject(DI.instancesRepository)
private instancesRepository: InstancesRepository,
@ -29,67 +30,46 @@ export class FederatedInstanceService implements OnApplicationShutdown {
private utilityService: UtilityService,
private idService: IdService,
) {
this.federatedInstanceCache = new RedisKVCache<MiInstance | null>(this.redisClient, 'federatedInstance', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60 * 3, // 3m
fetcher: (key) => this.instancesRepository.findOneBy({ host: key }),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => {
const parsed = JSON.parse(value);
if (parsed == null) return null;
return {
...parsed,
firstRetrievedAt: new Date(parsed.firstRetrievedAt),
latestRequestReceivedAt: parsed.latestRequestReceivedAt ? new Date(parsed.latestRequestReceivedAt) : null,
infoUpdatedAt: parsed.infoUpdatedAt ? new Date(parsed.infoUpdatedAt) : null,
notRespondingSince: parsed.notRespondingSince ? new Date(parsed.notRespondingSince) : null,
};
},
});
this.federatedInstanceCache = new MemoryKVCache(1000 * 60 * 3); // 3m
this.redisForSub.on('message', this.onMessage);
}
@bindThis
public async fetchOrRegister(host: string): Promise<MiInstance> {
host = this.utilityService.toPuny(host);
const cached = await this.federatedInstanceCache.get(host);
const cached = this.federatedInstanceCache.get(host);
if (cached) return cached;
const index = await this.instancesRepository.findOneBy({ host });
let index = await this.instancesRepository.findOneBy({ host });
if (index == null) {
let i;
try {
i = await this.instancesRepository.insertOne({
await this.instancesRepository.createQueryBuilder('instance')
.insert()
.values({
id: this.idService.gen(),
host,
firstRetrievedAt: new Date(),
});
} catch (e: unknown) {
if (e instanceof QueryFailedError) {
if (isDuplicateKeyValueError(e)) {
i = await this.instancesRepository.findOneBy({ host });
}
}
isBlocked: this.utilityService.isBlockedHost(host),
isSilenced: this.utilityService.isSilencedHost(host),
isMediaSilenced: this.utilityService.isMediaSilencedHost(host),
isAllowListed: this.utilityService.isAllowListedHost(host),
isBubbled: this.utilityService.isBubbledHost(host),
})
.orIgnore()
.execute();
if (i == null) {
throw e;
}
}
this.federatedInstanceCache.set(host, i);
return i;
} else {
this.federatedInstanceCache.set(host, index);
return index;
index = await this.instancesRepository.findOneByOrFail({ host });
}
this.federatedInstanceCache.set(host, index);
return index;
}
@bindThis
public async fetch(host: string): Promise<MiInstance | null> {
host = this.utilityService.toPuny(host);
const cached = await this.federatedInstanceCache.get(host);
const cached = this.federatedInstanceCache.get(host);
if (cached !== undefined) return cached;
const index = await this.instancesRepository.findOneBy({ host });
@ -117,8 +97,35 @@ export class FederatedInstanceService implements OnApplicationShutdown {
this.federatedInstanceCache.set(result.host, result);
}
private syncCache(before: Serialized<MiMeta | undefined>, after: Serialized<MiMeta>): void {
const changed =
diffArraysSimple(before?.blockedHosts, after.blockedHosts) ||
diffArraysSimple(before?.silencedHosts, after.silencedHosts) ||
diffArraysSimple(before?.mediaSilencedHosts, after.mediaSilencedHosts) ||
diffArraysSimple(before?.federationHosts, after.federationHosts) ||
diffArraysSimple(before?.bubbleInstances, after.bubbleInstances);
if (changed) {
// We have to clear the whole thing, otherwise subdomains won't be synced.
this.federatedInstanceCache.clear();
}
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
if (type === 'metaUpdated') {
this.syncCache(body.before, body.after);
}
}
}
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.federatedInstanceCache.dispose();
}

21
packages/backend/src/core/FetchInstanceMetadataService.ts
View file

@ -7,7 +7,7 @@ import { URL } from 'node:url';
import { Inject, Injectable } from '@nestjs/common';
import tinycolor from 'tinycolor2';
import * as Redis from 'ioredis';
import { load as cheerio } from 'cheerio';
import { load as cheerio } from 'cheerio/slim';
import type { MiInstance } from '@/models/Instance.js';
import type Logger from '@/logger.js';
import { DI } from '@/di-symbols.js';
@ -15,7 +15,8 @@ import { LoggerService } from '@/core/LoggerService.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import type { CheerioAPI } from 'cheerio';
import { renderInlineError } from '@/misc/render-inline-error.js';
import type { CheerioAPI } from 'cheerio/slim';
type NodeInfo = {
openRegistrations?: unknown;
@ -90,7 +91,7 @@ export class FetchInstanceMetadataService {
}
}
this.logger.info(`Fetching metadata of ${instance.host} ...`);
this.logger.debug(`Fetching metadata of ${instance.host} ...`);
const [info, dom, manifest] = await Promise.all([
this.fetchNodeinfo(instance).catch(() => null),
@ -106,7 +107,7 @@ export class FetchInstanceMetadataService {
this.getDescription(info, dom, manifest).catch(() => null),
]);
this.logger.succ(`Successfuly fetched metadata of ${instance.host}`);
this.logger.debug(`Successfuly fetched metadata of ${instance.host}`);
const updates = {
infoUpdatedAt: new Date(),
@ -128,9 +129,9 @@ export class FetchInstanceMetadataService {
await this.federatedInstanceService.update(instance.id, updates);
this.logger.succ(`Successfuly updated metadata of ${instance.host}`);
this.logger.info(`Successfully updated metadata of ${instance.host}`);
} catch (e) {
this.logger.error(`Failed to update metadata of ${instance.host}: ${e}`);
this.logger.error(`Failed to update metadata of ${instance.host}: ${renderInlineError(e)}`);
} finally {
await this.unlock(host);
}
@ -138,7 +139,7 @@ export class FetchInstanceMetadataService {
@bindThis
private async fetchNodeinfo(instance: MiInstance): Promise<NodeInfo> {
this.logger.info(`Fetching nodeinfo of ${instance.host} ...`);
this.logger.debug(`Fetching nodeinfo of ${instance.host} ...`);
try {
const wellknown = await this.httpRequestService.getJson('https://' + instance.host + '/.well-known/nodeinfo')
@ -170,11 +171,11 @@ export class FetchInstanceMetadataService {
throw err.statusCode ?? err.message;
});
this.logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
this.logger.debug(`Successfuly fetched nodeinfo of ${instance.host}`);
return info as NodeInfo;
} catch (err) {
this.logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${err}`);
this.logger.warn(`Failed to fetch nodeinfo of ${instance.host}: ${renderInlineError(err)}`);
throw err;
}
@ -182,7 +183,7 @@ export class FetchInstanceMetadataService {
@bindThis
private async fetchDom(instance: MiInstance): Promise<CheerioAPI> {
this.logger.info(`Fetching HTML of ${instance.host} ...`);
this.logger.debug(`Fetching HTML of ${instance.host} ...`);
const url = 'https://' + instance.host;

9
packages/backend/src/core/FileInfoService.ts
View file

@ -46,11 +46,13 @@ const TYPE_SVG = {
@Injectable()
export class FileInfoService {
private logger: Logger;
private ffprobeLogger: Logger;
constructor(
private loggerService: LoggerService,
) {
this.logger = this.loggerService.getLogger('file-info');
this.ffprobeLogger = this.logger.createSubLogger('ffprobe');
}
/**
@ -162,20 +164,19 @@ export class FileInfoService {
*/
@bindThis
private hasVideoTrackOnVideoFile(path: string): Promise<boolean> {
const sublogger = this.logger.createSubLogger('ffprobe');
sublogger.info(`Checking the video file. File path: ${path}`);
this.ffprobeLogger.debug(`Checking the video file. File path: ${path}`);
return new Promise((resolve) => {
try {
FFmpeg.ffprobe(path, (err, metadata) => {
if (err) {
sublogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err);
this.ffprobeLogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err);
resolve(true);
return;
}
resolve(metadata.streams.some((stream) => stream.codec_type === 'video'));
});
} catch (err) {
sublogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err as Error);
this.ffprobeLogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err as Error);
resolve(true);
}
});

10
packages/backend/src/core/GlobalEventService.ts
View file

@ -265,6 +265,7 @@ export interface InternalEventTypes {
unmute: { muterId: MiUser['id']; muteeId: MiUser['id']; };
userListMemberAdded: { userListId: MiUserList['id']; memberId: MiUser['id']; };
userListMemberRemoved: { userListId: MiUserList['id']; memberId: MiUser['id']; };
quantumCacheUpdated: { name: string, keys: string[] };
}
type EventTypesToEventPayload<T> = EventUnionFromDictionary<UndefinedAsNullAll<SerializedAll<T>>>;
@ -353,12 +354,12 @@ export class GlobalEventService {
}
@bindThis
private publish(channel: StreamChannels, type: string | null, value?: any): void {
private async publish(channel: StreamChannels, type: string | null, value?: any): Promise<void> {
const message = type == null ? value : value == null ?
{ type: type, body: null } :
{ type: type, body: value };
this.redisForPub.publish(this.config.host, JSON.stringify({
await this.redisForPub.publish(this.config.host, JSON.stringify({
channel: channel,
message: message,
}));
@ -369,6 +370,11 @@ export class GlobalEventService {
this.publish('internal', type, typeof value === 'undefined' ? null : value);
}
@bindThis
public async publishInternalEventAsync<K extends keyof InternalEventTypes>(type: K, value?: InternalEventTypes[K]): Promise<void> {
await this.publish('internal', type, typeof value === 'undefined' ? null : value);
}
@bindThis
public publishBroadcastStream<K extends keyof BroadcastTypes>(type: K, value?: BroadcastTypes[K]): void {
this.publish('broadcast', type, typeof value === 'undefined' ? null : value);

10
packages/backend/src/core/HttpRequestService.ts
View file

@ -235,7 +235,7 @@ export class HttpRequestService {
}
@bindThis
public async getActivityJson(url: string, isLocalAddressAllowed = false): Promise<IObjectWithId> {
public async getActivityJson(url: string, isLocalAddressAllowed = false, allowAnonymous = false): Promise<IObjectWithId> {
this.apUtilityService.assertApUrl(url);
const res = await this.send(url, {
@ -255,7 +255,11 @@ export class HttpRequestService {
// Make sure the object ID matches the final URL (which is where it actually exists).
// The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match.
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
if (allowAnonymous && activity.id == null) {
activity.id = res.url;
} else {
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
}
return activity as IObjectWithId;
}
@ -327,7 +331,7 @@ export class HttpRequestService {
});
if (!res.ok && extra.throwErrorWhenResponseNotOk) {
throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
throw new StatusError(`request error from ${url}`, res.status, res.statusText);
}
if (res.ok) {

103
packages/backend/src/core/InternalEventService.ts Normal file
View file

@ -0,0 +1,103 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import Redis from 'ioredis';
import { DI } from '@/di-symbols.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import type { GlobalEvents, InternalEventTypes } from '@/core/GlobalEventService.js';
import { bindThis } from '@/decorators.js';
export type Listener<K extends keyof InternalEventTypes> = (value: InternalEventTypes[K], key: K, isLocal: boolean) => void | Promise<void>;
export interface ListenerProps {
ignoreLocal?: boolean,
ignoreRemote?: boolean,
}
@Injectable()
export class InternalEventService implements OnApplicationShutdown {
private readonly listeners = new Map<keyof InternalEventTypes, Map<Listener<keyof InternalEventTypes>, ListenerProps>>();
constructor(
@Inject(DI.redisForSub)
private readonly redisForSub: Redis.Redis,
private readonly globalEventService: GlobalEventService,
) {
this.redisForSub.on('message', this.onMessage);
}
@bindThis
public on<K extends keyof InternalEventTypes>(type: K, listener: Listener<K>, props?: ListenerProps): void {
let set = this.listeners.get(type);
if (!set) {
set = new Map();
this.listeners.set(type, set);
}
// Functionally, this is just a set with metadata on the values.
set.set(listener as Listener<keyof InternalEventTypes>, props ?? {});
}
@bindThis
public off<K extends keyof InternalEventTypes>(type: K, listener: Listener<K>): void {
this.listeners.get(type)?.delete(listener as Listener<keyof InternalEventTypes>);
}
@bindThis
public async emit<K extends keyof InternalEventTypes>(type: K, value: InternalEventTypes[K]): Promise<void> {
await this.emitInternal(type, value, true);
await this.globalEventService.publishInternalEventAsync(type, { ...value, _pid: process.pid });
}
@bindThis
private async emitInternal<K extends keyof InternalEventTypes>(type: K, value: InternalEventTypes[K], isLocal: boolean): Promise<void> {
const listeners = this.listeners.get(type);
if (!listeners) {
return;
}
const promises: Promise<void>[] = [];
for (const [listener, props] of listeners) {
if ((isLocal && !props.ignoreLocal) || (!isLocal && !props.ignoreRemote)) {
const promise = Promise.resolve(listener(value, type, isLocal));
promises.push(promise);
}
}
await Promise.all(promises);
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
if (!isLocalInternalEvent(body) || body._pid !== process.pid) {
await this.emitInternal(type, body as InternalEventTypes[keyof InternalEventTypes], false);
}
}
}
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.listeners.clear();
}
@bindThis
public onApplicationShutdown(): void {
this.dispose();
}
}
interface LocalInternalEvent {
_pid: number;
}
function isLocalInternalEvent(body: object): body is LocalInternalEvent {
return '_pid' in body && typeof(body._pid) === 'number';
}

25
packages/backend/src/core/LatestNoteService.ts
View file

@ -7,6 +7,7 @@ import { DI } from '@/di-symbols.js';
import type { LatestNotesRepository, NotesRepository } from '@/models/_.js';
import { LoggerService } from '@/core/LoggerService.js';
import Logger from '@/logger.js';
import { QueryService } from './QueryService.js';
@Injectable()
export class LatestNoteService {
@ -14,11 +15,12 @@ export class LatestNoteService {
constructor(
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private readonly notesRepository: NotesRepository,
@Inject(DI.latestNotesRepository)
private latestNotesRepository: LatestNotesRepository,
private readonly latestNotesRepository: LatestNotesRepository,
private readonly queryService: QueryService,
loggerService: LoggerService,
) {
this.logger = loggerService.getLogger('LatestNoteService');
@ -91,7 +93,7 @@ export class LatestNoteService {
// Find the newest remaining note for the user.
// We exclude DMs and pure renotes.
const nextLatest = await this.notesRepository
const query = this.notesRepository
.createQueryBuilder('note')
.select()
.where({
@ -106,18 +108,11 @@ export class LatestNoteService {
? Not(null)
: null,
})
.andWhere(`
(
note."renoteId" IS NULL
OR note.text IS NOT NULL
OR note.cw IS NOT NULL
OR note."replyId" IS NOT NULL
OR note."hasPoll"
OR note."fileIds" != '{}'
)
`)
.orderBy({ id: 'DESC' })
.getOne();
.orderBy({ id: 'DESC' });
this.queryService.andIsNotRenote(query, 'note');
const nextLatest = await query.getOne();
if (!nextLatest) return;
// Record it as the latest

113
packages/backend/src/core/MetaService.ts
View file

@ -4,7 +4,7 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { DataSource } from 'typeorm';
import { DataSource, EntityManager } from 'typeorm';
import * as Redis from 'ioredis';
import { DI } from '@/di-symbols.js';
import { MiMeta } from '@/models/Meta.js';
@ -12,6 +12,9 @@ import { GlobalEventService } from '@/core/GlobalEventService.js';
import { bindThis } from '@/decorators.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { FeaturedService } from '@/core/FeaturedService.js';
import { MiInstance } from '@/models/Instance.js';
import { diffArrays } from '@/misc/diff-arrays.js';
import type { MetasRepository } from '@/models/_.js';
import type { OnApplicationShutdown } from '@nestjs/common';
@Injectable()
@ -26,6 +29,9 @@ export class MetaService implements OnApplicationShutdown {
@Inject(DI.db)
private db: DataSource,
@Inject(DI.metasRepository)
private readonly metasRepository: MetasRepository,
private featuredService: FeaturedService,
private globalEventService: GlobalEventService,
) {
@ -67,35 +73,35 @@ export class MetaService implements OnApplicationShutdown {
public async fetch(noCache = false): Promise<MiMeta> {
if (!noCache && this.cache) return this.cache;
return await this.db.transaction(async transactionalEntityManager => {
// 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する
const metas = await transactionalEntityManager.find(MiMeta, {
order: {
// 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する
let meta = await this.metasRepository.createQueryBuilder('meta')
.select()
.orderBy({
id: 'DESC',
})
.limit(1)
.getOne();
if (!meta) {
await this.metasRepository.createQueryBuilder('meta')
.insert()
.values({
id: 'x',
})
.orIgnore()
.execute();
meta = await this.metasRepository.createQueryBuilder('meta')
.select()
.orderBy({
id: 'DESC',
},
});
})
.limit(1)
.getOneOrFail();
}
const meta = metas[0];
if (meta) {
this.cache = meta;
return meta;
} else {
// metaが空のときfetchMetaが同時に呼ばれるとここが同時に呼ばれてしまうことがあるのでフェイルセーフなupsertを使う
const saved = await transactionalEntityManager
.upsert(
MiMeta,
{
id: 'x',
},
['id'],
)
.then((x) => transactionalEntityManager.findOneByOrFail(MiMeta, x.identifiers[0]));
this.cache = saved;
return saved;
}
});
this.cache = meta;
return meta;
}
@bindThis
@ -103,7 +109,7 @@ export class MetaService implements OnApplicationShutdown {
let before: MiMeta | undefined;
const updated = await this.db.transaction(async transactionalEntityManager => {
const metas = await transactionalEntityManager.find(MiMeta, {
const metas: (MiMeta | undefined)[] = await transactionalEntityManager.find(MiMeta, {
order: {
id: 'DESC',
},
@ -126,6 +132,10 @@ export class MetaService implements OnApplicationShutdown {
},
});
// Propagate changes to blockedHosts, silencedHosts, mediaSilencedHosts, federationInstances, and bubbleInstances to the relevant instance rows
// Do this inside the transaction to avoid potential race condition (when an instance gets registered while we're updating).
await this.persistBlocks(transactionalEntityManager, before ?? {}, afters[0]);
return afters[0];
});
@ -159,4 +169,49 @@ export class MetaService implements OnApplicationShutdown {
public onApplicationShutdown(signal?: string | undefined): void {
this.dispose();
}
private async persistBlocks(tem: EntityManager, before: Partial<MiMeta>, after: Partial<MiMeta>): Promise<void> {
await this.persistBlock(tem, before.blockedHosts, after.blockedHosts, 'isBlocked');
await this.persistBlock(tem, before.silencedHosts, after.silencedHosts, 'isSilenced');
await this.persistBlock(tem, before.mediaSilencedHosts, after.mediaSilencedHosts, 'isMediaSilenced');
await this.persistBlock(tem, before.federationHosts, after.federationHosts, 'isAllowListed');
await this.persistBlock(tem, before.bubbleInstances, after.bubbleInstances, 'isBubbled');
}
private async persistBlock(tem: EntityManager, before: string[] | undefined, after: string[] | undefined, field: keyof MiInstance): Promise<void> {
const { added, removed } = diffArrays(before, after);
if (removed.length > 0) {
await this.updateInstancesByHost(tem, field, false, removed);
}
if (added.length > 0) {
await this.updateInstancesByHost(tem, field, true, added);
}
}
private async updateInstancesByHost(tem: EntityManager, field: keyof MiInstance, value: boolean, hosts: string[]): Promise<void> {
// Use non-array queries when possible, as they are indexed and can be much faster.
if (hosts.length === 1) {
const pattern = genHostPattern(hosts[0]);
await tem
.createQueryBuilder(MiInstance, 'instance')
.update()
.set({ [field]: value })
.where('(lower(reverse("host")) || \'.\') LIKE :pattern', { pattern })
.execute();
} else if (hosts.length > 1) {
const patterns = hosts.map(host => genHostPattern(host));
await tem
.createQueryBuilder(MiInstance, 'instance')
.update()
.set({ [field]: value })
.where('(lower(reverse("host")) || \'.\') LIKE ANY (:patterns)', { patterns })
.execute();
}
}
}
function genHostPattern(host: string): string {
return host.toLowerCase().split('').reverse().join('') + '.%';
}

497
packages/backend/src/core/MfmService.ts
View file

@ -5,25 +5,22 @@
import { URL } from 'node:url';
import { Inject, Injectable } from '@nestjs/common';
import * as parse5 from 'parse5';
import { type Document, type HTMLParagraphElement, Window } from 'happy-dom';
import { isText, isTag, Text } from 'domhandler';
import * as htmlparser2 from 'htmlparser2';
import { Node, Document, ChildNode, Element, ParentNode } from 'domhandler';
import * as domserializer from 'dom-serializer';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { intersperse } from '@/misc/prelude/array.js';
import { normalizeForSearch } from '@/misc/normalize-for-search.js';
import type { IMentionedRemoteUsers } from '@/models/Note.js';
import { bindThis } from '@/decorators.js';
import type { DefaultTreeAdapterMap } from 'parse5';
import type * as mfm from '@transfem-org/sfm-js';
const treeAdapter = parse5.defaultTreeAdapter;
type Node = DefaultTreeAdapterMap['node'];
type ChildNode = DefaultTreeAdapterMap['childNode'];
import type * as mfm from 'mfm-js';
const urlRegex = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+/;
const urlRegexFull = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+$/;
export type Appender = (document: Document, body: HTMLParagraphElement) => void;
export type Appender = (document: Document, body: Element) => void;
@Injectable()
export class MfmService {
@ -40,7 +37,7 @@ export class MfmService {
const normalizedHashtagNames = hashtagNames == null ? undefined : new Set<string>(hashtagNames.map(x => normalizeForSearch(x)));
const dom = parse5.parseFragment(html);
const dom = htmlparser2.parseDocument(html);
let text = '';
@ -51,57 +48,50 @@ export class MfmService {
return text.trim();
function getText(node: Node): string {
if (treeAdapter.isTextNode(node)) return node.value;
if (!treeAdapter.isElementNode(node)) return '';
if (node.nodeName === 'br') return '\n';
if (isText(node)) return node.data;
if (!isTag(node)) return '';
if (node.tagName === 'br') return '\n';
if (node.childNodes) {
return node.childNodes.map(n => getText(n)).join('');
}
return '';
return node.childNodes.map(n => getText(n)).join('');
}
function appendChildren(childNodes: ChildNode[]): void {
if (childNodes) {
for (const n of childNodes) {
analyze(n);
}
for (const n of childNodes) {
analyze(n);
}
}
function analyze(node: Node) {
if (treeAdapter.isTextNode(node)) {
text += node.value;
if (isText(node)) {
text += node.data;
return;
}
// Skip comment or document type node
if (!treeAdapter.isElementNode(node)) {
if (!isTag(node)) {
return;
}
switch (node.nodeName) {
switch (node.tagName) {
case 'br': {
text += '\n';
break;
return;
}
case 'a': {
const txt = getText(node);
const rel = node.attrs.find(x => x.name === 'rel');
const href = node.attrs.find(x => x.name === 'href');
const rel = node.attribs.rel;
const href = node.attribs.href;
// ハッシュタグ
if (normalizedHashtagNames && href && normalizedHashtagNames.has(normalizeForSearch(txt))) {
text += txt;
// メンション
} else if (txt.startsWith('@') && !(rel && rel.value.startsWith('me '))) {
} else if (txt.startsWith('@') && !(rel && rel.startsWith('me '))) {
const part = txt.split('@');
if (part.length === 2 && href) {
//#region ホスト名部分が省略されているので復元する
const acct = `${txt}@${(new URL(href.value)).hostname}`;
const acct = `${txt}@${(new URL(href)).hostname}`;
text += acct;
//#endregion
} else if (part.length === 3) {
@ -116,25 +106,32 @@ export class MfmService {
if (!href) {
return txt;
}
if (!txt || txt === href.value) { // #6383: Missing text node
if (href.value.match(urlRegexFull)) {
return href.value;
if (!txt || txt === href) { // #6383: Missing text node
if (href.match(urlRegexFull)) {
return href;
} else {
return `<${href.value}>`;
return `<${href}>`;
}
}
if (href.value.match(urlRegex) && !href.value.match(urlRegexFull)) {
return `[${txt}](<${href.value}>)`; // #6846
if (href.match(urlRegex) && !href.match(urlRegexFull)) {
return `[${txt}](<${href}>)`; // #6846
} else {
return `[${txt}](${href.value})`;
return `[${txt}](${href})`;
}
};
text += generateLink();
}
break;
return;
}
}
// Don't produce invalid empty MFM
if (node.childNodes.length < 1) {
return;
}
switch (node.tagName) {
case 'h1': {
text += '**【';
appendChildren(node.childNodes);
@ -185,14 +182,17 @@ export class MfmService {
case 'ruby--': {
let ruby: [string, string][] = [];
for (const child of node.childNodes) {
if (child.nodeName === 'rp') {
if (isText(child) && !/\s|\[|\]/.test(child.data)) {
ruby.push([child.data, '']);
continue;
}
if (treeAdapter.isTextNode(child) && !/\s|\[|\]/.test(child.value)) {
ruby.push([child.value, '']);
if (!isTag(child)) {
continue;
}
if (child.nodeName === 'rt' && ruby.length > 0) {
if (child.tagName === 'rp') {
continue;
}
if (child.tagName === 'rt' && ruby.length > 0) {
const rt = getText(child);
if (/\s|\[|\]/.test(rt)) {
// If any space is included in rt, it is treated as a normal text
@ -217,7 +217,7 @@ export class MfmService {
// block code (<pre><code>)
case 'pre': {
if (node.childNodes.length === 1 && node.childNodes[0].nodeName === 'code') {
if (node.childNodes.length === 1 && isTag(node.childNodes[0]) && node.childNodes[0].tagName === 'code') {
text += '\n```\n';
text += getText(node.childNodes[0]);
text += '\n```\n';
@ -302,17 +302,17 @@ export class MfmService {
let nonRtNodes = [];
// scan children, ignore `rp`, split on `rt`
for (const child of node.childNodes) {
if (treeAdapter.isTextNode(child)) {
if (isText(child)) {
nonRtNodes.push(child);
continue;
}
if (!treeAdapter.isElementNode(child)) {
if (!isTag(child)) {
continue;
}
if (child.nodeName === 'rp') {
if (child.tagName === 'rp') {
continue;
}
if (child.nodeName === 'rt') {
if (child.tagName === 'rt') {
// the only case in which we don't need a `$[group ]`
// is when both sides of the ruby are simple words
const needsGroup = nonRtNodes.length > 1 ||
@ -350,45 +350,44 @@ export class MfmService {
return null;
}
const { happyDOM, window } = new Window();
const doc = new Document([]);
const doc = window.document;
const body = new Element('p', {});
doc.childNodes.push(body);
const body = doc.createElement('p');
function appendChildren(children: mfm.MfmNode[], targetElement: any): void {
if (children) {
for (const child of children.map(x => (handlers as any)[x.type](x))) targetElement.appendChild(child);
function appendChildren(children: mfm.MfmNode[], targetElement: ParentNode): void {
for (const child of children.map(x => handle(x))) {
targetElement.childNodes.push(child);
}
}
function fnDefault(node: mfm.MfmFn) {
const el = doc.createElement('i');
const el = new Element('i', {});
appendChildren(node.children, el);
return el;
}
const handlers: { [K in mfm.MfmNode['type']]: (node: mfm.NodeType<K>) => any } = {
const handlers: { [K in mfm.MfmNode['type']]: (node: mfm.NodeType<K>) => ChildNode } = {
bold: (node) => {
const el = doc.createElement('b');
const el = new Element('b', {});
appendChildren(node.children, el);
return el;
},
small: (node) => {
const el = doc.createElement('small');
const el = new Element('small', {});
appendChildren(node.children, el);
return el;
},
strike: (node) => {
const el = doc.createElement('del');
const el = new Element('del', {});
appendChildren(node.children, el);
return el;
},
italic: (node) => {
const el = doc.createElement('i');
const el = new Element('i', {});
appendChildren(node.children, el);
return el;
},
@ -399,11 +398,12 @@ export class MfmService {
const text = node.children[0].type === 'text' ? node.children[0].props.text : '';
try {
const date = new Date(parseInt(text, 10) * 1000);
const el = doc.createElement('time');
el.setAttribute('datetime', date.toISOString());
el.textContent = date.toISOString();
const el = new Element('time', {
datetime: date.toISOString(),
});
el.childNodes.push(new Text(date.toISOString()));
return el;
} catch (err) {
} catch {
return fnDefault(node);
}
}
@ -412,20 +412,20 @@ export class MfmService {
if (node.children.length === 1) {
const child = node.children[0];
const text = child.type === 'text' ? child.props.text : '';
const rubyEl = doc.createElement('ruby');
const rtEl = doc.createElement('rt');
const rubyEl = new Element('ruby', {});
const rtEl = new Element('rt', {});
// ruby未対応のHTMLサニタイザーを通したときにルビが「劉備（りゅうび）」となるようにする
const rpStartEl = doc.createElement('rp');
rpStartEl.appendChild(doc.createTextNode('('));
const rpEndEl = doc.createElement('rp');
rpEndEl.appendChild(doc.createTextNode(')'));
const rpStartEl = new Element('rp', {});
rpStartEl.childNodes.push(new Text('('));
const rpEndEl = new Element('rp', {});
rpEndEl.childNodes.push(new Text(')'));
rubyEl.appendChild(doc.createTextNode(text.split(' ')[0]));
rtEl.appendChild(doc.createTextNode(text.split(' ')[1]));
rubyEl.appendChild(rpStartEl);
rubyEl.appendChild(rtEl);
rubyEl.appendChild(rpEndEl);
rubyEl.childNodes.push(new Text(text.split(' ')[0]));
rtEl.childNodes.push(new Text(text.split(' ')[1]));
rubyEl.childNodes.push(rpStartEl);
rubyEl.childNodes.push(rtEl);
rubyEl.childNodes.push(rpEndEl);
return rubyEl;
} else {
const rt = node.children.at(-1);
@ -435,20 +435,20 @@ export class MfmService {
}
const text = rt.type === 'text' ? rt.props.text : '';
const rubyEl = doc.createElement('ruby');
const rtEl = doc.createElement('rt');
const rubyEl = new Element('ruby', {});
const rtEl = new Element('rt', {});
// ruby未対応のHTMLサニタイザーを通したときにルビが「劉備（りゅうび）」となるようにする
const rpStartEl = doc.createElement('rp');
rpStartEl.appendChild(doc.createTextNode('('));
const rpEndEl = doc.createElement('rp');
rpEndEl.appendChild(doc.createTextNode(')'));
const rpStartEl = new Element('rp', {});
rpStartEl.childNodes.push(new Text('('));
const rpEndEl = new Element('rp', {});
rpEndEl.childNodes.push(new Text(')'));
appendChildren(node.children.slice(0, node.children.length - 1), rubyEl);
rtEl.appendChild(doc.createTextNode(text.trim()));
rubyEl.appendChild(rpStartEl);
rubyEl.appendChild(rtEl);
rubyEl.appendChild(rpEndEl);
rtEl.childNodes.push(new Text(text.trim()));
rubyEl.childNodes.push(rpStartEl);
rubyEl.childNodes.push(rtEl);
rubyEl.childNodes.push(rpEndEl);
return rubyEl;
}
}
@ -456,7 +456,7 @@ export class MfmService {
// hack for ruby, should never be needed because we should
// never send this out to other instances
case 'group': {
const el = doc.createElement('span');
const el = new Element('span', {});
appendChildren(node.children, el);
return el;
}
@ -468,125 +468,135 @@ export class MfmService {
},
blockCode: (node) => {
const pre = doc.createElement('pre');
const inner = doc.createElement('code');
inner.textContent = node.props.code;
pre.appendChild(inner);
const pre = new Element('pre', {});
const inner = new Element('code', {});
inner.childNodes.push(new Text(node.props.code));
pre.childNodes.push(inner);
return pre;
},
center: (node) => {
const el = doc.createElement('div');
const el = new Element('div', {});
appendChildren(node.children, el);
return el;
},
emojiCode: (node) => {
return doc.createTextNode(`\u200B:${node.props.name}:\u200B`);
return new Text(`\u200B:${node.props.name}:\u200B`);
},
unicodeEmoji: (node) => {
return doc.createTextNode(node.props.emoji);
return new Text(node.props.emoji);
},
hashtag: (node) => {
const a = doc.createElement('a');
a.setAttribute('href', `${this.config.url}/tags/${node.props.hashtag}`);
a.textContent = `#${node.props.hashtag}`;
a.setAttribute('rel', 'tag');
const a = new Element('a', {
href: `${this.config.url}/tags/${node.props.hashtag}`,
rel: 'tag',
});
a.childNodes.push(new Text(`#${node.props.hashtag}`));
return a;
},
inlineCode: (node) => {
const el = doc.createElement('code');
el.textContent = node.props.code;
const el = new Element('code', {});
el.childNodes.push(new Text(node.props.code));
return el;
},
mathInline: (node) => {
const el = doc.createElement('code');
el.textContent = node.props.formula;
const el = new Element('code', {});
el.childNodes.push(new Text(node.props.formula));
return el;
},
mathBlock: (node) => {
const el = doc.createElement('code');
el.textContent = node.props.formula;
const el = new Element('code', {});
el.childNodes.push(new Text(node.props.formula));
return el;
},
link: (node) => {
const a = doc.createElement('a');
a.setAttribute('href', node.props.url);
const a = new Element('a', {
href: node.props.url,
});
appendChildren(node.children, a);
return a;
},
mention: (node) => {
const a = doc.createElement('a');
const { username, host, acct } = node.props;
const remoteUserInfo = mentionedRemoteUsers.find(remoteUser => remoteUser.username.toLowerCase() === username.toLowerCase() && remoteUser.host?.toLowerCase() === host?.toLowerCase());
a.setAttribute('href', remoteUserInfo
? (remoteUserInfo.url ? remoteUserInfo.url : remoteUserInfo.uri)
: `${this.config.url}/${acct.endsWith(`@${this.config.url}`) ? acct.substring(0, acct.length - this.config.url.length - 1) : acct}`);
a.className = 'u-url mention';
a.textContent = acct;
const a = new Element('a', {
href: remoteUserInfo
? (remoteUserInfo.url ? remoteUserInfo.url : remoteUserInfo.uri)
: `${this.config.url}/${acct.endsWith(`@${this.config.url}`) ? acct.substring(0, acct.length - this.config.url.length - 1) : acct}`,
class: 'u-url mention',
});
a.childNodes.push(new Text(acct));
return a;
},
quote: (node) => {
const el = doc.createElement('blockquote');
const el = new Element('blockquote', {});
appendChildren(node.children, el);
return el;
},
text: (node) => {
if (!node.props.text.match(/[\r\n]/)) {
return doc.createTextNode(node.props.text);
return new Text(node.props.text);
}
const el = doc.createElement('span');
const nodes = node.props.text.split(/\r\n|\r|\n/).map(x => doc.createTextNode(x));
const el = new Element('span', {});
const nodes = node.props.text.split(/\r\n|\r|\n/).map(x => new Text(x));
for (const x of intersperse<FIXME | 'br'>('br', nodes)) {
el.appendChild(x === 'br' ? doc.createElement('br') : x);
el.childNodes.push(x === 'br' ? new Element('br', {}) : x);
}
return el;
},
url: (node) => {
const a = doc.createElement('a');
a.setAttribute('href', node.props.url);
a.textContent = node.props.url;
const a = new Element('a', {
href: node.props.url,
});
a.childNodes.push(new Text(node.props.url));
return a;
},
search: (node) => {
const a = doc.createElement('a');
a.setAttribute('href', `https://www.google.com/search?q=${node.props.query}`);
a.textContent = node.props.content;
const a = new Element('a', {
href: `https://www.google.com/search?q=${node.props.query}`,
});
a.childNodes.push(new Text(node.props.content));
return a;
},
plain: (node) => {
const el = doc.createElement('span');
const el = new Element('span', {});
appendChildren(node.children, el);
return el;
},
};
// Utility function to make TypeScript behave
function handle<T extends mfm.MfmNode>(node: T): ChildNode {
const handler = handlers[node.type] as (node: T) => ChildNode;
return handler(node);
}
appendChildren(nodes, body);
for (const additionalAppender of additionalAppenders) {
additionalAppender(doc, body);
}
const serialized = body.outerHTML;
happyDOM.close().catch(err => {});
return serialized;
return domserializer.render(body, {
encodeEntities: 'utf8'
});
}
// the toMastoApiHtml function was taken from Iceshrimp and written by zotan and modified by marie to work with the current MK version
@ -598,55 +608,55 @@ export class MfmService {
return null;
}
const { happyDOM, window } = new Window();
const doc = new Document([]);
const doc = window.document;
const body = new Element('p', {});
doc.childNodes.push(body);
const body = doc.createElement('p');
function appendChildren(children: mfm.MfmNode[], targetElement: any): void {
if (children) {
for (const child of children.map((x) => (handlers as any)[x.type](x))) targetElement.appendChild(child);
function appendChildren(children: mfm.MfmNode[], targetElement: ParentNode): void {
for (const child of children) {
const result = handle(child);
targetElement.childNodes.push(result);
}
}
const handlers: {
[K in mfm.MfmNode['type']]: (node: mfm.NodeType<K>) => any;
[K in mfm.MfmNode['type']]: (node: mfm.NodeType<K>) => ChildNode;
} = {
bold(node) {
const el = doc.createElement('span');
el.textContent = '**';
const el = new Element('span', {});
el.childNodes.push(new Text('**'));
appendChildren(node.children, el);
el.textContent += '**';
el.childNodes.push(new Text('**'));
return el;
},
small(node) {
const el = doc.createElement('small');
const el = new Element('small', {});
appendChildren(node.children, el);
return el;
},
strike(node) {
const el = doc.createElement('span');
el.textContent = '~~';
const el = new Element('span', {});
el.childNodes.push(new Text('~~'));
appendChildren(node.children, el);
el.textContent += '~~';
el.childNodes.push(new Text('~~'));
return el;
},
italic(node) {
const el = doc.createElement('span');
el.textContent = '*';
const el = new Element('span', {});
el.childNodes.push(new Text('*'));
appendChildren(node.children, el);
el.textContent += '*';
el.childNodes.push(new Text('*'));
return el;
},
fn(node) {
switch (node.props.name) {
case 'group': { // hack for ruby
const el = doc.createElement('span');
const el = new Element('span', {});
appendChildren(node.children, el);
return el;
}
@ -654,119 +664,121 @@ export class MfmService {
if (node.children.length === 1) {
const child = node.children[0];
const text = child.type === 'text' ? child.props.text : '';
const rubyEl = doc.createElement('ruby');
const rtEl = doc.createElement('rt');
const rubyEl = new Element('ruby', {});
const rtEl = new Element('rt', {});
const rpStartEl = doc.createElement('rp');
rpStartEl.appendChild(doc.createTextNode('('));
const rpEndEl = doc.createElement('rp');
rpEndEl.appendChild(doc.createTextNode(')'));
const rpStartEl = new Element('rp', {});
rpStartEl.childNodes.push(new Text('('));
const rpEndEl = new Element('rp', {});
rpEndEl.childNodes.push(new Text(')'));
rubyEl.appendChild(doc.createTextNode(text.split(' ')[0]));
rtEl.appendChild(doc.createTextNode(text.split(' ')[1]));
rubyEl.appendChild(rpStartEl);
rubyEl.appendChild(rtEl);
rubyEl.appendChild(rpEndEl);
rubyEl.childNodes.push(new Text(text.split(' ')[0]));
rtEl.childNodes.push(new Text(text.split(' ')[1]));
rubyEl.childNodes.push(rpStartEl);
rubyEl.childNodes.push(rtEl);
rubyEl.childNodes.push(rpEndEl);
return rubyEl;
} else {
const rt = node.children.at(-1);
if (!rt) {
const el = doc.createElement('span');
const el = new Element('span', {});
appendChildren(node.children, el);
return el;
}
const text = rt.type === 'text' ? rt.props.text : '';
const rubyEl = doc.createElement('ruby');
const rtEl = doc.createElement('rt');
const rubyEl = new Element('ruby', {});
const rtEl = new Element('rt', {});
const rpStartEl = doc.createElement('rp');
rpStartEl.appendChild(doc.createTextNode('('));
const rpEndEl = doc.createElement('rp');
rpEndEl.appendChild(doc.createTextNode(')'));
const rpStartEl = new Element('rp', {});
rpStartEl.childNodes.push(new Text('('));
const rpEndEl = new Element('rp', {});
rpEndEl.childNodes.push(new Text(')'));
appendChildren(node.children.slice(0, node.children.length - 1), rubyEl);
rtEl.appendChild(doc.createTextNode(text.trim()));
rubyEl.appendChild(rpStartEl);
rubyEl.appendChild(rtEl);
rubyEl.appendChild(rpEndEl);
rtEl.childNodes.push(new Text(text.trim()));
rubyEl.childNodes.push(rpStartEl);
rubyEl.childNodes.push(rtEl);
rubyEl.childNodes.push(rpEndEl);
return rubyEl;
}
}
default: {
const el = doc.createElement('span');
el.textContent = '*';
const el = new Element('span', {});
el.childNodes.push(new Text('*'));
appendChildren(node.children, el);
el.textContent += '*';
el.childNodes.push(new Text('*'));
return el;
}
}
},
blockCode(node) {
const pre = doc.createElement('pre');
const inner = doc.createElement('code');
const pre = new Element('pre', {});
const inner = new Element('code', {});
const nodes = node.props.code
.split(/\r\n|\r|\n/)
.map((x) => doc.createTextNode(x));
.map((x) => new Text(x));
for (const x of intersperse<FIXME | 'br'>('br', nodes)) {
inner.appendChild(x === 'br' ? doc.createElement('br') : x);
inner.childNodes.push(x === 'br' ? new Element('br', {}) : x);
}
pre.appendChild(inner);
pre.childNodes.push(inner);
return pre;
},
center(node) {
const el = doc.createElement('div');
const el = new Element('div', {});
appendChildren(node.children, el);
return el;
},
emojiCode(node) {
return doc.createTextNode(`\u200B:${node.props.name}:\u200B`);
return new Text(`\u200B:${node.props.name}:\u200B`);
},
unicodeEmoji(node) {
return doc.createTextNode(node.props.emoji);
return new Text(node.props.emoji);
},
hashtag: (node) => {
const a = doc.createElement('a');
a.setAttribute('href', `${this.config.url}/tags/${node.props.hashtag}`);
a.textContent = `#${node.props.hashtag}`;
a.setAttribute('rel', 'tag');
a.setAttribute('class', 'hashtag');
const a = new Element('a', {
href: `${this.config.url}/tags/${node.props.hashtag}`,
rel: 'tag',
class: 'hashtag',
});
a.childNodes.push(new Text(`#${node.props.hashtag}`));
return a;
},
inlineCode(node) {
const el = doc.createElement('code');
el.textContent = node.props.code;
const el = new Element('code', {});
el.childNodes.push(new Text(node.props.code));
return el;
},
mathInline(node) {
const el = doc.createElement('code');
el.textContent = node.props.formula;
const el = new Element('code', {});
el.childNodes.push(new Text(node.props.formula));
return el;
},
mathBlock(node) {
const el = doc.createElement('code');
el.textContent = node.props.formula;
const el = new Element('code', {});
el.childNodes.push(new Text(node.props.formula));
return el;
},
link(node) {
const a = doc.createElement('a');
a.setAttribute('rel', 'nofollow noopener noreferrer');
a.setAttribute('target', '_blank');
a.setAttribute('href', node.props.url);
const a = new Element('a', {
rel: 'nofollow noopener noreferrer',
target: '_blank',
href: node.props.url,
});
appendChildren(node.children, a);
return a;
},
@ -775,92 +787,107 @@ export class MfmService {
const { username, host, acct } = node.props;
const resolved = mentionedRemoteUsers.find(remoteUser => remoteUser.username === username && remoteUser.host === host);
const el = doc.createElement('span');
const el = new Element('span', {});
if (!resolved) {
el.textContent = acct;
el.childNodes.push(new Text(acct));
} else {
el.setAttribute('class', 'h-card');
el.setAttribute('translate', 'no');
const a = doc.createElement('a');
a.setAttribute('href', resolved.url ? resolved.url : resolved.uri);
a.className = 'u-url mention';
const span = doc.createElement('span');
span.textContent = resolved.username || username;
a.textContent = '@';
a.appendChild(span);
el.appendChild(a);
el.attribs.class = 'h-card';
el.attribs.translate = 'no';
const a = new Element('a', {
href: resolved.url ? resolved.url : resolved.uri,
class: 'u-url mention',
});
const span = new Element('span', {});
span.childNodes.push(new Text(resolved.username || username));
a.childNodes.push(new Text('@'));
a.childNodes.push(span);
el.childNodes.push(a);
}
return el;
},
quote(node) {
const el = doc.createElement('blockquote');
const el = new Element('blockquote', {});
appendChildren(node.children, el);
return el;
},
text(node) {
const el = doc.createElement('span');
if (!node.props.text.match(/[\r\n]/)) {
return new Text(node.props.text);
}
const el = new Element('span', {});
const nodes = node.props.text
.split(/\r\n|\r|\n/)
.map((x) => doc.createTextNode(x));
.map((x) => new Text(x));
for (const x of intersperse<FIXME | 'br'>('br', nodes)) {
el.appendChild(x === 'br' ? doc.createElement('br') : x);
el.childNodes.push(x === 'br' ? new Element('br', {}) : x);
}
return el;
},
url(node) {
const a = doc.createElement('a');
a.setAttribute('rel', 'nofollow noopener noreferrer');
a.setAttribute('target', '_blank');
a.setAttribute('href', node.props.url);
a.textContent = node.props.url.replace(/^https?:\/\//, '');
const a = new Element('a', {
rel: 'nofollow noopener noreferrer',
target: '_blank',
href: node.props.url,
});
a.childNodes.push(new Text(node.props.url.replace(/^https?:\/\//, '')));
return a;
},
search: (node) => {
const a = doc.createElement('a');
a.setAttribute('href', `https://www.google.com/search?q=${node.props.query}`);
a.textContent = node.props.content;
const a = new Element('a', {
href: `https://www.google.com/search?q=${node.props.query}`,
});
a.childNodes.push(new Text(node.props.content));
return a;
},
plain(node) {
const el = doc.createElement('span');
const el = new Element('span', {});
appendChildren(node.children, el);
return el;
},
};
// Utility function to make TypeScript behave
function handle<T extends mfm.MfmNode>(node: T): ChildNode {
const handler = handlers[node.type] as (node: T) => ChildNode;
return handler(node);
}
appendChildren(nodes, body);
if (quoteUri !== null) {
const a = doc.createElement('a');
a.setAttribute('href', quoteUri);
a.textContent = quoteUri.replace(/^https?:\/\//, '');
const a = new Element('a', {
href: quoteUri,
});
a.childNodes.push(new Text(quoteUri.replace(/^https?:\/\//, '')));
const quote = doc.createElement('span');
quote.setAttribute('class', 'quote-inline');
quote.appendChild(doc.createElement('br'));
quote.appendChild(doc.createElement('br'));
quote.innerHTML += 'RE: ';
quote.appendChild(a);
const quote = new Element('span', {
class: 'quote-inline',
});
quote.childNodes.push(new Element('br', {}));
quote.childNodes.push(new Element('br', {}));
quote.childNodes.push(new Text('RE: '));
quote.childNodes.push(a);
body.appendChild(quote);
body.childNodes.push(quote);
}
let result = body.outerHTML;
let result = domserializer.render(body, {
encodeEntities: 'utf8'
});
if (inline) {
result = result.replace(/^<p>/, '').replace(/<\/p>$/, '');
}
happyDOM.close().catch(() => {});
return result;
}
}

71
packages/backend/src/core/NoteCreateService.ts
View file

@ -4,7 +4,7 @@
*/
import { setImmediate } from 'node:timers/promises';
import * as mfm from '@transfem-org/sfm-js';
import * as mfm from 'mfm-js';
import { In, DataSource, IsNull, LessThan } from 'typeorm';
import * as Redis from 'ioredis';
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
@ -296,7 +296,7 @@ export class NoteCreateService implements OnApplicationShutdown {
case 'followers':
// 他人のfollowers noteはreject
if (data.renote.userId !== user.id) {
throw new Error('Renote target is not public or home');
throw new IdentifiableError('b6352a84-e5cd-4b05-a26c-63437a6b98ba', 'Renote target is not public or home');
}
// Renote対象がfollowersならfollowersにする
@ -304,7 +304,7 @@ export class NoteCreateService implements OnApplicationShutdown {
break;
case 'specified':
// specified / direct noteはreject
throw new Error('Renote target is not public or home');
throw new IdentifiableError('b6352a84-e5cd-4b05-a26c-63437a6b98ba', 'Renote target is not public or home');
}
}
@ -317,7 +317,7 @@ export class NoteCreateService implements OnApplicationShutdown {
if (data.renote.userId !== user.id) {
const blocked = await this.userBlockingService.checkBlocked(data.renote.userId, user.id);
if (blocked) {
throw new Error('blocked');
throw new IdentifiableError('b6352a84-e5cd-4b05-a26c-63437a6b98ba', 'Renote target is blocked');
}
}
}
@ -489,10 +489,10 @@ export class NoteCreateService implements OnApplicationShutdown {
// should really not happen, but better safe than sorry
if (data.reply?.id === insert.id) {
throw new Error('A note can\'t reply to itself');
throw new IdentifiableError('ea93b7c2-3d6c-4e10-946b-00d50b1a75cb', 'A note can\'t reply to itself');
}
if (data.renote?.id === insert.id) {
throw new Error('A note can\'t renote itself');
throw new IdentifiableError('ea93b7c2-3d6c-4e10-946b-00d50b1a75cb', 'A note can\'t renote itself');
}
if (data.uri != null) insert.uri = data.uri;
@ -549,8 +549,6 @@ export class NoteCreateService implements OnApplicationShutdown {
throw err;
}
console.error(e);
throw e;
}
}
@ -608,11 +606,11 @@ export class NoteCreateService implements OnApplicationShutdown {
}
if (data.reply == null) {
// TODO: キャッシュ
this.followingsRepository.findBy({
followeeId: user.id,
notify: 'normal',
}).then(async followings => {
this.cacheService.userFollowersCache.fetch(user.id).then(async followingsMap => {
const followings = Array
.from(followingsMap.values())
.filter(f => f.notify === 'normal');
if (note.visibility !== 'specified') {
const isPureRenote = this.isRenote(data) && !this.isQuote(data) ? true : false;
for (const following of followings) {
@ -733,7 +731,7 @@ export class NoteCreateService implements OnApplicationShutdown {
//#region AP deliver
if (!data.localOnly && this.userEntityService.isLocalUser(user)) {
trackTask(async () => {
const noteActivity = await this.renderNoteOrRenoteActivity(data, note, user);
const noteActivity = await this.apRendererService.renderNoteOrRenoteActivity(note, user, { renote: data.renote });
const dm = this.apDeliverManagerService.createDeliverManager(user, noteActivity);
// メンションされたリモートユーザーに配送
@ -876,17 +874,6 @@ export class NoteCreateService implements OnApplicationShutdown {
this.notesRepository.increment({ id: reply.id }, 'repliesCount', 1);
}
@bindThis
private async renderNoteOrRenoteActivity(data: Option, note: MiNote, user: MiUser) {
if (data.localOnly) return null;
const content = this.isRenote(data) && !this.isQuote(data)
? this.apRendererService.renderAnnounce(data.renote.uri ? data.renote.uri : `${this.config.url}/notes/${data.renote.id}`, note)
: this.apRendererService.renderCreate(await this.apRendererService.renderNote(note, user, false), note);
return this.apRendererService.addContext(content);
}
@bindThis
private index(note: MiNote) {
if (note.text == null && note.cw == null) return;
@ -950,14 +937,7 @@ export class NoteCreateService implements OnApplicationShutdown {
// TODO: キャッシュ？
// eslint-disable-next-line prefer-const
let [followings, userListMemberships] = await Promise.all([
this.followingsRepository.find({
where: {
followeeId: user.id,
followerHost: IsNull(),
isFollowerHibernated: false,
},
select: ['followerId', 'withReplies'],
}),
this.cacheService.getNonHibernatedFollowers(user.id),
this.userListMembershipsRepository.find({
where: {
userId: user.id,
@ -973,6 +953,7 @@ export class NoteCreateService implements OnApplicationShutdown {
// TODO: あまりにも数が多いと redisPipeline.exec に失敗する(理由は不明)ため、3万件程度を目安に分割して実行するようにする
for (const following of followings) {
if (following.followerHost !== null) continue;
// 基本的にvisibleUserIdsには自身のidが含まれている前提であること
if (note.visibility === 'specified' && !note.visibleUserIds.some(v => v === following.followerId)) continue;
@ -1074,17 +1055,19 @@ export class NoteCreateService implements OnApplicationShutdown {
});
if (hibernatedUsers.length > 0) {
this.usersRepository.update({
id: In(hibernatedUsers.map(x => x.id)),
}, {
isHibernated: true,
});
this.followingsRepository.update({
followerId: In(hibernatedUsers.map(x => x.id)),
}, {
isFollowerHibernated: true,
});
await Promise.all([
this.usersRepository.update({
id: In(hibernatedUsers.map(x => x.id)),
}, {
isHibernated: true,
}),
this.followingsRepository.update({
followerId: In(hibernatedUsers.map(x => x.id)),
}, {
isFollowerHibernated: true,
}),
this.cacheService.hibernatedUserCache.setMany(hibernatedUsers.map(x => [x.id, true])),
]);
}
}

55
packages/backend/src/core/NoteEditService.ts
View file

@ -4,7 +4,7 @@
*/
import { setImmediate } from 'node:timers/promises';
import * as mfm from '@transfem-org/sfm-js';
import * as mfm from 'mfm-js';
import { DataSource, In, IsNull, LessThan } from 'typeorm';
import * as Redis from 'ioredis';
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
@ -309,7 +309,7 @@ export class NoteEditService implements OnApplicationShutdown {
if (this.isRenote(data)) {
if (data.renote.id === oldnote.id) {
throw new UnrecoverableError(`edit failed for ${oldnote.id}: cannot renote itself`);
throw new IdentifiableError('ea93b7c2-3d6c-4e10-946b-00d50b1a75cb', `edit failed for ${oldnote.id}: cannot renote itself`);
}
switch (data.renote.visibility) {
@ -325,7 +325,7 @@ export class NoteEditService implements OnApplicationShutdown {
case 'followers':
// 他人のfollowers noteはreject
if (data.renote.userId !== user.id) {
throw new Error('Renote target is not public or home');
throw new IdentifiableError('b6352a84-e5cd-4b05-a26c-63437a6b98ba', 'Renote target is not public or home');
}
// Renote対象がfollowersならfollowersにする
@ -333,7 +333,7 @@ export class NoteEditService implements OnApplicationShutdown {
break;
case 'specified':
// specified / direct noteはreject
throw new Error('Renote target is not public or home');
throw new IdentifiableError('b6352a84-e5cd-4b05-a26c-63437a6b98ba', 'Renote target is not public or home');
}
}
@ -675,7 +675,7 @@ export class NoteEditService implements OnApplicationShutdown {
//#region AP deliver
if (!data.localOnly && this.userEntityService.isLocalUser(user)) {
trackTask(async () => {
const noteActivity = await this.renderNoteOrRenoteActivity(data, note, user);
const noteActivity = await this.apRendererService.renderNoteOrRenoteActivity(note, user, { renote: data.renote });
const dm = this.apDeliverManagerService.createDeliverManager(user, noteActivity);
// メンションされたリモートユーザーに配送
@ -770,17 +770,6 @@ export class NoteEditService implements OnApplicationShutdown {
(note.files != null && note.files.length > 0);
}
@bindThis
private async renderNoteOrRenoteActivity(data: Option, note: MiNote, user: MiUser) {
if (data.localOnly) return null;
const content = this.isRenote(data) && !this.isQuote(data)
? this.apRendererService.renderAnnounce(data.renote.uri ? data.renote.uri : `${this.config.url}/notes/${data.renote.id}`, note)
: this.apRendererService.renderUpdate(await this.apRendererService.renderUpNote(note, user, false), user);
return this.apRendererService.addContext(content);
}
@bindThis
private index(note: MiNote) {
if (note.text == null && note.cw == null) return;
@ -833,14 +822,7 @@ export class NoteEditService implements OnApplicationShutdown {
// TODO: キャッシュ？
// eslint-disable-next-line prefer-const
let [followings, userListMemberships] = await Promise.all([
this.followingsRepository.find({
where: {
followeeId: user.id,
followerHost: IsNull(),
isFollowerHibernated: false,
},
select: ['followerId', 'withReplies'],
}),
this.cacheService.getNonHibernatedFollowers(user.id),
this.userListMembershipsRepository.find({
where: {
userId: user.id,
@ -856,6 +838,7 @@ export class NoteEditService implements OnApplicationShutdown {
// TODO: あまりにも数が多いと redisPipeline.exec に失敗する(理由は不明)ため、3万件程度を目安に分割して実行するようにする
for (const following of followings) {
if (following.followerHost !== null) continue;
// 基本的にvisibleUserIdsには自身のidが含まれている前提であること
if (note.visibility === 'specified' && !note.visibleUserIds.some(v => v === following.followerId)) continue;
@ -957,17 +940,19 @@ export class NoteEditService implements OnApplicationShutdown {
});
if (hibernatedUsers.length > 0) {
this.usersRepository.update({
id: In(hibernatedUsers.map(x => x.id)),
}, {
isHibernated: true,
});
this.followingsRepository.update({
followerId: In(hibernatedUsers.map(x => x.id)),
}, {
isFollowerHibernated: true,
});
await Promise.all([
this.usersRepository.update({
id: In(hibernatedUsers.map(x => x.id)),
}, {
isHibernated: true,
}),
this.followingsRepository.update({
followerId: In(hibernatedUsers.map(x => x.id)),
}, {
isFollowerHibernated: true,
}),
this.cacheService.hibernatedUserCache.setMany(hibernatedUsers.map(x => [x.id, true])),
]);
}
}

4
packages/backend/src/core/NotePiningService.ts
View file

@ -61,7 +61,7 @@ export class NotePiningService {
});
if (note == null) {
throw new IdentifiableError('70c4e51f-5bea-449c-a030-53bee3cce202', 'No such note.');
throw new IdentifiableError('70c4e51f-5bea-449c-a030-53bee3cce202', `Note ${noteId} does not exist`);
}
await this.db.transaction(async tem => {
@ -102,7 +102,7 @@ export class NotePiningService {
});
if (note == null) {
throw new IdentifiableError('b302d4cf-c050-400a-bbb3-be208681f40c', 'No such note.');
throw new IdentifiableError('b302d4cf-c050-400a-bbb3-be208681f40c', `Note ${noteId} does not exist`);
}
this.userNotePiningsRepository.delete({

12
packages/backend/src/core/NotificationService.ts
View file

@ -113,27 +113,27 @@ export class NotificationService implements OnApplicationShutdown {
}
if (recieveConfig?.type === 'following') {
const isFollowing = await this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => Object.hasOwn(followings, notifierId));
const isFollowing = await this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => followings.has(notifierId));
if (!isFollowing) {
return null;
}
} else if (recieveConfig?.type === 'follower') {
const isFollower = await this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => Object.hasOwn(followings, notifieeId));
const isFollower = await this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => followings.has(notifieeId));
if (!isFollower) {
return null;
}
} else if (recieveConfig?.type === 'mutualFollow') {
const [isFollowing, isFollower] = await Promise.all([
this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => Object.hasOwn(followings, notifierId)),
this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => Object.hasOwn(followings, notifieeId)),
this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => followings.has(notifierId)),
this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => followings.has(notifieeId)),
]);
if (!(isFollowing && isFollower)) {
return null;
}
} else if (recieveConfig?.type === 'followingOrFollower') {
const [isFollowing, isFollower] = await Promise.all([
this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => Object.hasOwn(followings, notifierId)),
this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => Object.hasOwn(followings, notifieeId)),
this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => followings.has(notifierId)),
this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => followings.has(notifieeId)),
]);
if (!isFollowing && !isFollower) {
return null;

19
packages/backend/src/core/PushNotificationService.ts
View file

@ -12,7 +12,8 @@ import type { Packed } from '@/misc/json-schema.js';
import { getNoteSummary } from '@/misc/get-note-summary.js';
import type { MiMeta, MiSwSubscription, SwSubscriptionsRepository } from '@/models/_.js';
import { bindThis } from '@/decorators.js';
import { RedisKVCache } from '@/misc/cache.js';
import { QuantumKVCache } from '@/misc/QuantumKVCache.js';
import { InternalEventService } from '@/core/InternalEventService.js';
// Defined also packages/sw/types.ts#L13
type PushNotificationsTypes = {
@ -48,7 +49,7 @@ function truncateBody<T extends keyof PushNotificationsTypes>(type: T, body: Pus
@Injectable()
export class PushNotificationService implements OnApplicationShutdown {
private subscriptionsCache: RedisKVCache<MiSwSubscription[]>;
private subscriptionsCache: QuantumKVCache<MiSwSubscription[]>;
constructor(
@Inject(DI.config)
@ -62,13 +63,11 @@ export class PushNotificationService implements OnApplicationShutdown {
@Inject(DI.swSubscriptionsRepository)
private swSubscriptionsRepository: SwSubscriptionsRepository,
private readonly internalEventService: InternalEventService,
) {
this.subscriptionsCache = new RedisKVCache<MiSwSubscription[]>(this.redisClient, 'userSwSubscriptions', {
this.subscriptionsCache = new QuantumKVCache<MiSwSubscription[]>(this.internalEventService, 'userSwSubscriptions', {
lifetime: 1000 * 60 * 60 * 1, // 1h
memoryCacheLifetime: 1000 * 60 * 3, // 3m
fetcher: (key) => this.swSubscriptionsRepository.findBy({ userId: key }),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => JSON.parse(value),
});
}
@ -114,8 +113,8 @@ export class PushNotificationService implements OnApplicationShutdown {
endpoint: subscription.endpoint,
auth: subscription.auth,
publickey: subscription.publickey,
}).then(() => {
this.refreshCache(userId);
}).then(async () => {
await this.refreshCache(userId);
});
}
});
@ -123,8 +122,8 @@ export class PushNotificationService implements OnApplicationShutdown {
}
@bindThis
public refreshCache(userId: string): void {
this.subscriptionsCache.refresh(userId);
public async refreshCache(userId: string): Promise<void> {
await this.subscriptionsCache.refresh(userId);
}
@bindThis

621
packages/backend/src/core/QueryService.ts
View file

@ -4,13 +4,14 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { Brackets, ObjectLiteral } from 'typeorm';
import { Brackets, Not, WhereExpressionBuilder } from 'typeorm';
import { DI } from '@/di-symbols.js';
import type { MiUser } from '@/models/User.js';
import type { UserProfilesRepository, FollowingsRepository, ChannelFollowingsRepository, BlockingsRepository, NoteThreadMutingsRepository, MutingsRepository, RenoteMutingsRepository, MiMeta } from '@/models/_.js';
import { MiInstance } from '@/models/Instance.js';
import type { UserProfilesRepository, FollowingsRepository, ChannelFollowingsRepository, BlockingsRepository, NoteThreadMutingsRepository, MutingsRepository, RenoteMutingsRepository, MiMeta, InstancesRepository } from '@/models/_.js';
import { bindThis } from '@/decorators.js';
import { IdService } from '@/core/IdService.js';
import type { SelectQueryBuilder } from 'typeorm';
import type { SelectQueryBuilder, ObjectLiteral } from 'typeorm';
@Injectable()
export class QueryService {
@ -36,6 +37,9 @@ export class QueryService {
@Inject(DI.renoteMutingsRepository)
private renoteMutingsRepository: RenoteMutingsRepository,
@Inject(DI.instancesRepository)
private readonly instancesRepository: InstancesRepository,
@Inject(DI.meta)
private meta: MiMeta,
@ -72,218 +76,485 @@ export class QueryService {
// ここでいうBlockedは被Blockedの意
@bindThis
public generateBlockedUserQueryForNotes(q: SelectQueryBuilder<any>, me: { id: MiUser['id'] }): void {
const blockingQuery = this.blockingsRepository.createQueryBuilder('blocking')
.select('blocking.blockerId')
.where('blocking.blockeeId = :blockeeId', { blockeeId: me.id });
public generateBlockedUserQueryForNotes<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, me: { id: MiUser['id'] }): SelectQueryBuilder<E> {
// 投稿の作者にブロックされていない かつ
// 投稿の返信先の作者にブロックされていない かつ
// 投稿の引用元の作者にブロックされていない
q
.andWhere(`note.userId NOT IN (${ blockingQuery.getQuery() })`)
.andWhere(new Brackets(qb => {
qb
.where('note.replyUserId IS NULL')
.orWhere(`note.replyUserId NOT IN (${ blockingQuery.getQuery() })`);
}))
.andWhere(new Brackets(qb => {
qb
.where('note.renoteUserId IS NULL')
.orWhere(`note.renoteUserId NOT IN (${ blockingQuery.getQuery() })`);
}));
q.setParameters(blockingQuery.getParameters());
return this
.andNotBlockingUser(q, 'note.userId', ':meId')
.andWhere(new Brackets(qb => this
.orNotBlockingUser(qb, 'note.replyUserId', ':meId')
.orWhere('note.replyUserId IS NULL')))
.andWhere(new Brackets(qb => this
.orNotBlockingUser(qb, 'note.renoteUserId', ':meId')
.orWhere('note.renoteUserId IS NULL')))
.setParameters({ meId: me.id });
}
@bindThis
public generateBlockQueryForUsers(q: SelectQueryBuilder<any>, me: { id: MiUser['id'] }): void {
const blockingQuery = this.blockingsRepository.createQueryBuilder('blocking')
.select('blocking.blockeeId')
.where('blocking.blockerId = :blockerId', { blockerId: me.id });
const blockedQuery = this.blockingsRepository.createQueryBuilder('blocking')
.select('blocking.blockerId')
.where('blocking.blockeeId = :blockeeId', { blockeeId: me.id });
q.andWhere(`user.id NOT IN (${ blockingQuery.getQuery() })`);
q.setParameters(blockingQuery.getParameters());
q.andWhere(`user.id NOT IN (${ blockedQuery.getQuery() })`);
q.setParameters(blockedQuery.getParameters());
public generateBlockQueryForUsers<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, me: { id: MiUser['id'] }): SelectQueryBuilder<E> {
this.andNotBlockingUser(q, ':meId', 'user.id');
this.andNotBlockingUser(q, 'user.id', ':meId');
return q.setParameters({ meId: me.id });
}
@bindThis
public generateMutedNoteThreadQuery(q: SelectQueryBuilder<any>, me: { id: MiUser['id'] }): void {
const mutedQuery = this.noteThreadMutingsRepository.createQueryBuilder('threadMuted')
.select('threadMuted.threadId')
.where('threadMuted.userId = :userId', { userId: me.id });
q.andWhere(`note.id NOT IN (${ mutedQuery.getQuery() })`);
q.andWhere(new Brackets(qb => {
qb
.where('note.threadId IS NULL')
.orWhere(`note.threadId NOT IN (${ mutedQuery.getQuery() })`);
}));
q.setParameters(mutedQuery.getParameters());
public generateMutedNoteThreadQuery<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, me: { id: MiUser['id'] }): SelectQueryBuilder<E> {
return this
.andNotMutingThread(q, ':meId', 'note.id')
.andWhere(new Brackets(qb => this
.orNotMutingThread(qb, ':meId', 'note.threadId')
.orWhere('note.threadId IS NULL')))
.setParameters({ meId: me.id });
}
@bindThis
public generateMutedUserQueryForNotes(q: SelectQueryBuilder<any>, me: { id: MiUser['id'] }, exclude?: { id: MiUser['id'] }): void {
const mutingQuery = this.mutingsRepository.createQueryBuilder('muting')
.select('muting.muteeId')
.where('muting.muterId = :muterId', { muterId: me.id });
if (exclude) {
mutingQuery.andWhere('muting.muteeId != :excludeId', { excludeId: exclude.id });
}
const mutingInstanceQuery = this.userProfilesRepository.createQueryBuilder('user_profile')
.select('user_profile.mutedInstances')
.where('user_profile.userId = :muterId', { muterId: me.id });
public generateMutedUserQueryForNotes<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, me: { id: MiUser['id'] }, exclude?: { id: MiUser['id'] }): SelectQueryBuilder<E> {
// 投稿の作者をミュートしていない かつ
// 投稿の返信先の作者をミュートしていない かつ
// 投稿の引用元の作者をミュートしていない
q
.andWhere(`note.userId NOT IN (${ mutingQuery.getQuery() })`)
.andWhere(new Brackets(qb => {
qb
.where('note.replyUserId IS NULL')
.orWhere(`note.replyUserId NOT IN (${ mutingQuery.getQuery() })`);
}))
.andWhere(new Brackets(qb => {
qb
.where('note.renoteUserId IS NULL')
.orWhere(`note.renoteUserId NOT IN (${ mutingQuery.getQuery() })`);
}))
return this
.andNotMutingUser(q, ':meId', 'note.userId', exclude)
.andWhere(new Brackets(qb => this
.orNotMutingUser(qb, ':meId', 'note.replyUserId', exclude)
.orWhere('note.replyUserId IS NULL')))
.andWhere(new Brackets(qb => this
.orNotMutingUser(qb, ':meId', 'note.renoteUserId', exclude)
.orWhere('note.renoteUserId IS NULL')))
// TODO exclude should also pass a host to skip these instances
// mute instances
.andWhere(new Brackets(qb => {
qb
.andWhere('note.userHost IS NULL')
.orWhere(`NOT ((${ mutingInstanceQuery.getQuery() })::jsonb ? note.userHost)`);
}))
.andWhere(new Brackets(qb => {
qb
.where('note.replyUserHost IS NULL')
.orWhere(`NOT ((${ mutingInstanceQuery.getQuery() })::jsonb ? note.replyUserHost)`);
}))
.andWhere(new Brackets(qb => {
qb
.where('note.renoteUserHost IS NULL')
.orWhere(`NOT ((${ mutingInstanceQuery.getQuery() })::jsonb ? note.renoteUserHost)`);
}));
q.setParameters(mutingQuery.getParameters());
q.setParameters(mutingInstanceQuery.getParameters());
.andWhere(new Brackets(qb => this
.andNotMutingInstance(qb, ':meId', 'note.userHost')
.orWhere('note.userHost IS NULL')))
.andWhere(new Brackets(qb => this
.orNotMutingInstance(qb, ':meId', 'note.replyUserHost')
.orWhere('note.replyUserHost IS NULL')))
.andWhere(new Brackets(qb => this
.orNotMutingInstance(qb, ':meId', 'note.renoteUserHost')
.orWhere('note.renoteUserHost IS NULL')))
.setParameters({ meId: me.id });
}
@bindThis
public generateMutedUserQueryForUsers(q: SelectQueryBuilder<any>, me: { id: MiUser['id'] }): void {
const mutingQuery = this.mutingsRepository.createQueryBuilder('muting')
.select('muting.muteeId')
.where('muting.muterId = :muterId', { muterId: me.id });
q.andWhere(`user.id NOT IN (${ mutingQuery.getQuery() })`);
q.setParameters(mutingQuery.getParameters());
public generateMutedUserQueryForUsers<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, me: { id: MiUser['id'] }): SelectQueryBuilder<E> {
return this
.andNotMutingUser(q, ':meId', 'user.id')
.setParameters({ meId: me.id });
}
// This intentionally skips isSuspended, isDeleted, makeNotesFollowersOnlyBefore, makeNotesHiddenBefore, and requireSigninToViewContents.
// NoteEntityService checks these automatically and calls hideNote() to hide them without breaking threads.
// For moderation purposes, you can set isSilenced to forcibly hide existing posts by a user.
@bindThis
public generateVisibilityQuery(q: SelectQueryBuilder<any>, me?: { id: MiUser['id'] } | null): void {
public generateVisibilityQuery<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, me?: { id: MiUser['id'] } | null): SelectQueryBuilder<E> {
// This code must always be synchronized with the checks in Notes.isVisibleForMe.
if (me == null) {
q.andWhere(new Brackets(qb => {
qb
.where('note.visibility = \'public\'')
.orWhere('note.visibility = \'home\'');
}));
} else {
const followingQuery = this.followingsRepository.createQueryBuilder('following')
.select('following.followeeId')
.where('following.followerId = :meId');
return q.andWhere(new Brackets(qb => {
// Public post
qb.orWhere('note.visibility = \'public\'')
.orWhere('note.visibility = \'home\'');
q.andWhere(new Brackets(qb => {
if (me != null) {
qb
// 公開投稿である
.where(new Brackets(qb => {
qb
.where('note.visibility = \'public\'')
.orWhere('note.visibility = \'home\'');
}))
// または 自分自身
.orWhere('note.userId = :meId')
// または 自分宛て
// My post
.orWhere(':meId = note.userId')
// Visible to me
.orWhere(':meIdAsList <@ note.visibleUserIds')
.orWhere(new Brackets(qb => {
qb
// または フォロワー宛ての投稿であり、
.where('note.visibility = \'followers\'')
.andWhere(new Brackets(qb => {
qb
// 自分がフォロワーである
.where(`note.userId IN (${ followingQuery.getQuery() })`)
// または 自分の投稿へのリプライ
.orWhere('note.replyUserId = :meId')
.orWhere(':meIdAsList <@ note.mentions');
}));
}));
// Followers-only post
.orWhere(new Brackets(qb => qb
.andWhere(new Brackets(qbb => this
// Following author
.orFollowingUser(qbb, ':meId', 'note.userId')
// Mentions me
.orWhere(':meIdAsList <@ note.mentions')
// Reply to me
.orWhere(':meId = note.replyUserId')))
.andWhere('note.visibility = \'followers\'')));
q.setParameters({ meId: me.id, meIdAsList: [me.id] });
}
}));
}
@bindThis
public generateMutedUserRenotesQueryForNotes<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, me: { id: MiUser['id'] }): SelectQueryBuilder<E> {
return q
.andWhere(new Brackets(qb => this
.orNotMutingRenote(qb, ':meId', 'note.userId')
.orWhere('note.renoteId IS NULL')
.orWhere('note.text IS NOT NULL')
.orWhere('note.cw IS NOT NULL')
.orWhere('note.replyId IS NOT NULL')
.orWhere('note.hasPoll = true')
.orWhere('note.fileIds != \'{}\'')))
.setParameters({ meId: me.id });
}
@bindThis
public generateExcludedRenotesQueryForNotes<Q extends WhereExpressionBuilder>(q: Q): Q {
return this.andIsNotRenote(q, 'note');
}
@bindThis
public generateBlockedHostQueryForNote<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, excludeAuthor?: boolean): SelectQueryBuilder<E> {
const checkFor = (key: 'user' | 'replyUser' | 'renoteUser') => this
.leftJoinInstance(q, `note.${key}Instance`, `${key}Instance`)
.andWhere(new Brackets(qb => {
qb
.orWhere(`"${key}Instance" IS NULL`) // local
.orWhere(`"${key}Instance"."isBlocked" = false`); // not blocked
if (excludeAuthor) {
qb.orWhere(`note.userId = note.${key}Id`); // author
}
}));
q.setParameters({ meId: me.id, meIdAsList: [me.id] });
if (!excludeAuthor) {
checkFor('user');
}
checkFor('replyUser');
checkFor('renoteUser');
return q;
}
@bindThis
public generateMutedUserRenotesQueryForNotes(q: SelectQueryBuilder<any>, me: { id: MiUser['id'] }): void {
public generateSilencedUserQueryForNotes<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, me?: { id: MiUser['id'] } | null): SelectQueryBuilder<E> {
if (!me) {
return q.andWhere('user.isSilenced = false');
}
return this
.leftJoinInstance(q, 'note.userInstance', 'userInstance')
.andWhere(new Brackets(qb => this
// case 1: we are following the user
.orFollowingUser(qb, ':meId', 'note.userId')
// case 2: user not silenced AND instance not silenced
.orWhere(new Brackets(qbb => qbb
.andWhere(new Brackets(qbbb => qbbb
.orWhere('"userInstance"."isSilenced" = false')
.orWhere('"userInstance" IS NULL')))
.andWhere('user.isSilenced = false')))))
.setParameters({ meId: me.id });
}
/**
* Left-joins an instance in to the query with a given alias and optional condition.
* These calls are de-duplicated - multiple uses of the same alias are skipped.
*/
@bindThis
public leftJoinInstance<E extends ObjectLiteral>(q: SelectQueryBuilder<E>, relation: string | typeof MiInstance, alias: string, condition?: string): SelectQueryBuilder<E> {
// Skip if it's already joined, otherwise we'll get an error
if (!q.expressionMap.joinAttributes.some(j => j.alias.name === alias)) {
q.leftJoin(relation, alias, condition);
}
return q;
}
/**
* Adds OR condition that noteProp (note ID) refers to a quote.
* The prop should be an expression, not a raw value.
*/
@bindThis
public orIsQuote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string): Q {
return this.addIsQuote(q, noteProp, 'orWhere');
}
/**
* Adds AND condition that noteProp (note ID) refers to a quote.
* The prop should be an expression, not a raw value.
*/
@bindThis
public andIsQuote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string): Q {
return this.addIsQuote(q, noteProp, 'andWhere');
}
private addIsQuote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string, join: 'andWhere' | 'orWhere'): Q {
return q[join](new Brackets(qb => qb
.andWhere(`${noteProp}.renoteId IS NOT NULL`)
.andWhere(new Brackets(qbb => qbb
.orWhere(`${noteProp}.text IS NOT NULL`)
.orWhere(`${noteProp}.cw IS NOT NULL`)
.orWhere(`${noteProp}.replyId IS NOT NULL`)
.orWhere(`${noteProp}.hasPoll = true`)
.orWhere(`${noteProp}.fileIds != '{}'`)))));
}
/**
* Adds OR condition that noteProp (note ID) does not refer to a quote.
* The prop should be an expression, not a raw value.
*/
@bindThis
public orIsNotQuote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string): Q {
return this.addIsNotQuote(q, noteProp, 'orWhere');
}
/**
* Adds AND condition that noteProp (note ID) does not refer to a quote.
* The prop should be an expression, not a raw value.
*/
@bindThis
public andIsNotQuote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string): Q {
return this.addIsNotQuote(q, noteProp, 'andWhere');
}
private addIsNotQuote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string, join: 'andWhere' | 'orWhere'): Q {
return q[join](new Brackets(qb => qb
.orWhere(`${noteProp}.renoteId IS NULL`)
.orWhere(new Brackets(qb => qb
.andWhere(`${noteProp}.text IS NULL`)
.andWhere(`${noteProp}.cw IS NULL`)
.andWhere(`${noteProp}.replyId IS NULL`)
.andWhere(`${noteProp}.hasPoll = false`)
.andWhere(`${noteProp}.fileIds = '{}'`)))));
}
/**
* Adds OR condition that noteProp (note ID) refers to a renote.
* The prop should be an expression, not a raw value.
*/
@bindThis
public orIsRenote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string): Q {
return this.addIsRenote(q, noteProp, 'orWhere');
}
/**
* Adds AND condition that noteProp (note ID) refers to a renote.
* The prop should be an expression, not a raw value.
*/
@bindThis
public andIsRenote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string): Q {
return this.addIsRenote(q, noteProp, 'andWhere');
}
private addIsRenote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string, join: 'andWhere' | 'orWhere'): Q {
return q[join](new Brackets(qb => qb
.andWhere(`${noteProp}.renoteId IS NOT NULL`)
.andWhere(`${noteProp}.text IS NULL`)
.andWhere(`${noteProp}.cw IS NULL`)
.andWhere(`${noteProp}.replyId IS NULL`)
.andWhere(`${noteProp}.hasPoll = false`)
.andWhere(`${noteProp}.fileIds = '{}'`)));
}
/**
* Adds OR condition that noteProp (note ID) does not refer to a renote.
* The prop should be an expression, not a raw value.
*/
@bindThis
public orIsNotRenote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string): Q {
return this.addIsNotRenote(q, noteProp, 'orWhere');
}
/**
* Adds AND condition that noteProp (note ID) does not refer to a renote.
* The prop should be an expression, not a raw value.
*/
@bindThis
public andIsNotRenote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string): Q {
return this.addIsNotRenote(q, noteProp, 'andWhere');
}
private addIsNotRenote<Q extends WhereExpressionBuilder>(q: Q, noteProp: string, join: 'andWhere' | 'orWhere'): Q {
return q[join](new Brackets(qb => qb
.orWhere(`${noteProp}.renoteId IS NULL`)
.orWhere(`${noteProp}.text IS NOT NULL`)
.orWhere(`${noteProp}.cw IS NOT NULL`)
.orWhere(`${noteProp}.replyId IS NOT NULL`)
.orWhere(`${noteProp}.hasPoll = true`)
.orWhere(`${noteProp}.fileIds != '{}'`)));
}
/**
* Adds OR condition that followerProp (user ID) is following followeeProp (user ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public orFollowingUser<Q extends WhereExpressionBuilder>(q: Q, followerProp: string, followeeProp: string): Q {
return this.addFollowingUser(q, followerProp, followeeProp, 'orWhere');
}
/**
* Adds AND condition that followerProp (user ID) is following followeeProp (user ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public andFollowingUser<Q extends WhereExpressionBuilder>(q: Q, followerProp: string, followeeProp: string): Q {
return this.addFollowingUser(q, followerProp, followeeProp, 'andWhere');
}
private addFollowingUser<Q extends WhereExpressionBuilder>(q: Q, followerProp: string, followeeProp: string, join: 'andWhere' | 'orWhere'): Q {
const followingQuery = this.followingsRepository.createQueryBuilder('following')
.select('1')
.andWhere(`following.followerId = ${followerProp}`)
.andWhere(`following.followeeId = ${followeeProp}`);
return q[join](`EXISTS (${followingQuery.getQuery()})`, followingQuery.getParameters());
};
/**
* Adds OR condition that followerProp (user ID) is following followeeProp (channel ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public orFollowingChannel<Q extends WhereExpressionBuilder>(q: Q, followerProp: string, followeeProp: string): Q {
return this.addFollowingChannel(q, followerProp, followeeProp, 'orWhere');
}
/**
* Adds AND condition that followerProp (user ID) is following followeeProp (channel ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public andFollowingChannel<Q extends WhereExpressionBuilder>(q: Q, followerProp: string, followeeProp: string): Q {
return this.addFollowingChannel(q, followerProp, followeeProp, 'andWhere');
}
private addFollowingChannel<Q extends WhereExpressionBuilder>(q: Q, followerProp: string, followeeProp: string, join: 'andWhere' | 'orWhere'): Q {
const followingQuery = this.channelFollowingsRepository.createQueryBuilder('following')
.select('1')
.andWhere(`following.followerId = ${followerProp}`)
.andWhere(`following.followeeId = ${followeeProp}`);
return q[join](`EXISTS (${followingQuery.getQuery()})`, followingQuery.getParameters());
}
/**
* Adds OR condition that blockerProp (user ID) is not blocking blockeeProp (user ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public orNotBlockingUser<Q extends WhereExpressionBuilder>(q: Q, blockerProp: string, blockeeProp: string): Q {
return this.excludeBlockingUser(q, blockerProp, blockeeProp, 'orWhere');
}
/**
* Adds AND condition that blockerProp (user ID) is not blocking blockeeProp (user ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public andNotBlockingUser<Q extends WhereExpressionBuilder>(q: Q, blockerProp: string, blockeeProp: string): Q {
return this.excludeBlockingUser(q, blockerProp, blockeeProp, 'andWhere');
}
private excludeBlockingUser<Q extends WhereExpressionBuilder>(q: Q, blockerProp: string, blockeeProp: string, join: 'andWhere' | 'orWhere'): Q {
const blockingQuery = this.blockingsRepository.createQueryBuilder('blocking')
.select('1')
.andWhere(`blocking.blockerId = ${blockerProp}`)
.andWhere(`blocking.blockeeId = ${blockeeProp}`);
return q[join](`NOT EXISTS (${blockingQuery.getQuery()})`, blockingQuery.getParameters());
};
/**
* Adds OR condition that muterProp (user ID) is not muting muteeProp (user ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public orNotMutingUser<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string, exclude?: { id: MiUser['id'] }): Q {
return this.excludeMutingUser(q, muterProp, muteeProp, 'orWhere', exclude);
}
/**
* Adds AND condition that muterProp (user ID) is not muting muteeProp (user ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public andNotMutingUser<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string, exclude?: { id: MiUser['id'] }): Q {
return this.excludeMutingUser(q, muterProp, muteeProp, 'andWhere', exclude);
}
private excludeMutingUser<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string, join: 'andWhere' | 'orWhere', exclude?: { id: MiUser['id'] }): Q {
const mutingQuery = this.mutingsRepository.createQueryBuilder('muting')
.select('1')
.andWhere(`muting.muterId = ${muterProp}`)
.andWhere(`muting.muteeId = ${muteeProp}`);
if (exclude) {
mutingQuery.andWhere({ muteeId: Not(exclude.id) });
}
return q[join](`NOT EXISTS (${mutingQuery.getQuery()})`, mutingQuery.getParameters());
}
/**
* Adds OR condition that muterProp (user ID) is not muting renotes by muteeProp (user ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public orNotMutingRenote<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string): Q {
return this.excludeMutingRenote(q, muterProp, muteeProp, 'orWhere');
}
/**
* Adds AND condition that muterProp (user ID) is not muting renotes by muteeProp (user ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public andNotMutingRenote<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string): Q {
return this.excludeMutingRenote(q, muterProp, muteeProp, 'andWhere');
}
private excludeMutingRenote<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string, join: 'andWhere' | 'orWhere'): Q {
const mutingQuery = this.renoteMutingsRepository.createQueryBuilder('renote_muting')
.select('renote_muting.muteeId')
.where('renote_muting.muterId = :muterId', { muterId: me.id });
.select('1')
.andWhere(`renote_muting.muterId = ${muterProp}`)
.andWhere(`renote_muting.muteeId = ${muteeProp}`);
q.andWhere(new Brackets(qb => {
qb
.where(new Brackets(qb => {
qb.where('note.renoteId IS NOT NULL');
qb.andWhere('note.text IS NULL');
qb.andWhere(`note.userId NOT IN (${ mutingQuery.getQuery() })`);
}))
.orWhere('note.renoteId IS NULL')
.orWhere('note.text IS NOT NULL');
}));
return q[join](`NOT EXISTS (${mutingQuery.getQuery()})`, mutingQuery.getParameters());
};
q.setParameters(mutingQuery.getParameters());
/**
* Adds OR condition that muterProp (user ID) is not muting muteeProp (instance host).
* Both props should be expressions, not raw values.
*/
@bindThis
public orNotMutingInstance<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string): Q {
return this.excludeMutingInstance(q, muterProp, muteeProp, 'orWhere');
}
/**
* Adds AND condition that muterProp (user ID) is not muting muteeProp (instance host).
* Both props should be expressions, not raw values.
*/
@bindThis
public generateBlockedHostQueryForNote(q: SelectQueryBuilder<any>, excludeAuthor?: boolean): void {
let nonBlockedHostQuery: (part: string) => string;
if (this.meta.blockedHosts.length === 0) {
nonBlockedHostQuery = () => '1=1';
} else {
nonBlockedHostQuery = (match: string) => `('.' || ${match}) NOT ILIKE ALL(select '%.' || x from (select unnest("blockedHosts") as x from "meta") t)`;
}
public andNotMutingInstance<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string): Q {
return this.excludeMutingInstance(q, muterProp, muteeProp, 'andWhere');
}
if (excludeAuthor) {
const instanceSuspension = (user: string) => new Brackets(qb => qb
.where(`note.${user}Id IS NULL`) // no corresponding user
.orWhere(`note.userId = note.${user}Id`)
.orWhere(`note.${user}Host IS NULL`) // local
.orWhere(nonBlockedHostQuery(`note.${user}Host`)));
private excludeMutingInstance<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string, join: 'andWhere' | 'orWhere'): Q {
const mutingInstanceQuery = this.userProfilesRepository.createQueryBuilder('user_profile')
.select('1')
.andWhere(`user_profile.userId = ${muterProp}`)
.andWhere(`"user_profile"."mutedInstances"::jsonb ? ${muteeProp}`);
q
.andWhere(instanceSuspension('replyUser'))
.andWhere(instanceSuspension('renoteUser'));
} else {
const instanceSuspension = (user: string) => new Brackets(qb => qb
.where(`note.${user}Id IS NULL`) // no corresponding user
.orWhere(`note.${user}Host IS NULL`) // local
.orWhere(nonBlockedHostQuery(`note.${user}Host`)));
return q[join](`NOT EXISTS (${mutingInstanceQuery.getQuery()})`, mutingInstanceQuery.getParameters());
}
q
.andWhere(instanceSuspension('user'))
.andWhere(instanceSuspension('replyUser'))
.andWhere(instanceSuspension('renoteUser'));
}
/**
* Adds OR condition that muterProp (user ID) is not muting muteeProp (note ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public orNotMutingThread<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string): Q {
return this.excludeMutingThread(q, muterProp, muteeProp, 'orWhere');
}
/**
* Adds AND condition that muterProp (user ID) is not muting muteeProp (note ID).
* Both props should be expressions, not raw values.
*/
@bindThis
public andNotMutingThread<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string): Q {
return this.excludeMutingThread(q, muterProp, muteeProp, 'andWhere');
}
private excludeMutingThread<Q extends WhereExpressionBuilder>(q: Q, muterProp: string, muteeProp: string, join: 'andWhere' | 'orWhere'): Q {
const threadMutedQuery = this.noteThreadMutingsRepository.createQueryBuilder('threadMuted')
.select('1')
.andWhere(`threadMuted.userId = ${muterProp}`)
.andWhere(`threadMuted.threadId = ${muteeProp}`);
return q[join](`NOT EXISTS (${threadMutedQuery.getQuery()})`, threadMutedQuery.getParameters());
}
}

52
packages/backend/src/core/ReactionService.ts
View file

@ -10,7 +10,7 @@ import { IdentifiableError } from '@/misc/identifiable-error.js';
import type { MiRemoteUser, MiUser } from '@/models/User.js';
import type { MiNote } from '@/models/Note.js';
import { IdService } from '@/core/IdService.js';
import type { MiNoteReaction } from '@/models/NoteReaction.js';
import { MiNoteReaction } from '@/models/NoteReaction.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { NotificationService } from '@/core/NotificationService.js';
@ -31,6 +31,7 @@ import { isQuote, isRenote } from '@/misc/is-renote.js';
import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js';
import { PER_NOTE_REACTION_USER_PAIR_CACHE_MAX } from '@/const.js';
import { CacheService } from '@/core/CacheService.js';
import type { DataSource } from 'typeorm';
const FALLBACK = '\u2764';
@ -89,6 +90,9 @@ export class ReactionService {
@Inject(DI.emojisRepository)
private emojisRepository: EmojisRepository,
@Inject(DI.db)
private readonly db: DataSource,
private utilityService: UtilityService,
private customEmojiService: CustomEmojiService,
private roleService: RoleService,
@ -113,12 +117,12 @@ export class ReactionService {
if (note.userId !== user.id) {
const blocked = await this.userBlockingService.checkBlocked(note.userId, user.id);
if (blocked) {
throw new IdentifiableError('e70412a4-7197-4726-8e74-f3e0deb92aa7');
throw new IdentifiableError('e70412a4-7197-4726-8e74-f3e0deb92aa7', 'Note not accessible for you.');
}
}
// check visibility
if (!await this.noteEntityService.isVisibleForMe(note, user.id)) {
if (!await this.noteEntityService.isVisibleForMe(note, user.id, { me: user })) {
throw new IdentifiableError('68e9d2d1-48bf-42c2-b90a-b20e09fd3d48', 'Note not accessible for you.');
}
@ -176,26 +180,28 @@ export class ReactionService {
reaction,
};
try {
await this.noteReactionsRepository.insert(record);
} catch (e) {
if (isDuplicateKeyValueError(e)) {
const exists = await this.noteReactionsRepository.findOneByOrFail({
noteId: note.id,
userId: user.id,
});
const result = await this.db.transaction(async tem => {
await tem.createQueryBuilder(MiNoteReaction, 'noteReaction')
.insert()
.values(record)
.orIgnore()
.execute();
if (exists.reaction !== reaction) {
// 別のリアクションがすでにされていたら置き換える
await this.delete(user, note);
await this.noteReactionsRepository.insert(record);
} else {
// 同じリアクションがすでにされていたらエラー
throw new IdentifiableError('51c42bb4-931a-456b-bff7-e5a8a70dd298');
}
} else {
throw e;
return await tem.createQueryBuilder(MiNoteReaction, 'noteReaction')
.select()
.where({ noteId: note.id, userId: user.id })
.getOneOrFail();
});
if (result.id !== record.id) {
// Conflict with the same ID => nothing to do.
if (result.reaction === record.reaction) {
return;
}
// 別のリアクションがすでにされていたら置き換える
await this.delete(user, note);
await this.noteReactionsRepository.insert(record);
}
// Increment reactions count
@ -316,14 +322,14 @@ export class ReactionService {
});
if (exist == null) {
throw new IdentifiableError('60527ec9-b4cb-4a88-a6bd-32d3ad26817d', 'not reacted');
throw new IdentifiableError('60527ec9-b4cb-4a88-a6bd-32d3ad26817d', 'reaction does not exist');
}
// Delete reaction
const result = await this.noteReactionsRepository.delete(exist.id);
if (result.affected !== 1) {
throw new IdentifiableError('60527ec9-b4cb-4a88-a6bd-32d3ad26817d', 'not reacted');
throw new IdentifiableError('60527ec9-b4cb-4a88-a6bd-32d3ad26817d', 'reaction does not exist');
}
// Decrement reactions count

55
packages/backend/src/core/RemoteUserResolveService.ts
View file

@ -3,7 +3,6 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { URL } from 'node:url';
import { Inject, Injectable } from '@nestjs/common';
import chalk from 'chalk';
import { IsNull } from 'typeorm';
@ -18,6 +17,7 @@ import { RemoteLoggerService } from '@/core/RemoteLoggerService.js';
import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js';
import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js';
import { bindThis } from '@/decorators.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
@Injectable()
export class RemoteUserResolveService {
@ -44,27 +44,13 @@ export class RemoteUserResolveService {
const usernameLower = username.toLowerCase();
if (host == null) {
this.logger.info(`return local user: ${usernameLower}`);
return await this.usersRepository.findOneBy({ usernameLower, host: IsNull() }).then(u => {
if (u == null) {
throw new Error('user not found');
} else {
return u;
}
}) as MiLocalUser;
return await this.usersRepository.findOneByOrFail({ usernameLower, host: IsNull() }) as MiLocalUser;
}
host = this.utilityService.toPuny(host);
if (host === this.utilityService.toPuny(this.config.host)) {
this.logger.info(`return local user: ${usernameLower}`);
return await this.usersRepository.findOneBy({ usernameLower, host: IsNull() }).then(u => {
if (u == null) {
throw new Error('user not found');
} else {
return u;
}
}) as MiLocalUser;
return await this.usersRepository.findOneByOrFail({ usernameLower, host: IsNull() }) as MiLocalUser;
}
const user = await this.usersRepository.findOneBy({ usernameLower, host }) as MiRemoteUser | null;
@ -82,7 +68,7 @@ export class RemoteUserResolveService {
.getUserFromApId(self.href)
.then((u) => {
if (u == null) {
throw new Error('local user not found');
throw new Error(`local user not found: ${self.href}`);
} else {
return u;
}
@ -90,7 +76,7 @@ export class RemoteUserResolveService {
}
}
this.logger.succ(`return new remote user: ${chalk.magenta(acctLower)}`);
this.logger.info(`Fetching new remote user ${chalk.magenta(acctLower)} from ${self.href}`);
return await this.apPersonService.createPerson(self.href);
}
@ -101,18 +87,16 @@ export class RemoteUserResolveService {
lastFetchedAt: new Date(),
});
this.logger.info(`try resync: ${acctLower}`);
const self = await this.resolveSelf(acctLower);
if (user.uri !== self.href) {
// if uri mismatch, Fix (user@host <=> AP's Person id(RemoteUser.uri)) mapping.
this.logger.info(`uri missmatch: ${acctLower}`);
this.logger.info(`recovery missmatch uri for (username=${username}, host=${host}) from ${user.uri} to ${self.href}`);
this.logger.warn(`Detected URI mismatch for ${acctLower}`);
// validate uri
const uri = new URL(self.href);
if (uri.hostname !== host) {
throw new Error('Invalid uri');
const uriHost = this.utilityService.extractDbHost(self.href);
if (uriHost !== host) {
throw new Error(`Failed to correct URI for ${acctLower}: new URI ${self.href} has different host from previous URI ${user.uri}`);
}
await this.usersRepository.update({
@ -121,37 +105,28 @@ export class RemoteUserResolveService {
}, {
uri: self.href,
});
} else {
this.logger.info(`uri is fine: ${acctLower}`);
}
this.logger.info(`Corrected URI for ${acctLower} from ${user.uri} to ${self.href}`);
await this.apPersonService.updatePerson(self.href);
this.logger.info(`return resynced remote user: ${acctLower}`);
return await this.usersRepository.findOneBy({ uri: self.href }).then(u => {
if (u == null) {
throw new Error('user not found');
} else {
return u as MiLocalUser | MiRemoteUser;
}
});
return await this.usersRepository.findOneByOrFail({ uri: self.href }) as MiLocalUser | MiRemoteUser;
}
this.logger.info(`return existing remote user: ${acctLower}`);
return user;
}
@bindThis
private async resolveSelf(acctLower: string): Promise<ILink> {
this.logger.info(`WebFinger for ${chalk.yellow(acctLower)}`);
const finger = await this.webfingerService.webfinger(acctLower).catch(err => {
this.logger.error(`Failed to WebFinger for ${chalk.yellow(acctLower)}: ${ err.statusCode ?? err.message }`);
throw new Error(`Failed to WebFinger for ${acctLower}: ${ err.statusCode ?? err.message }`);
this.logger.error(`Failed to WebFinger for ${chalk.yellow(acctLower)}: ${renderInlineError(err)}`);
throw new Error(`Failed to WebFinger for ${acctLower}: error thrown`, { cause: err });
});
const self = finger.links.find(link => link.rel != null && link.rel.toLowerCase() === 'self');
if (!self) {
this.logger.error(`Failed to WebFinger for ${chalk.yellow(acctLower)}: self link not found`);
throw new Error('self link not found');
throw new Error(`Failed to WebFinger for ${acctLower}: self link not found`);
}
return self;
}

4
packages/backend/src/core/ReversiService.ts
View file

@ -587,6 +587,8 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
lastActiveDate: parsed.user1.lastActiveDate != null ? new Date(parsed.user1.lastActiveDate) : null,
lastFetchedAt: parsed.user1.lastFetchedAt != null ? new Date(parsed.user1.lastFetchedAt) : null,
movedAt: parsed.user1.movedAt != null ? new Date(parsed.user1.movedAt) : null,
instance: null,
userProfile: null,
} : null,
user2: parsed.user2 != null ? {
...parsed.user2,
@ -597,6 +599,8 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
lastActiveDate: parsed.user2.lastActiveDate != null ? new Date(parsed.user2.lastActiveDate) : null,
lastFetchedAt: parsed.user2.lastFetchedAt != null ? new Date(parsed.user2.lastFetchedAt) : null,
movedAt: parsed.user2.movedAt != null ? new Date(parsed.user2.movedAt) : null,
instance: null,
userProfile: null,
} : null,
};
} else {

4
packages/backend/src/core/RoleService.ts
View file

@ -86,10 +86,10 @@ export const DEFAULT_POLICIES: RolePolicies = {
canManageCustomEmojis: false,
canManageAvatarDecorations: false,
canSearchNotes: false,
canUseTranslator: true,
canUseTranslator: false,
canHideAds: false,
driveCapacityMb: 100,
maxFileSizeMb: 10,
maxFileSizeMb: 25,
alwaysMarkNsfw: false,
canUpdateBioMedia: true,
pinLimit: 5,

12
packages/backend/src/core/UserBlockingService.ts
View file

@ -77,8 +77,10 @@ export class UserBlockingService implements OnModuleInit {
await this.blockingsRepository.insert(blocking);
this.cacheService.userBlockingCache.refresh(blocker.id);
this.cacheService.userBlockedCache.refresh(blockee.id);
await Promise.all([
this.cacheService.userBlockingCache.delete(blocker.id),
this.cacheService.userBlockedCache.delete(blockee.id),
]);
this.globalEventService.publishInternalEvent('blockingCreated', {
blockerId: blocker.id,
@ -168,8 +170,10 @@ export class UserBlockingService implements OnModuleInit {
await this.blockingsRepository.delete(blocking.id);
this.cacheService.userBlockingCache.refresh(blocker.id);
this.cacheService.userBlockedCache.refresh(blockee.id);
await Promise.all([
this.cacheService.userBlockingCache.delete(blocker.id),
this.cacheService.userBlockedCache.delete(blockee.id),
]);
this.globalEventService.publishInternalEvent('blockingDeleted', {
blockerId: blocker.id,

125
packages/backend/src/core/UserFollowingService.ts
View file

@ -29,6 +29,7 @@ import { AccountMoveService } from '@/core/AccountMoveService.js';
import { UtilityService } from '@/core/UtilityService.js';
import type { ThinUser } from '@/queue/types.js';
import { LoggerService } from '@/core/LoggerService.js';
import { InternalEventService } from '@/core/InternalEventService.js';
import type Logger from '../logger.js';
type Local = MiLocalUser | {
@ -86,6 +87,7 @@ export class UserFollowingService implements OnModuleInit {
private accountMoveService: AccountMoveService,
private perUserFollowingChart: PerUserFollowingChart,
private instanceChart: InstanceChart,
private readonly internalEventService: InternalEventService,
loggerService: LoggerService,
) {
@ -145,12 +147,7 @@ export class UserFollowingService implements OnModuleInit {
if (blocked) throw new IdentifiableError('3338392a-f764-498d-8855-db939dcf8c48', 'blocked');
}
if (await this.followingsRepository.exists({
where: {
followerId: follower.id,
followeeId: followee.id,
},
})) {
if (await this.cacheService.isFollowing(follower, followee)) {
// すでにフォロー関係が存在している場合
if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) {
// リモート → ローカル: acceptを送り返しておしまい
@ -178,24 +175,14 @@ export class UserFollowingService implements OnModuleInit {
let autoAccept = false;
// 鍵アカウントであっても、既にフォローされていた場合はスルー
const isFollowing = await this.followingsRepository.exists({
where: {
followerId: follower.id,
followeeId: followee.id,
},
});
const isFollowing = await this.cacheService.isFollowing(follower, followee);
if (isFollowing) {
autoAccept = true;
}
// フォローしているユーザーは自動承認オプション
if (!autoAccept && (this.userEntityService.isLocalUser(followee) && followeeProfile.autoAcceptFollowed)) {
const isFollowed = await this.followingsRepository.exists({
where: {
followerId: followee.id,
followeeId: follower.id,
},
});
const isFollowed = await this.cacheService.isFollowing(followee, follower); // intentionally reversed parameters
if (isFollowed) autoAccept = true;
}
@ -204,12 +191,7 @@ export class UserFollowingService implements OnModuleInit {
if (followee.isLocked && !autoAccept) {
autoAccept = !!(await this.accountMoveService.validateAlsoKnownAs(
follower,
(oldSrc, newSrc) => this.followingsRepository.exists({
where: {
followeeId: followee.id,
followerId: newSrc.id,
},
}),
(oldSrc, newSrc) => this.cacheService.isFollowing(newSrc, followee),
true,
));
}
@ -264,7 +246,8 @@ export class UserFollowingService implements OnModuleInit {
}
});
this.cacheService.userFollowingsCache.refresh(follower.id);
// Handled by CacheService
//this.cacheService.userFollowingsCache.refresh(follower.id);
const requestExist = await this.followRequestsRepository.exists({
where: {
@ -291,7 +274,7 @@ export class UserFollowingService implements OnModuleInit {
}, followee.id);
}
this.globalEventService.publishInternalEvent('follow', { followerId: follower.id, followeeId: followee.id });
await this.internalEventService.emit('follow', { followerId: follower.id, followeeId: followee.id });
const [followeeUser, followerUser] = await Promise.all([
this.usersRepository.findOneByOrFail({ id: followee.id }),
@ -363,31 +346,29 @@ export class UserFollowingService implements OnModuleInit {
},
silent = false,
): Promise<void> {
const following = await this.followingsRepository.findOne({
relations: {
follower: true,
followee: true,
},
where: {
followerId: follower.id,
followeeId: followee.id,
},
});
const [
followerUser,
followeeUser,
following,
] = await Promise.all([
this.cacheService.findUserById(follower.id),
this.cacheService.findUserById(followee.id),
this.cacheService.userFollowingsCache.fetch(follower.id).then(fs => fs.get(followee.id)),
]);
if (following === null || !following.follower || !following.followee) {
if (following == null) {
this.logger.warn('フォロー解除がリクエストされましたがフォローしていませんでした');
return;
}
await this.followingsRepository.delete(following.id);
await this.internalEventService.emit('unfollow', { followerId: follower.id, followeeId: followee.id });
this.cacheService.userFollowingsCache.refresh(follower.id);
this.decrementFollowing(following.follower, following.followee);
this.decrementFollowing(followerUser, followeeUser);
if (!silent && this.userEntityService.isLocalUser(follower)) {
// Publish unfollow event
this.userEntityService.pack(followee.id, follower, {
this.userEntityService.pack(followeeUser, follower, {
schema: 'UserDetailedNotMe',
}).then(async packed => {
this.globalEventService.publishMainStream(follower.id, 'unfollow', packed);
@ -412,8 +393,6 @@ export class UserFollowingService implements OnModuleInit {
follower: MiUser,
followee: MiUser,
): Promise<void> {
this.globalEventService.publishInternalEvent('unfollow', { followerId: follower.id, followeeId: followee.id });
// Neither followee nor follower has moved.
if (!follower.movedToUri && !followee.movedToUri) {
//#region Decrement following / followers counts
@ -687,22 +666,22 @@ export class UserFollowingService implements OnModuleInit {
*/
@bindThis
private async removeFollow(followee: Both, follower: Both): Promise<void> {
const following = await this.followingsRepository.findOne({
relations: {
followee: true,
follower: true,
},
where: {
followeeId: followee.id,
followerId: follower.id,
},
});
const [
followerUser,
followeeUser,
following,
] = await Promise.all([
this.cacheService.findUserById(follower.id),
this.cacheService.findUserById(followee.id),
this.cacheService.userFollowingsCache.fetch(follower.id).then(fs => fs.get(followee.id)),
]);
if (!following || !following.followee || !following.follower) return;
if (!following) return;
await this.followingsRepository.delete(following.id);
await this.internalEventService.emit('unfollow', { followerId: follower.id, followeeId: followee.id });
this.decrementFollowing(following.follower, following.followee);
this.decrementFollowing(followerUser, followeeUser);
}
/**
@ -733,36 +712,26 @@ export class UserFollowingService implements OnModuleInit {
}
@bindThis
public getFollowees(userId: MiUser['id']) {
return this.followingsRepository.createQueryBuilder('following')
.select('following.followeeId')
.where('following.followerId = :followerId', { followerId: userId })
.getMany();
public async getFollowees(userId: MiUser['id']) {
const followings = await this.cacheService.userFollowingsCache.fetch(userId);
return Array.from(followings.values());
}
@bindThis
public isFollowing(followerId: MiUser['id'], followeeId: MiUser['id']) {
return this.followingsRepository.exists({
where: {
followerId,
followeeId,
},
});
public async isFollowing(followerId: MiUser['id'], followeeId: MiUser['id']) {
return this.cacheService.isFollowing(followerId, followeeId);
}
@bindThis
public async isMutual(aUserId: MiUser['id'], bUserId: MiUser['id']) {
const count = await this.followingsRepository.createQueryBuilder('following')
.where(new Brackets(qb => {
qb.where('following.followerId = :aUserId', { aUserId })
.andWhere('following.followeeId = :bUserId', { bUserId });
}))
.orWhere(new Brackets(qb => {
qb.where('following.followerId = :bUserId', { bUserId })
.andWhere('following.followeeId = :aUserId', { aUserId });
}))
.getCount();
const [
isFollowing,
isFollowed,
] = await Promise.all([
this.isFollowing(aUserId, bUserId),
this.isFollowing(bUserId, aUserId),
]);
return count === 2;
return isFollowing && isFollowed;
}
}

14
packages/backend/src/core/UserKeypairService.ts
View file

@ -7,14 +7,14 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import * as Redis from 'ioredis';
import type { MiUser } from '@/models/User.js';
import type { UserKeypairsRepository } from '@/models/_.js';
import { RedisKVCache } from '@/misc/cache.js';
import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js';
import type { MiUserKeypair } from '@/models/UserKeypair.js';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
@Injectable()
export class UserKeypairService implements OnApplicationShutdown {
private cache: RedisKVCache<MiUserKeypair>;
private cache: MemoryKVCache<MiUserKeypair>;
constructor(
@Inject(DI.redis)
@ -23,18 +23,12 @@ export class UserKeypairService implements OnApplicationShutdown {
@Inject(DI.userKeypairsRepository)
private userKeypairsRepository: UserKeypairsRepository,
) {
this.cache = new RedisKVCache<MiUserKeypair>(this.redisClient, 'userKeypair', {
lifetime: 1000 * 60 * 60 * 24, // 24h
memoryCacheLifetime: 1000 * 60 * 60, // 1h
fetcher: (key) => this.userKeypairsRepository.findOneByOrFail({ userId: key }),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => JSON.parse(value),
});
this.cache = new MemoryKVCache<MiUserKeypair>(1000 * 60 * 60 * 24); // 24h
}
@bindThis
public async getUserKeypair(userId: MiUser['id']): Promise<MiUserKeypair> {
return await this.cache.fetch(userId);
return await this.cache.fetch(userId, () => this.userKeypairsRepository.findOneByOrFail({ userId }));
}
@bindThis

30
packages/backend/src/core/UserListService.ts
View file

@ -11,21 +11,22 @@ import type { MiUser } from '@/models/User.js';
import type { MiUserList } from '@/models/UserList.js';
import type { MiUserListMembership } from '@/models/UserListMembership.js';
import { IdService } from '@/core/IdService.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import type { GlobalEvents, InternalEventTypes } from '@/core/GlobalEventService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { DI } from '@/di-symbols.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { bindThis } from '@/decorators.js';
import { QueueService } from '@/core/QueueService.js';
import { RedisKVCache } from '@/misc/cache.js';
import { QuantumKVCache } from '@/misc/QuantumKVCache.js';
import { RoleService } from '@/core/RoleService.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { InternalEventService } from '@/core/InternalEventService.js';
@Injectable()
export class UserListService implements OnApplicationShutdown, OnModuleInit {
public static TooManyUsersError = class extends Error {};
public membersCache: RedisKVCache<Set<string>>;
public membersCache: QuantumKVCache<Set<string>>;
private roleService: RoleService;
constructor(
@ -48,16 +49,15 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit {
private globalEventService: GlobalEventService,
private queueService: QueueService,
private systemAccountService: SystemAccountService,
private readonly internalEventService: InternalEventService,
) {
this.membersCache = new RedisKVCache<Set<string>>(this.redisClient, 'userListMembers', {
this.membersCache = new QuantumKVCache<Set<string>>(this.internalEventService, 'userListMembers', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60, // 1m
fetcher: (key) => this.userListMembershipsRepository.find({ where: { userListId: key }, select: ['userId'] }).then(xs => new Set(xs.map(x => x.userId))),
toRedisConverter: (value) => JSON.stringify(Array.from(value)),
fromRedisConverter: (value) => new Set(JSON.parse(value)),
});
this.redisForSub.on('message', this.onMessage);
this.internalEventService.on('userListMemberAdded', this.onMessage);
this.internalEventService.on('userListMemberRemoved', this.onMessage);
}
async onModuleInit() {
@ -65,15 +65,12 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit {
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
private async onMessage<E extends 'userListMemberAdded' | 'userListMemberRemoved'>(body: InternalEventTypes[E], type: E): Promise<void> {
{
switch (type) {
case 'userListMemberAdded': {
const { userListId, memberId } = body;
const members = await this.membersCache.get(userListId);
const members = this.membersCache.get(userListId);
if (members) {
members.add(memberId);
}
@ -81,7 +78,7 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit {
}
case 'userListMemberRemoved': {
const { userListId, memberId } = body;
const members = await this.membersCache.get(userListId);
const members = this.membersCache.get(userListId);
if (members) {
members.delete(memberId);
}
@ -150,7 +147,8 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit {
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.internalEventService.off('userListMemberAdded', this.onMessage);
this.internalEventService.off('userListMemberRemoved', this.onMessage);
this.membersCache.dispose();
}

7
packages/backend/src/core/UserMutingService.ts
View file

@ -32,7 +32,7 @@ export class UserMutingService {
muteeId: target.id,
});
this.cacheService.userMutingsCache.refresh(user.id);
await this.cacheService.userMutingsCache.delete(user.id);
}
@bindThis
@ -43,9 +43,6 @@ export class UserMutingService {
id: In(mutings.map(m => m.id)),
});
const muterIds = [...new Set(mutings.map(m => m.muterId))];
for (const muterId of muterIds) {
this.cacheService.userMutingsCache.refresh(muterId);
}
await this.cacheService.userMutingsCache.deleteMany(mutings.map(m => m.muterId));
}
}

7
packages/backend/src/core/UserRenoteMutingService.ts
View file

@ -33,7 +33,7 @@ export class UserRenoteMutingService {
muteeId: target.id,
});
await this.cacheService.renoteMutingsCache.refresh(user.id);
await this.cacheService.renoteMutingsCache.delete(user.id);
}
@bindThis
@ -44,9 +44,6 @@ export class UserRenoteMutingService {
id: In(mutings.map(m => m.id)),
});
const muterIds = [...new Set(mutings.map(m => m.muterId))];
for (const muterId of muterIds) {
await this.cacheService.renoteMutingsCache.refresh(muterId);
}
await this.cacheService.renoteMutingsCache.deleteMany(mutings.map(m => m.muterId));
}
}

21
packages/backend/src/core/UserService.ts
View file

@ -10,6 +10,7 @@ import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { SystemWebhookService } from '@/core/SystemWebhookService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { CacheService } from '@/core/CacheService.js';
@Injectable()
export class UserService {
@ -20,6 +21,7 @@ export class UserService {
private followingsRepository: FollowingsRepository,
private systemWebhookService: SystemWebhookService,
private userEntityService: UserEntityService,
private readonly cacheService: CacheService,
) {
}
@ -38,14 +40,17 @@ export class UserService {
});
const wokeUp = result.isHibernated;
if (wokeUp) {
this.usersRepository.update(user.id, {
isHibernated: false,
});
this.followingsRepository.update({
followerId: user.id,
}, {
isFollowerHibernated: false,
});
await Promise.all([
this.usersRepository.update(user.id, {
isHibernated: false,
}),
this.followingsRepository.update({
followerId: user.id,
}, {
isFollowerHibernated: false,
}),
this.cacheService.hibernatedUserCache.set(user.id, false),
]);
}
} else {
this.usersRepository.update(user.id, {

94
packages/backend/src/core/UserSuspendService.ts
View file

@ -6,7 +6,7 @@
import { Inject, Injectable } from '@nestjs/common';
import { Not, IsNull } from 'typeorm';
import type { FollowingsRepository, FollowRequestsRepository, UsersRepository } from '@/models/_.js';
import type { MiUser } from '@/models/User.js';
import { MiUser } from '@/models/User.js';
import { QueueService } from '@/core/QueueService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { DI } from '@/di-symbols.js';
@ -16,9 +16,16 @@ import { bindThis } from '@/decorators.js';
import { RelationshipJobData } from '@/queue/types.js';
import { ModerationLogService } from '@/core/ModerationLogService.js';
import { isSystemAccount } from '@/misc/is-system-account.js';
import { CacheService } from '@/core/CacheService.js';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import { trackPromise } from '@/misc/promise-tracker.js';
@Injectable()
export class UserSuspendService {
private readonly logger: Logger;
constructor(
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
@ -34,7 +41,11 @@ export class UserSuspendService {
private globalEventService: GlobalEventService,
private apRendererService: ApRendererService,
private moderationLogService: ModerationLogService,
private readonly cacheService: CacheService,
loggerService: LoggerService,
) {
this.logger = loggerService.getLogger('user-suspend');
}
@bindThis
@ -45,16 +56,16 @@ export class UserSuspendService {
isSuspended: true,
});
this.moderationLogService.log(moderator, 'suspend', {
await this.moderationLogService.log(moderator, 'suspend', {
userId: user.id,
userUsername: user.username,
userHost: user.host,
});
(async () => {
await this.postSuspend(user).catch(e => {});
await this.unFollowAll(user).catch(e => {});
})();
trackPromise((async () => {
await this.postSuspend(user);
await this.freezeAll(user);
})().catch(e => this.logger.error(`Error suspending user ${user.id}: ${renderInlineError(e)}`)));
}
@bindThis
@ -63,33 +74,36 @@ export class UserSuspendService {
isSuspended: false,
});
this.moderationLogService.log(moderator, 'unsuspend', {
await this.moderationLogService.log(moderator, 'unsuspend', {
userId: user.id,
userUsername: user.username,
userHost: user.host,
});
(async () => {
await this.postUnsuspend(user).catch(e => {});
})();
trackPromise((async () => {
await this.postUnsuspend(user);
await this.unFreezeAll(user);
})().catch(e => this.logger.error(`Error un-suspending for user ${user.id}: ${renderInlineError(e)}`)));
}
@bindThis
private async postSuspend(user: { id: MiUser['id']; host: MiUser['host'] }): Promise<void> {
this.globalEventService.publishInternalEvent('userChangeSuspendedState', { id: user.id, isSuspended: true });
/*
this.followRequestsRepository.delete({
followeeId: user.id,
});
this.followRequestsRepository.delete({
followerId: user.id,
});
*/
if (this.userEntityService.isLocalUser(user)) {
// 知り得る全SharedInboxにDelete配信
const content = this.apRendererService.addContext(this.apRendererService.renderDelete(this.userEntityService.genLocalUserUri(user.id), user));
const queue: string[] = [];
const queue = new Map<string, boolean>();
const followings = await this.followingsRepository.find({
where: [
@ -102,12 +116,12 @@ export class UserSuspendService {
const inboxes = followings.map(x => x.followerSharedInbox ?? x.followeeSharedInbox);
for (const inbox of inboxes) {
if (inbox != null && !queue.includes(inbox)) queue.push(inbox);
if (inbox != null) {
queue.set(inbox, true);
}
}
for (const inbox of queue) {
this.queueService.deliver(user, content, inbox, true);
}
await this.queueService.deliverMany(user, content, queue);
}
}
@ -119,7 +133,7 @@ export class UserSuspendService {
// 知り得る全SharedInboxにUndo Delete配信
const content = this.apRendererService.addContext(this.apRendererService.renderUndo(this.apRendererService.renderDelete(this.userEntityService.genLocalUserUri(user.id), user), user));
const queue: string[] = [];
const queue = new Map<string, boolean>();
const followings = await this.followingsRepository.find({
where: [
@ -132,23 +146,19 @@ export class UserSuspendService {
const inboxes = followings.map(x => x.followerSharedInbox ?? x.followeeSharedInbox);
for (const inbox of inboxes) {
if (inbox != null && !queue.includes(inbox)) queue.push(inbox);
if (inbox != null) {
queue.set(inbox, true);
}
}
for (const inbox of queue) {
this.queueService.deliver(user as any, content, inbox, true);
}
await this.queueService.deliverMany(user, content, queue);
}
}
@bindThis
private async unFollowAll(follower: MiUser) {
const followings = await this.followingsRepository.find({
where: {
followerId: follower.id,
followeeId: Not(IsNull()),
},
});
const followings = await this.cacheService.userFollowingsCache.fetch(follower.id)
.then(fs => Array.from(fs.values()).filter(f => f.followeeHost != null));
const jobs: RelationshipJobData[] = [];
for (const following of followings) {
@ -162,4 +172,36 @@ export class UserSuspendService {
}
this.queueService.createUnfollowJob(jobs);
}
@bindThis
private async freezeAll(user: MiUser): Promise<void> {
// Freeze follow relations with all remote users
await this.followingsRepository
.createQueryBuilder('following')
.orWhere({
followeeId: user.id,
followerHost: Not(IsNull()),
})
.update({
isFollowerHibernated: true,
})
.execute();
}
@bindThis
private async unFreezeAll(user: MiUser): Promise<void> {
// Restore follow relations with all remote users
await this.followingsRepository
.createQueryBuilder('following')
.innerJoin(MiUser, 'follower', 'user.id = following.followerId')
.andWhere('follower.isHibernated = false') // Don't unfreeze if the follower is *actually* frozen
.andWhere({
followeeId: user.id,
followerHost: Not(IsNull()),
})
.update({
isFollowerHibernated: false,
})
.execute();
}
}

39
packages/backend/src/core/UtilityService.ts
View file

@ -49,22 +49,49 @@ export class UtilityService {
return regexp.test(email);
}
public isBlockedHost(host: string | null): boolean;
public isBlockedHost(blockedHosts: string[], host: string | null): boolean;
@bindThis
public isBlockedHost(blockedHosts: string[], host: string | null): boolean {
public isBlockedHost(blockedHostsOrHost: string[] | string | null, host?: string | null): boolean {
const blockedHosts = Array.isArray(blockedHostsOrHost) ? blockedHostsOrHost : this.meta.blockedHosts;
host = Array.isArray(blockedHostsOrHost) ? host : blockedHostsOrHost;
if (host == null) return false;
return blockedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
public isSilencedHost(host: string | null): boolean;
public isSilencedHost(silencedHosts: string[], host: string | null): boolean;
@bindThis
public isSilencedHost(silencedHosts: string[] | undefined, host: string | null): boolean {
if (!silencedHosts || host == null) return false;
public isSilencedHost(silencedHostsOrHost: string[] | string | null, host?: string | null): boolean {
const silencedHosts = Array.isArray(silencedHostsOrHost) ? silencedHostsOrHost : this.meta.silencedHosts;
host = Array.isArray(silencedHostsOrHost) ? host : silencedHostsOrHost;
if (host == null) return false;
return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
public isMediaSilencedHost(host: string | null): boolean;
public isMediaSilencedHost(silencedHosts: string[], host: string | null): boolean;
@bindThis
public isMediaSilencedHost(silencedHosts: string[] | undefined, host: string | null): boolean {
if (!silencedHosts || host == null) return false;
return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
public isMediaSilencedHost(mediaSilencedHostsOrHost: string[] | string | null, host?: string | null): boolean {
const mediaSilencedHosts = Array.isArray(mediaSilencedHostsOrHost) ? mediaSilencedHostsOrHost : this.meta.mediaSilencedHosts;
host = Array.isArray(mediaSilencedHostsOrHost) ? host : mediaSilencedHostsOrHost;
if (host == null) return false;
return mediaSilencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
@bindThis
public isAllowListedHost(host: string | null): boolean {
if (host == null) return false;
return this.meta.federationHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
@bindThis
public isBubbledHost(host: string | null): boolean {
if (host == null) return false;
return this.meta.bubbleInstances.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
@bindThis

64
packages/backend/src/core/VideoProcessingService.ts
View file

@ -3,24 +3,41 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import fs from 'node:fs/promises';
import { Inject, Injectable } from '@nestjs/common';
import FFmpeg from 'fluent-ffmpeg';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { ImageProcessingService } from '@/core/ImageProcessingService.js';
import type { IImage } from '@/core/ImageProcessingService.js';
import { createTempDir } from '@/misc/create-temp.js';
import { createTemp, createTempDir } from '@/misc/create-temp.js';
import { bindThis } from '@/decorators.js';
import { appendQuery, query } from '@/misc/prelude/url.js';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
// faststart is only supported for MP4, M4A, M4W and MOV files (the MOV family).
// WebM (and Matroska) files always support faststart-like behavior.
const supportedMimeTypes = new Map([
['video/mp4', 'mp4'],
['video/m4a', 'mp4'],
['video/m4v', 'mp4'],
['video/quicktime', 'mov'],
]);
@Injectable()
export class VideoProcessingService {
private readonly logger: Logger;
constructor(
@Inject(DI.config)
private config: Config,
private imageProcessingService: ImageProcessingService,
private loggerService: LoggerService,
) {
this.logger = this.loggerService.getLogger('video-processing');
}
@bindThis
@ -60,5 +77,50 @@ export class VideoProcessingService {
}),
);
}
/**
* Optimize video for web playback by adding faststart flag.
* This allows the video to start playing before it is fully downloaded.
* The original file is modified in-place.
* @param source Path to the video file
* @param mimeType The MIME type of the video
* @returns Promise that resolves when optimization is complete
*/
@bindThis
public async webOptimizeVideo(source: string, mimeType: string): Promise<void> {
const outputFormat = supportedMimeTypes.get(mimeType);
if (!outputFormat) {
this.logger.debug(`Skipping web optimization for unsupported MIME type: ${mimeType}`);
return;
}
const [tempPath, cleanup] = await createTemp();
try {
await new Promise<void>((resolve, reject) => {
FFmpeg(source)
.format(outputFormat) // Specify output format
.addOutputOptions('-c copy') // Copy streams without re-encoding
.addOutputOptions('-movflags +faststart')
.on('error', reject)
.on('end', async () => {
try {
// Replace original file with optimized version
await fs.copyFile(tempPath, source);
this.logger.info(`Web-optimized video: ${source}`);
resolve();
} catch (copyError) {
reject(copyError);
}
})
.save(tempPath);
});
} catch (error) {
this.logger.warn(`Failed to web-optimize video: ${source}`, { error });
throw error;
} finally {
cleanup();
}
}
}

16
packages/backend/src/core/WebAuthnService.ts
View file

@ -17,6 +17,8 @@ import type { Config } from '@/config.js';
import { bindThis } from '@/decorators.js';
import { MiUser } from '@/models/_.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { LoggerService } from '@/core/LoggerService.js';
import Logger from '@/logger.js';
import type {
AuthenticationResponseJSON,
AuthenticatorTransportFuture,
@ -28,6 +30,8 @@ import type {
@Injectable()
export class WebAuthnService {
private readonly logger: Logger;
constructor(
@Inject(DI.config)
private config: Config,
@ -40,7 +44,9 @@ export class WebAuthnService {
@Inject(DI.userSecurityKeysRepository)
private userSecurityKeysRepository: UserSecurityKeysRepository,
loggerService: LoggerService,
) {
this.logger = loggerService.getLogger('web-authn');
}
@bindThis
@ -114,8 +120,8 @@ export class WebAuthnService {
requireUserVerification: true,
});
} catch (error) {
console.error(error);
throw new IdentifiableError('5c1446f8-8ca7-4d31-9f39-656afe9c5d87', 'verification failed');
this.logger.error(error as Error, 'Error authenticating webauthn');
throw new IdentifiableError('5c1446f8-8ca7-4d31-9f39-656afe9c5d87', 'verification failed', true, error);
}
const { verified } = verification;
@ -221,7 +227,7 @@ export class WebAuthnService {
requireUserVerification: true,
});
} catch (error) {
throw new IdentifiableError('b18c89a7-5b5e-4cec-bb5b-0419f332d430', `verification failed: ${error}`);
throw new IdentifiableError('b18c89a7-5b5e-4cec-bb5b-0419f332d430', `verification failed`, true, error);
}
const { verified, authenticationInfo } = verification;
@ -301,8 +307,8 @@ export class WebAuthnService {
requireUserVerification: true,
});
} catch (error) {
console.error(error);
throw new IdentifiableError('b18c89a7-5b5e-4cec-bb5b-0419f332d430', 'verification failed');
this.logger.error(error as Error, 'Error authenticating webauthn');
throw new IdentifiableError('b18c89a7-5b5e-4cec-bb5b-0419f332d430', 'verification failed', true, error);
}
const { verified, authenticationInfo } = verification;

19
packages/backend/src/core/WebfingerService.ts
View file

@ -5,10 +5,11 @@
import { URL } from 'node:url';
import { Injectable } from '@nestjs/common';
import { XMLParser } from 'fast-xml-parser';
import { load as cheerio } from 'cheerio/slim';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
import type Logger from '@/logger.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import { RemoteLoggerService } from './RemoteLoggerService.js';
export type ILink = {
@ -100,16 +101,14 @@ export class WebfingerService {
private async fetchWebFingerTemplateFromHostMeta(url: string): Promise<string | null> {
try {
const res = await this.httpRequestService.getHtml(url, 'application/xrd+xml');
const options = {
ignoreAttributes: false,
isArray: (_name: string, jpath: string) => jpath === 'XRD.Link',
};
const parser = new XMLParser(options);
const hostMeta = parser.parse(res);
const template = (hostMeta['XRD']['Link'] as Array<any>).filter(p => p['@_rel'] === 'lrdd')[0]['@_template'];
return template.indexOf('{uri}') < 0 ? null : template;
const hostMeta = cheerio(res, {
xml: true,
});
const template = hostMeta('XRD > Link[rel="lrdd"][template*="{uri}"]').attr('template');
return template ?? null;
} catch (err) {
this.logger.error(`error while request host-meta for ${url}: ${err}`);
this.logger.error(`error while request host-meta for ${url}: ${renderInlineError(err)}`);
return null;
}
}

13
packages/backend/src/core/WebhookTestService.ts
View file

@ -13,6 +13,7 @@ import { type WebhookEventTypes } from '@/models/Webhook.js';
import { CustomEmojiService } from '@/core/CustomEmojiService.js';
import { type UserWebhookPayload, UserWebhookService } from '@/core/UserWebhookService.js';
import { QueueService } from '@/core/QueueService.js';
import { IdService } from '@/core/IdService.js';
import { ModeratorInactivityRemainingTime } from '@/queue/processors/CheckModeratorsActivityProcessorService.js';
const oneDayMillis = 24 * 60 * 60 * 1000;
@ -63,6 +64,7 @@ function generateDummyUser(override?: Partial<MiUser>): MiUser {
emojis: [],
score: 0,
host: null,
instance: null,
inbox: null,
sharedInbox: null,
featured: null,
@ -76,6 +78,8 @@ function generateDummyUser(override?: Partial<MiUser>): MiUser {
mandatoryCW: null,
rejectQuotes: false,
allowUnsignedFetch: 'staff',
userProfile: null,
attributionDomains: [],
...override,
};
}
@ -114,10 +118,13 @@ function generateDummyNote(override?: Partial<MiNote>): MiNote {
channelId: null,
channel: null,
userHost: null,
userInstance: null,
replyUserId: null,
replyUserHost: null,
replyUserInstance: null,
renoteUserId: null,
renoteUserHost: null,
renoteUserInstance: null,
updatedAt: null,
processErrors: [],
...override,
@ -160,6 +167,7 @@ export class WebhookTestService {
private userWebhookService: UserWebhookService,
private systemWebhookService: SystemWebhookService,
private queueService: QueueService,
private readonly idService: IdService,
) {
}
@ -358,8 +366,10 @@ export class WebhookTestService {
id: 'dummy-abuse-report1',
targetUserId: 'dummy-target-user',
targetUser: null,
targetUserInstance: null,
reporterId: 'dummy-reporter-user',
reporter: null,
reporterInstance: null,
assigneeId: null,
assignee: null,
resolved: false,
@ -441,6 +451,8 @@ export class WebhookTestService {
offsetX: it.offsetX,
offsetY: it.offsetY,
})),
createdAt: this.idService.parse(user.id).date.toISOString(),
description: '',
isBot: user.isBot,
isCat: user.isCat,
emojis: await this.customEmojiService.populateEmojis(user.emojis, user.host),
@ -449,6 +461,7 @@ export class WebhookTestService {
isAdmin: false,
isModerator: false,
isSystem: false,
instance: undefined,
...override,
};
}

19
packages/backend/src/core/activitypub/ApDbResolverService.ts
View file

@ -165,18 +165,23 @@ export class ApDbResolverService implements OnApplicationShutdown {
*/
@bindThis
public async refetchPublicKeyForApId(user: MiRemoteUser): Promise<MiUserPublickey | null> {
this.apLoggerService.logger.debug('Re-fetching public key for user', { userId: user.id, uri: user.uri });
this.apLoggerService.logger.debug(`Updating public key for user ${user.id} (${user.uri})`);
const oldKey = await this.apPersonService.findPublicKeyByUserId(user.id);
await this.apPersonService.updatePerson(user.uri);
const newKey = await this.apPersonService.findPublicKeyByUserId(user.id);
const key = await this.apPersonService.findPublicKeyByUserId(user.id);
if (key) {
this.apLoggerService.logger.info('Re-fetched public key for user', { userId: user.id, uri: user.uri });
if (newKey) {
if (oldKey && newKey.keyPem === oldKey.keyPem) {
this.apLoggerService.logger.debug(`Public key is up-to-date for user ${user.id} (${user.uri})`);
} else {
this.apLoggerService.logger.info(`Updated public key for user ${user.id} (${user.uri})`);
}
} else {
this.apLoggerService.logger.warn('Failed to re-fetch key for user', { userId: user.id, uri: user.uri });
this.apLoggerService.logger.warn(`Failed to update public key for user ${user.id} (${user.uri})`);
}
return key;
return newKey ?? oldKey;
}
@bindThis

55
packages/backend/src/core/activitypub/ApDeliverManagerService.ts
View file

@ -5,7 +5,6 @@
import { Inject, Injectable } from '@nestjs/common';
import { IsNull, Not } from 'typeorm';
import { UnrecoverableError } from 'bullmq';
import { DI } from '@/di-symbols.js';
import type { FollowingsRepository } from '@/models/_.js';
import type { MiLocalUser, MiRemoteUser, MiUser } from '@/models/User.js';
@ -14,6 +13,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { bindThis } from '@/decorators.js';
import type { IActivity } from '@/core/activitypub/type.js';
import { ThinUser } from '@/queue/types.js';
import { CacheService } from '@/core/CacheService.js';
interface IRecipe {
type: string;
@ -41,23 +41,21 @@ class DeliverManager {
/**
* Constructor
* @param userEntityService
* @param followingsRepository
* @param queueService
* @param cacheService
* @param actor Actor
* @param activity Activity to deliver
*/
constructor(
private userEntityService: UserEntityService,
private followingsRepository: FollowingsRepository,
private queueService: QueueService,
private readonly cacheService: CacheService,
actor: { id: MiUser['id']; host: null; },
activity: IActivity | null,
) {
// 型で弾いてはいるが一応ローカルユーザーかチェック
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (actor.host != null) throw new Error('actor.host must be null');
if (actor.host != null) throw new Error(`deliver failed for ${actor.id}: host is not null`);
// パフォーマンス向上のためキューに突っ込むのはidのみに絞る
this.actor = {
@ -114,23 +112,23 @@ class DeliverManager {
// Process follower recipes first to avoid duplication when processing direct recipes later.
if (this.recipes.some(r => isFollowers(r))) {
// followers deliver
// TODO: SELECT DISTINCT ON ("followerSharedInbox") "followerSharedInbox" みたいな問い合わせにすればよりパフォーマンス向上できそう
// ただ、sharedInboxがnullなリモートユーザーも稀におり、その対応ができなさそう？
const followers = await this.followingsRepository.find({
where: {
followeeId: this.actor.id,
followerHost: Not(IsNull()),
},
select: {
followerSharedInbox: true,
followerInbox: true,
},
});
const followers = await this.cacheService.userFollowersCache
.fetch(this.actor.id)
.then(f => Array
.from(f.values())
.filter(f => f.followerHost != null)
.map(f => ({
followerInbox: f.followerInbox,
followerSharedInbox: f.followerSharedInbox,
})));
for (const following of followers) {
const inbox = following.followerSharedInbox ?? following.followerInbox;
if (inbox === null) throw new UnrecoverableError(`inbox is null: following ${following.id}`);
inboxes.set(inbox, following.followerSharedInbox != null);
if (following.followerSharedInbox) {
inboxes.set(following.followerSharedInbox, true);
} else if (following.followerInbox) {
inboxes.set(following.followerInbox, false);
}
}
}
@ -152,11 +150,8 @@ class DeliverManager {
@Injectable()
export class ApDeliverManagerService {
constructor(
@Inject(DI.followingsRepository)
private followingsRepository: FollowingsRepository,
private userEntityService: UserEntityService,
private queueService: QueueService,
private readonly cacheService: CacheService,
) {
}
@ -168,9 +163,8 @@ export class ApDeliverManagerService {
@bindThis
public async deliverToFollowers(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity): Promise<void> {
const manager = new DeliverManager(
this.userEntityService,
this.followingsRepository,
this.queueService,
this.cacheService,
actor,
activity,
);
@ -187,9 +181,8 @@ export class ApDeliverManagerService {
@bindThis
public async deliverToUser(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity, to: MiRemoteUser): Promise<void> {
const manager = new DeliverManager(
this.userEntityService,
this.followingsRepository,
this.queueService,
this.cacheService,
actor,
activity,
);
@ -206,9 +199,8 @@ export class ApDeliverManagerService {
@bindThis
public async deliverToUsers(actor: { id: MiLocalUser['id']; host: null; }, activity: IActivity, targets: MiRemoteUser[]): Promise<void> {
const manager = new DeliverManager(
this.userEntityService,
this.followingsRepository,
this.queueService,
this.cacheService,
actor,
activity,
);
@ -219,9 +211,8 @@ export class ApDeliverManagerService {
@bindThis
public createDeliverManager(actor: { id: MiUser['id']; host: null; }, activity: IActivity | null): DeliverManager {
return new DeliverManager(
this.userEntityService,
this.followingsRepository,
this.queueService,
this.cacheService,
actor,
activity,

106
packages/backend/src/core/activitypub/ApInboxService.ts
View file

@ -32,11 +32,13 @@ import { AbuseReportService } from '@/core/AbuseReportService.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import { fromTuple } from '@/misc/from-tuple.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import InstanceChart from '@/core/chart/charts/instance.js';
import FederationChart from '@/core/chart/charts/federation.js';
import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js';
import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js';
import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js';
import { CacheService } from '@/core/CacheService.js';
import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js';
import { ApNoteService } from './models/ApNoteService.js';
import { ApLoggerService } from './ApLoggerService.js';
import { ApDbResolverService } from './ApDbResolverService.js';
@ -97,6 +99,7 @@ export class ApInboxService {
private readonly instanceChart: InstanceChart,
private readonly federationChart: FederationChart,
private readonly updateInstanceQueue: UpdateInstanceQueue,
private readonly cacheService: CacheService,
) {
this.logger = this.apLoggerService.logger;
}
@ -106,25 +109,29 @@ export class ApInboxService {
let result = undefined as string | void;
if (isCollectionOrOrderedCollection(activity)) {
const results = [] as [string, string | void][];
// eslint-disable-next-line no-param-reassign
resolver ??= this.apResolverService.createResolver();
const items = toArray(isCollection(activity) ? activity.items : activity.orderedItems);
if (items.length >= resolver.getRecursionLimit()) {
throw new Error(`skipping activity: collection would surpass recursion limit: ${this.utilityService.extractDbHost(actor.uri)}`);
}
for (const item of items) {
const act = await resolver.resolve(item);
if (act.id == null || this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) {
this.logger.debug('skipping activity: activity id is null or mismatching');
continue;
const items = await resolver.resolveCollectionItems(activity);
for (let i = 0; i < items.length; i++) {
const act = items[i];
if (act.id != null) {
if (this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) {
this.logger.warn('skipping activity: activity id mismatch');
continue;
}
} else {
// Activity ID should only be string or undefined.
act.id = undefined;
}
const id = getNullableApId(act) ?? `${getNullableApId(activity)}#${i}`;
try {
results.push([getApId(item), await this.performOneActivity(actor, act, resolver)]);
const result = await this.performOneActivity(actor, act, resolver);
results.push([id, result]);
} catch (err) {
if (err instanceof Error || typeof err === 'string') {
this.logger.error(err);
this.logger.error(`Unhandled error in activity ${id}:`, err);
} else {
throw err;
}
@ -144,7 +151,8 @@ export class ApInboxService {
if (actor.lastFetchedAt == null || Date.now() - actor.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) {
setImmediate(() => {
// 同一ユーザーの情報を再度処理するので、使用済みのresolverを再利用してはいけない
this.apPersonService.updatePerson(actor.uri);
this.apPersonService.updatePerson(actor.uri)
.catch(err => this.logger.error(`Failed to update person: ${renderInlineError(err)}`));
});
}
}
@ -217,6 +225,10 @@ export class ApInboxService {
const note = await this.apNoteService.resolveNote(object, { resolver });
if (!note) return `skip: target note not found ${targetUri}`;
if (note.userHost == null && note.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot react to local-only note');
}
await this.apNoteService.extractEmojis(activity.tag ?? [], actor.host).catch(() => null);
try {
@ -246,7 +258,7 @@ export class ApInboxService {
resolver ??= this.apResolverService.createResolver();
const object = await resolver.resolve(activity.object).catch(err => {
this.logger.error(`Resolution failed: ${err}`);
this.logger.error(`Resolution failed: ${renderInlineError(err)}`);
throw err;
});
@ -319,7 +331,7 @@ export class ApInboxService {
if (targetUri.startsWith('bear:')) return 'skip: bearcaps url not supported.';
const target = await resolver.secureResolve(activityObject, uri).catch(e => {
this.logger.error(`Resolution failed: ${e}`);
this.logger.error(`Resolution failed: ${renderInlineError(e)}`);
throw e;
});
@ -350,25 +362,17 @@ export class ApInboxService {
}
// Announce対象をresolve
let renote;
try {
// The target ID is verified by secureResolve, so we know it shares host authority with the actor who sent it.
// This means we can pass that ID to resolveNote and avoid an extra fetch, which will fail if the note is private.
renote = await this.apNoteService.resolveNote(target, { resolver, sentFrom: getApId(target) });
if (renote == null) return 'announce target is null';
} catch (err) {
// 対象が4xxならスキップ
if (err instanceof StatusError) {
if (!err.isRetryable) {
return `skip: ignored announce target ${target.id} - ${err.statusCode}`;
}
return `Error in announce target ${target.id} - ${err.statusCode}`;
}
throw err;
// The target ID is verified by secureResolve, so we know it shares host authority with the actor who sent it.
// This means we can pass that ID to resolveNote and avoid an extra fetch, which will fail if the note is private.
const renote = await this.apNoteService.resolveNote(target, { resolver, sentFrom: getApId(target) });
if (renote == null) return 'announce target is null';
if (!await this.noteEntityService.isVisibleForMe(renote, actor.id, { me: actor })) {
return 'skip: invalid actor for this activity';
}
if (!await this.noteEntityService.isVisibleForMe(renote, actor.id)) {
return 'skip: invalid actor for this activity';
if (renote.userHost == null && renote.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot renote a local-only note');
}
this.logger.info(`Creating the (Re)Note: ${uri}`);
@ -443,9 +447,11 @@ export class ApInboxService {
setImmediate(() => {
// Don't re-use the resolver, or it may throw recursion errors.
// Instead, create a new resolver with an appropriately-reduced recursion limit.
this.apPersonService.updatePerson(actor.uri, this.apResolverService.createResolver({
const subResolver = this.apResolverService.createResolver({
recursionLimit: resolver.getRecursionLimit() - resolver.getHistory().length,
}));
});
this.apPersonService.updatePerson(actor.uri, subResolver)
.catch(err => this.logger.error(`Failed to update person: ${renderInlineError(err)}`));
});
}
});
@ -500,7 +506,7 @@ export class ApInboxService {
resolver ??= this.apResolverService.createResolver();
const object = await resolver.resolve(activityObject).catch(e => {
this.logger.error(`Resolution failed: ${e}`);
this.logger.error(`Resolution failed: ${renderInlineError(e)}`);
throw e;
});
@ -537,12 +543,6 @@ export class ApInboxService {
await this.apNoteService.createNote(note, actor, resolver, silent);
return 'ok';
} catch (err) {
if (err instanceof StatusError && !err.isRetryable) {
return `skip: ${err.statusCode}`;
} else {
throw err;
}
} finally {
unlock();
}
@ -675,7 +675,7 @@ export class ApInboxService {
resolver ??= this.apResolverService.createResolver();
const object = await resolver.resolve(activity.object).catch(e => {
this.logger.error(`Resolution failed: ${e}`);
this.logger.error(`Resolution failed: ${renderInlineError(e)}`);
throw e;
});
@ -747,7 +747,7 @@ export class ApInboxService {
resolver ??= this.apResolverService.createResolver();
const object = await resolver.resolve(activity.object).catch(e => {
this.logger.error(`Resolution failed: ${e}`);
this.logger.error(`Resolution failed: ${renderInlineError(e)}`);
throw e;
});
@ -768,12 +768,7 @@ export class ApInboxService {
return 'skip: follower not found';
}
const isFollowing = await this.followingsRepository.exists({
where: {
followerId: follower.id,
followeeId: actor.id,
},
});
const isFollowing = await this.cacheService.userFollowingsCache.fetch(follower.id).then(f => f.has(actor.id));
if (isFollowing) {
await this.userFollowingService.unfollow(follower, actor);
@ -832,12 +827,7 @@ export class ApInboxService {
},
});
const isFollowing = await this.followingsRepository.exists({
where: {
followerId: actor.id,
followeeId: followee.id,
},
});
const isFollowing = await this.cacheService.userFollowingsCache.fetch(actor.id).then(f => f.has(followee.id));
if (requestExist) {
await this.userFollowingService.cancelFollowRequest(followee, actor);
@ -879,7 +869,7 @@ export class ApInboxService {
resolver ??= this.apResolverService.createResolver();
const object = await resolver.resolve(activity.object).catch(e => {
this.logger.error(`Resolution failed: ${e}`);
this.logger.error(`Resolution failed: ${renderInlineError(e)}`);
throw e;
});

2
packages/backend/src/core/activitypub/ApMfmService.ts
View file

@ -4,7 +4,7 @@
*/
import { Injectable } from '@nestjs/common';
import * as mfm from '@transfem-org/sfm-js';
import * as mfm from 'mfm-js';
import { MfmService, Appender } from '@/core/MfmService.js';
import type { MiNote } from '@/models/Note.js';
import { bindThis } from '@/decorators.js';

235
packages/backend/src/core/activitypub/ApRendererService.ts
View file

@ -6,8 +6,9 @@
import { createPublicKey, randomUUID } from 'node:crypto';
import { Inject, Injectable } from '@nestjs/common';
import { In } from 'typeorm';
import * as mfm from '@transfem-org/sfm-js';
import * as mfm from 'mfm-js';
import { UnrecoverableError } from 'bullmq';
import { Element, Text } from 'domhandler';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import type { MiPartialLocalUser, MiLocalUser, MiPartialRemoteUser, MiRemoteUser, MiUser } from '@/models/User.js';
@ -31,10 +32,12 @@ import { IdService } from '@/core/IdService.js';
import { appendContentWarning } from '@/misc/append-content-warning.js';
import { QueryService } from '@/core/QueryService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { CacheService } from '@/core/CacheService.js';
import { isPureRenote, isQuote, isRenote } from '@/misc/is-renote.js';
import { JsonLdService } from './JsonLdService.js';
import { ApMfmService } from './ApMfmService.js';
import { CONTEXT } from './misc/contexts.js';
import { getApId, IOrderedCollection, IOrderedCollectionPage } from './type.js';
import { getApId, ILink, IOrderedCollection, IOrderedCollectionPage } from './type.js';
import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IMove, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate } from './type.js';
@Injectable()
@ -74,6 +77,7 @@ export class ApRendererService {
private idService: IdService,
private readonly queryService: QueryService,
private utilityService: UtilityService,
private readonly cacheService: CacheService,
) {
}
@ -231,7 +235,7 @@ export class ApRendererService {
*/
@bindThis
public async renderFollowUser(id: MiUser['id']): Promise<string> {
const user = await this.usersRepository.findOneByOrFail({ id: id }) as MiPartialLocalUser | MiPartialRemoteUser;
const user = await this.cacheService.findUserById(id) as MiPartialLocalUser | MiPartialRemoteUser;
return this.userEntityService.getUserUri(user);
}
@ -401,7 +405,7 @@ export class ApRendererService {
inReplyToNote = await this.notesRepository.findOneBy({ id: note.replyId });
if (inReplyToNote != null) {
const inReplyToUser = await this.usersRepository.findOneBy({ id: inReplyToNote.userId });
const inReplyToUser = await this.cacheService.findUserById(inReplyToNote.userId);
if (inReplyToUser) {
if (inReplyToNote.uri) {
@ -419,9 +423,9 @@ export class ApRendererService {
inReplyTo = null;
}
let quote;
let quote: string | undefined = undefined;
if (note.renoteId) {
if (isRenote(note) && isQuote(note)) {
const renote = await this.notesRepository.findOneBy({ id: note.renoteId });
if (renote) {
@ -475,16 +479,18 @@ export class ApRendererService {
// the claas name `quote-inline` is used in non-misskey clients for styling quote notes.
// For compatibility, the span part should be kept as possible.
apAppend.push((doc, body) => {
body.appendChild(doc.createElement('br'));
body.appendChild(doc.createElement('br'));
const span = doc.createElement('span');
span.className = 'quote-inline';
span.appendChild(doc.createTextNode('RE: '));
const link = doc.createElement('a');
link.setAttribute('href', quote);
link.textContent = quote;
span.appendChild(link);
body.appendChild(span);
body.childNodes.push(new Element('br', {}));
body.childNodes.push(new Element('br', {}));
const span = new Element('span', {
class: 'quote-inline',
});
span.childNodes.push(new Text('RE: '));
const link = new Element('a', {
href: quote,
});
link.childNodes.push(new Text(quote));
span.childNodes.push(link);
body.childNodes.push(span);
});
}
@ -500,12 +506,22 @@ export class ApRendererService {
const emojis = await this.getEmojis(note.emojis);
const apemojis = emojis.filter(emoji => !emoji.localOnly).map(emoji => this.renderEmoji(emoji));
const tag = [
const tag: IObject[] = [
...hashtagTags,
...mentionTags,
...apemojis,
];
// https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md
if (quote) {
tag.push({
type: 'Link',
mediaType: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
rel: 'https://misskey-hub.net/ns#_misskey_quote',
href: quote,
} satisfies ILink);
}
const asPoll = poll ? {
type: 'Question',
[poll.expiresAt && poll.expiresAt < new Date() ? 'closed' : 'endTime']: poll.expiresAt,
@ -529,6 +545,7 @@ export class ApRendererService {
attributedTo,
summary: summary ?? undefined,
content: content ?? undefined,
updated: note.updatedAt?.toISOString() ?? undefined,
_misskey_content: text,
source: {
content: text,
@ -537,6 +554,8 @@ export class ApRendererService {
_misskey_quote: quote,
quoteUrl: quote,
quoteUri: quote,
// https://codeberg.org/fediverse/fep/src/branch/main/fep/044f/fep-044f.md
quote: quote,
published: this.idService.parse(note.id).date.toISOString(),
to,
cc,
@ -613,6 +632,7 @@ export class ApRendererService {
enableRss: user.enableRss,
speakAsCat: user.speakAsCat,
attachment: attachment.length ? attachment : undefined,
attributionDomains: user.attributionDomains,
};
if (user.movedToUri) {
@ -740,162 +760,6 @@ export class ApRendererService {
};
}
@bindThis
public async renderUpNote(note: MiNote, author: MiUser, dive = true): Promise<IPost> {
const getPromisedFiles = async (ids: string[]): Promise<MiDriveFile[]> => {
if (ids.length === 0) return [];
const items = await this.driveFilesRepository.findBy({ id: In(ids) });
return ids.map(id => items.find(item => item.id === id)).filter((item): item is MiDriveFile => item != null);
};
let inReplyTo;
let inReplyToNote: MiNote | null;
if (note.replyId) {
inReplyToNote = await this.notesRepository.findOneBy({ id: note.replyId });
if (inReplyToNote != null) {
const inReplyToUser = await this.usersRepository.findOneBy({ id: inReplyToNote.userId });
if (inReplyToUser) {
if (inReplyToNote.uri) {
inReplyTo = inReplyToNote.uri;
} else {
if (dive) {
inReplyTo = await this.renderUpNote(inReplyToNote, inReplyToUser, false);
} else {
inReplyTo = `${this.config.url}/notes/${inReplyToNote.id}`;
}
}
}
}
} else {
inReplyTo = null;
}
let quote;
if (note.renoteId) {
const renote = await this.notesRepository.findOneBy({ id: note.renoteId });
if (renote) {
quote = renote.uri ? renote.uri : `${this.config.url}/notes/${renote.id}`;
}
}
const attributedTo = this.userEntityService.genLocalUserUri(note.userId);
const mentions = note.mentionedRemoteUsers ? (JSON.parse(note.mentionedRemoteUsers) as IMentionedRemoteUsers).map(x => x.uri) : [];
let to: string[] = [];
let cc: string[] = [];
if (note.visibility === 'public') {
to = ['https://www.w3.org/ns/activitystreams#Public'];
cc = [`${attributedTo}/followers`].concat(mentions);
} else if (note.visibility === 'home') {
to = [`${attributedTo}/followers`];
cc = ['https://www.w3.org/ns/activitystreams#Public'].concat(mentions);
} else if (note.visibility === 'followers') {
to = [`${attributedTo}/followers`];
cc = mentions;
} else {
to = mentions;
}
const mentionedUsers = note.mentions && note.mentions.length > 0 ? await this.usersRepository.findBy({
id: In(note.mentions),
}) : [];
const hashtagTags = note.tags.map(tag => this.renderHashtag(tag));
const mentionTags = mentionedUsers.map(u => this.renderMention(u as MiLocalUser | MiRemoteUser));
const files = await getPromisedFiles(note.fileIds);
const text = note.text ?? '';
let poll: MiPoll | null = null;
if (note.hasPoll) {
poll = await this.pollsRepository.findOneBy({ noteId: note.id });
}
const apAppend: Appender[] = [];
if (quote) {
// Append quote link as `<br><br><span class="quote-inline">RE: <a href="...">...</a></span>`
// the claas name `quote-inline` is used in non-misskey clients for styling quote notes.
// For compatibility, the span part should be kept as possible.
apAppend.push((doc, body) => {
body.appendChild(doc.createElement('br'));
body.appendChild(doc.createElement('br'));
const span = doc.createElement('span');
span.className = 'quote-inline';
span.appendChild(doc.createTextNode('RE: '));
const link = doc.createElement('a');
link.setAttribute('href', quote);
link.textContent = quote;
span.appendChild(link);
body.appendChild(span);
});
}
let summary = note.cw === '' ? String.fromCharCode(0x200B) : note.cw;
// Apply mandatory CW, if applicable
if (author.mandatoryCW) {
summary = appendContentWarning(summary, author.mandatoryCW);
}
const { content } = this.apMfmService.getNoteHtml(note, apAppend);
const emojis = await this.getEmojis(note.emojis);
const apemojis = emojis.filter(emoji => !emoji.localOnly).map(emoji => this.renderEmoji(emoji));
const tag = [
...hashtagTags,
...mentionTags,
...apemojis,
];
const asPoll = poll ? {
type: 'Question',
[poll.expiresAt && poll.expiresAt < new Date() ? 'closed' : 'endTime']: poll.expiresAt,
[poll.multiple ? 'anyOf' : 'oneOf']: poll.choices.map((text, i) => ({
type: 'Note',
name: text,
replies: {
type: 'Collection',
totalItems: poll!.votes[i],
},
})),
} as const : {};
return {
id: `${this.config.url}/notes/${note.id}`,
type: 'Note',
attributedTo,
summary: summary ?? undefined,
content: content ?? undefined,
updated: note.updatedAt?.toISOString(),
_misskey_content: text,
source: {
content: text,
mediaType: 'text/x.misskeymarkdown',
},
_misskey_quote: quote,
quoteUrl: quote,
quoteUri: quote,
published: this.idService.parse(note.id).date.toISOString(),
to,
cc,
inReplyTo,
attachment: files.map(x => this.renderDocument(x)),
sensitive: note.cw != null || files.some(file => file.isSensitive),
tag,
...asPoll,
};
}
@bindThis
public renderVote(user: { id: MiUser['id'] }, vote: MiPollVote, note: MiNote, poll: MiPoll, pollOwner: MiRemoteUser): ICreate {
return {
@ -935,9 +799,7 @@ export class ApRendererService {
const keypair = await this.userKeypairService.getUserKeypair(user.id);
const jsonLd = this.jsonLdService.use();
jsonLd.debug = false;
activity = await jsonLd.signRsaSignature2017(activity, keypair.privateKey, `${this.config.url}/users/${user.id}#main-key`);
activity = await this.jsonLdService.signRsaSignature2017(activity, keypair.privateKey, `${this.config.url}/users/${user.id}#main-key`);
return activity;
}
@ -1051,6 +913,27 @@ export class ApRendererService {
};
}
@bindThis
public async renderNoteOrRenoteActivity(note: MiNote, user: MiUser, hint?: { renote?: MiNote | null }) {
if (note.localOnly) return null;
if (isPureRenote(note)) {
const renote = hint?.renote ?? note.renote ?? await this.notesRepository.findOneByOrFail({ id: note.renoteId });
const apAnnounce = this.renderAnnounce(renote.uri ?? `${this.config.url}/notes/${renote.id}`, note);
return this.addContext(apAnnounce);
}
const apNote = await this.renderNote(note, user, false);
if (note.updatedAt != null) {
const apUpdate = this.renderUpdate(apNote, user);
return this.addContext(apUpdate);
} else {
const apCreate = this.renderCreate(apNote, note);
return this.addContext(apCreate);
}
}
@bindThis
private async getEmojis(names: string[]): Promise<MiEmoji[]> {
if (names.length === 0) return [];

84
packages/backend/src/core/activitypub/ApRequestService.ts
View file

@ -6,7 +6,7 @@
import * as crypto from 'node:crypto';
import { URL } from 'node:url';
import { Inject, Injectable } from '@nestjs/common';
import { Window } from 'happy-dom';
import { load as cheerio } from 'cheerio/slim';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import type { MiUser } from '@/models/User.js';
@ -18,6 +18,8 @@ import { bindThis } from '@/decorators.js';
import type Logger from '@/logger.js';
import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js';
import type { IObject, IObjectWithId } from './type.js';
import type { Cheerio, CheerioAPI } from 'cheerio/slim';
import type { AnyNode } from 'domhandler';
type Request = {
url: string;
@ -184,10 +186,11 @@ export class ApRequestService {
* Get AP object with http-signature
* @param user http-signature user
* @param url URL to fetch
* @param followAlternate
* @param allowAnonymous If a fetched object lacks an ID, then it will be auto-generated from the final URL. (default: false)
* @param followAlternate Whether to resolve HTML responses to their referenced canonical AP endpoint. (default: true)
*/
@bindThis
public async signedGet(url: string, user: { id: MiUser['id'] }, followAlternate?: boolean): Promise<IObjectWithId> {
public async signedGet(url: string, user: { id: MiUser['id'] }, allowAnonymous = false, followAlternate?: boolean): Promise<IObjectWithId> {
this.apUtilityService.assertApUrl(url);
const _followAlternate = followAlternate ?? true;
@ -218,53 +221,33 @@ export class ApRequestService {
(contentType ?? '').split(';')[0].trimEnd().toLowerCase() === 'text/html' &&
_followAlternate === true
) {
const html = await res.text();
const { window, happyDOM } = new Window({
settings: {
disableJavaScriptEvaluation: true,
disableJavaScriptFileLoading: true,
disableCSSFileLoading: true,
disableComputedStyleRendering: true,
handleDisabledFileLoadingAsSuccess: true,
navigation: {
disableMainFrameNavigation: true,
disableChildFrameNavigation: true,
disableChildPageNavigation: true,
disableFallbackToSetURL: true,
},
timer: {
maxTimeout: 0,
maxIntervalTime: 0,
maxIntervalIterations: 0,
},
},
});
const document = window.document;
let alternate: Cheerio<AnyNode> | null;
try {
document.documentElement.innerHTML = html;
const html = await res.text();
const document = cheerio(html);
// Search for any matching value in priority order:
// 1. Type=AP > Type=none > Type=anything
// 2. Alternate > Canonical
// 3. Page order (fallback)
const alternate =
document.querySelector('head > link[href][rel="alternate"][type="application/activity+json"]') ??
document.querySelector('head > link[href][rel="canonical"][type="application/activity+json"]') ??
document.querySelector('head > link[href][rel="alternate"]:not([type])') ??
document.querySelector('head > link[href][rel="canonical"]:not([type])') ??
document.querySelector('head > link[href][rel="alternate"]') ??
document.querySelector('head > link[href][rel="canonical"]');
if (alternate) {
const href = alternate.getAttribute('href');
if (href && this.apUtilityService.haveSameAuthority(url, href)) {
return await this.signedGet(href, user, false);
}
}
alternate = selectFirst(document, [
'head > link[href][rel="alternate"][type="application/activity+json"]',
'head > link[href][rel="canonical"][type="application/activity+json"]',
'head > link[href][rel="alternate"]:not([type])',
'head > link[href][rel="canonical"]:not([type])',
'head > link[href][rel="alternate"]',
'head > link[href][rel="canonical"]',
]);
} catch {
// something went wrong parsing the HTML, ignore the whole thing
} finally {
happyDOM.close().catch(err => {});
alternate = null;
}
if (alternate) {
const href = alternate.attr('href');
if (href && this.apUtilityService.haveSameAuthority(url, href)) {
return await this.signedGet(href, user, allowAnonymous, false);
}
}
}
//#endregion
@ -275,8 +258,23 @@ export class ApRequestService {
// Make sure the object ID matches the final URL (which is where it actually exists).
// The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match.
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
if (allowAnonymous && activity.id == null) {
activity.id = res.url;
} else {
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
}
return activity as IObjectWithId;
}
}
function selectFirst($: CheerioAPI, selectors: string[]): Cheerio<AnyNode> | null {
for (const selector of selectors) {
const selection = $(selector);
if (selection.length > 0) {
return selection;
}
}
return null;
}

229
packages/backend/src/core/activitypub/ApResolverService.ts
View file

@ -5,6 +5,7 @@
import { Inject, Injectable } from '@nestjs/common';
import { IsNull, Not } from 'typeorm';
import promiseLimit from 'promise-limit';
import type { MiLocalUser, MiRemoteUser } from '@/models/User.js';
import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta, SkApFetchLog } from '@/models/_.js';
import type { Config } from '@/config.js';
@ -19,11 +20,14 @@ import { ApLogService, calculateDurationSince, extractObjectContext } from '@/co
import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { getApId, getNullableApId, IObjectWithId, isCollectionOrOrderedCollection } from './type.js';
import { toArray } from '@/misc/prelude/array.js';
import { isPureRenote } from '@/misc/is-renote.js';
import { CacheService } from '@/core/CacheService.js';
import { AnyCollection, getApId, getNullableApId, IObjectWithId, isCollection, isCollectionOrOrderedCollection, isCollectionPage, isOrderedCollection, isOrderedCollectionPage } from './type.js';
import { ApDbResolverService } from './ApDbResolverService.js';
import { ApRendererService } from './ApRendererService.js';
import { ApRequestService } from './ApRequestService.js';
import type { IObject, ICollection, IOrderedCollection, ApObject } from './type.js';
import type { IObject, ApObject, IAnonymousObject } from './type.js';
export class Resolver {
private history: Set<string>;
@ -47,6 +51,7 @@ export class Resolver {
private loggerService: LoggerService,
private readonly apLogService: ApLogService,
private readonly apUtilityService: ApUtilityService,
private readonly cacheService: CacheService,
private recursionLimit = 256,
) {
this.history = new Set();
@ -63,34 +68,129 @@ export class Resolver {
return this.recursionLimit;
}
public async resolveCollection(value: string | IObjectWithId, allowAnonymous?: boolean, sentFromUri?: string): Promise<AnyCollection & IObjectWithId>;
public async resolveCollection(value: string | IObject, allowAnonymous: boolean | undefined, sentFromUri: string): Promise<AnyCollection & IObjectWithId>;
public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise<AnyCollection>;
@bindThis
public async resolveCollection(value: string | IObject): Promise<ICollection | IOrderedCollection> {
public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise<AnyCollection> {
const collection = typeof value === 'string'
? await this.resolve(value)
: value;
? sentFromUri
? await this.secureResolve(value, sentFromUri, allowAnonymous)
: await this.resolve(value, allowAnonymous)
: value; // TODO try and remove this eventually, as it's a major security foot-gun
if (isCollectionOrOrderedCollection(collection)) {
return collection;
} else {
throw new IdentifiableError('f100eccf-f347-43fb-9b45-96a0831fb635', `unrecognized collection type: ${collection.type}`);
throw new IdentifiableError('f100eccf-f347-43fb-9b45-96a0831fb635', `collection ${getApId(value)} has unsupported type: ${collection.type}`);
}
}
public async resolveCollectionItems(collection: IAnonymousObject, limit?: number | null, allowAnonymousItems?: true, concurrency?: number): Promise<IAnonymousObject[]>;
public async resolveCollectionItems(collection: string | IObjectWithId, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise<IObjectWithId[]>;
public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise<IObject[]>;
/**
* Recursively resolves items from a collection.
* Stops when reaching the resolution limit or an optional item limit - whichever is lower.
* This method supports Collection, OrderedCollection, and individual pages of either type.
* Malformed collections (mixing Ordered and un-Ordered types) are also supported.
* @param collection Collection to resolve from - can be a URL or object of any supported collection type.
* @param limit Maximum number of items to resolve. If null or undefined (default), then items will be resolved until reaching the recursion limit.
* @param allowAnonymousItems If true, collection items can be anonymous (lack an ID). If false (default), then an error is thrown when reaching an item without ID.
* @param concurrency Maximum number of items to resolve at once. (default: 4)
*/
@bindThis
public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency = 4): Promise<IObject[]> {
const resolvedItems: IObject[] = [];
// This is pulled up to avoid code duplication below
const iterate = async(items: ApObject, current: AnyCollection) => {
const sentFrom = current.id;
const itemArr = toArray(items);
const itemLimit = limit ?? Number.MAX_SAFE_INTEGER;
const allowAnonymous = allowAnonymousItems ?? false;
await this.resolveItemArray(itemArr, sentFrom, itemLimit, concurrency, allowAnonymous, resolvedItems);
};
let current: AnyCollection | null = await this.resolveCollection(collection);
do {
// Iterate all items in the current page
if (current.items) {
await iterate(current.items, current);
}
if (current.orderedItems) {
await iterate(current.orderedItems, current);
}
if (this.history.size >= this.recursionLimit) {
// Stop when we reach the fetch limit
current = null;
} else if (limit != null && resolvedItems.length >= limit) {
// Stop when we reach the item limit
current = null;
} else if (isCollection(current) || isOrderedCollection(current)) {
// Continue to first page
current = current.first ? await this.resolveCollection(current.first, true, current.id) : null;
} else if (isCollectionPage(current) || isOrderedCollectionPage(current)) {
// Continue to next page
current = current.next ? await this.resolveCollection(current.next, true, current.id) : null;
} else {
// Stop in all other conditions
current = null;
}
} while (current != null);
return resolvedItems;
}
private async resolveItemArray(source: (string | IObject)[], sentFrom: undefined, itemLimit: number, concurrency: number, allowAnonymousItems: true, destination: IAnonymousObject[]): Promise<void>;
private async resolveItemArray(source: (string | IObject)[], sentFrom: string, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObjectWithId[]): Promise<void>;
private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise<void>;
private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise<void> {
const recursionLimit = this.recursionLimit - this.history.size;
const batchLimit = Math.min(source.length, recursionLimit, itemLimit);
const limiter = promiseLimit<IObject>(concurrency);
const batch = await Promise.all(source
.slice(0, batchLimit)
.map(item => limiter(async () => {
if (sentFrom) {
// Use secureResolve to avoid re-fetching items that were included inline.
return await this.secureResolve(item, sentFrom, allowAnonymousItems);
} else if (allowAnonymousItems) {
return await this.resolveAnonymous(item);
} else {
// ID is required if we have neither sentFrom not allowAnonymousItems
const id = getApId(item);
return await this.resolve(id);
}
})));
destination.push(...batch);
};
/**
* Securely resolves an AP object or URL that has been sent from another instance.
* An input object is trusted if and only if its ID matches the authority of sentFromUri.
* In all other cases, the object is re-fetched from remote by input string or object ID.
* @param input The input object or URL to resolve
* @param sentFromUri The URL where this object originated. This MUST be accurate - all security checks depend on this value!
* @param allowAnonymous If true, anonymous objects are allowed and will have their ID set to sentFromUri. If false (default) then anonymous objects will be rejected with an error.
*/
@bindThis
public async secureResolve(input: ApObject, sentFromUri: string): Promise<IObjectWithId> {
public async secureResolve(input: string | IObject | [string | IObject], sentFromUri: string, allowAnonymous?: boolean): Promise<IObjectWithId> {
// Unpack arrays to get the value element.
const value = fromTuple(input);
if (value == null) {
throw new IdentifiableError('20058164-9de1-4573-8715-425753a21c1d', 'Cannot resolve null input');
// If anonymous input is allowed, then any object is automatically valid if we set the ID.
// We can short-circuit here and avoid un-necessary checks.
if (allowAnonymous && typeof(value) === 'object' && value.id == null) {
value.id = sentFromUri;
return value as IObjectWithId;
}
// This will throw if the input has no ID, which is good because we can't verify an anonymous object anyway.
const id = getApId(value);
// This ensures the input has a string ID, protecting against type confusion and rejecting anonymous objects.
const id = getApId(value, sentFromUri);
// Check if we can use the provided object as-is.
// Our security requires that the object ID matches the host authority that sent it, otherwise it can't be trusted.
@ -100,28 +200,52 @@ export class Resolver {
}
// If the checks didn't pass, then we must fetch the object and use that.
return await this.resolve(id);
return await this.resolve(id, allowAnonymous);
}
public async resolve(value: string | [string]): Promise<IObjectWithId>;
public async resolve(value: string | IObject | [string | IObject]): Promise<IObject>;
/**
* Resolves an anonymous object.
* The returned value will not have any ID present.
* If one is provided in the response, it will be removed automatically.
*/
@bindThis
public async resolve(value: string | IObject | [string | IObject]): Promise<IObject> {
public async resolveAnonymous(value: string | IObject | [string | IObject]): Promise<IAnonymousObject> {
value = fromTuple(value);
const object = await this.resolve(value);
object.id = undefined;
return object as IAnonymousObject;
}
public async resolve(value: string | [string], allowAnonymous?: boolean): Promise<IObjectWithId>;
public async resolve(value: string | IObjectWithId | [string | IObjectWithId], allowAnonymous?: boolean): Promise<IObjectWithId>;
public async resolve(value: string | IObject | [string | IObject], allowAnonymous?: boolean): Promise<IObject>;
/**
* Resolves a URL or object to an AP object.
* Tuples are expanded to their first element before anything else, and non-string inputs are returned as-is.
* Otherwise, the string URL is fetched and validated to represent a valid ActivityPub object.
* @param value The input value to resolve
* @param allowAnonymous Determines what to do if a response object lacks an ID field. If false (default), then an exception is thrown. If true, then the ID is populated from the final response URL.
*/
@bindThis
public async resolve(value: string | IObject | [string | IObject], allowAnonymous = false): Promise<IObject> {
value = fromTuple(value);
// TODO try and remove this eventually, as it's a major security foot-gun
if (typeof value !== 'string') {
return value;
}
const host = this.utilityService.extractDbHost(value);
if (this.config.activityLogging.enabled && !this.utilityService.isSelfHost(host)) {
return await this._resolveLogged(value, host);
return await this._resolveLogged(value, host, allowAnonymous);
} else {
return await this._resolve(value, host);
return await this._resolve(value, host, allowAnonymous);
}
}
private async _resolveLogged(requestUri: string, host: string): Promise<IObjectWithId> {
private async _resolveLogged(requestUri: string, host: string, allowAnonymous: boolean): Promise<IObjectWithId> {
const startTime = process.hrtime.bigint();
const log = await this.apLogService.createFetchLog({
@ -130,7 +254,7 @@ export class Resolver {
});
try {
const result = await this._resolve(requestUri, host, log);
const result = await this._resolve(requestUri, host, allowAnonymous, log);
log.accepted = true;
log.result = 'ok';
@ -150,20 +274,20 @@ export class Resolver {
}
}
private async _resolve(value: string, host: string, log?: SkApFetchLog): Promise<IObjectWithId> {
private async _resolve(value: string, host: string, allowAnonymous: boolean, log?: SkApFetchLog): Promise<IObjectWithId> {
if (value.includes('#')) {
// URLs with fragment parts cannot be resolved correctly because
// the fragment part does not get transmitted over HTTP(S).
// Avoid strange behaviour by not trying to resolve these at all.
throw new IdentifiableError('b94fd5b1-0e3b-4678-9df2-dad4cd515ab2', `cannot resolve URL with fragment: ${value}`);
throw new IdentifiableError('b94fd5b1-0e3b-4678-9df2-dad4cd515ab2', `failed to resolve ${value}: URL contains fragment`);
}
if (this.history.has(value)) {
throw new IdentifiableError('0dc86cf6-7cd6-4e56-b1e6-5903d62d7ea5', `cannot resolve already resolved URL: ${value}`);
throw new IdentifiableError('0dc86cf6-7cd6-4e56-b1e6-5903d62d7ea5', `failed to resolve ${value}: recursive resolution blocked`);
}
if (this.history.size > this.recursionLimit) {
throw new IdentifiableError('d592da9f-822f-4d91-83d7-4ceefabcf3d2', `hit recursion limit: ${value}`);
throw new IdentifiableError('d592da9f-822f-4d91-83d7-4ceefabcf3d2', `failed to resolve ${value}: hit recursion limit`);
}
this.history.add(value);
@ -173,7 +297,7 @@ export class Resolver {
}
if (!this.utilityService.isFederationAllowedHost(host)) {
throw new IdentifiableError('09d79f9e-64f1-4316-9cfa-e75c4d091574', `cannot fetch AP object ${value}: blocked instance ${host}`);
throw new IdentifiableError('09d79f9e-64f1-4316-9cfa-e75c4d091574', `failed to resolve ${value}: instance ${host} is blocked`);
}
if (this.config.signToActivityPubGet && !this.user) {
@ -181,8 +305,8 @@ export class Resolver {
}
const object = (this.user
? await this.apRequestService.signedGet(value, this.user)
: await this.httpRequestService.getActivityJson(value));
? await this.apRequestService.signedGet(value, this.user, allowAnonymous)
: await this.httpRequestService.getActivityJson(value, false, allowAnonymous));
if (log) {
const { object: objectOnly, context, contextHash } = extractObjectContext(object);
@ -203,12 +327,12 @@ export class Resolver {
!(object['@context'] as unknown[]).includes('https://www.w3.org/ns/activitystreams') :
object['@context'] !== 'https://www.w3.org/ns/activitystreams'
) {
throw new IdentifiableError('72180409-793c-4973-868e-5a118eb5519b', `invalid AP object ${value}: does not have ActivityStreams context`);
throw new IdentifiableError('72180409-793c-4973-868e-5a118eb5519b', `failed to resolve ${value}: response does not have ActivityStreams context`);
}
// The object ID is already validated to match the final URL's authority by signedGet / getActivityJson.
// We only need to validate that it also matches the original URL's authority, in case of redirects.
const objectId = getApId(object);
const objectId = getApId(object, value);
// We allow some limited cross-domain redirects, which means the host may have changed during fetch.
// Additional checks are needed to validate the scope of cross-domain redirects.
@ -219,64 +343,65 @@ export class Resolver {
// Check if the redirect bounce from [allowed domain] to [blocked domain].
if (!this.utilityService.isFederationAllowedHost(finalHost)) {
throw new IdentifiableError('0a72bf24-2d9b-4f1d-886b-15aaa31adeda', `cannot fetch AP object ${value}: redirected to blocked instance ${finalHost}`);
throw new IdentifiableError('0a72bf24-2d9b-4f1d-886b-15aaa31adeda', `failed to resolve ${value}: redirected to blocked instance ${finalHost}`);
}
}
return object;
}
// TODO try to remove this, as it creates a large attack surface
@bindThis
private resolveLocal(url: string): Promise<IObjectWithId> {
const parsed = this.apDbResolverService.parseUri(url);
if (!parsed.local) throw new IdentifiableError('02b40cd0-fa92-4b0c-acc9-fb2ada952ab8', `resolveLocal - not a local URL: ${url}`);
if (!parsed.local) throw new IdentifiableError('02b40cd0-fa92-4b0c-acc9-fb2ada952ab8', `failed to resolve local ${url}: not a local URL`);
switch (parsed.type) {
case 'notes':
return this.notesRepository.findOneByOrFail({ id: parsed.id })
return this.notesRepository.findOneOrFail({ where: { id: parsed.id, userHost: IsNull() }, relations: { user: true, renote: true } })
.then(async note => {
const author = await this.usersRepository.findOneByOrFail({ id: note.userId });
const author = note.user ?? await this.cacheService.findUserById(note.userId);
if (parsed.rest === 'activity') {
// this refers to the create activity and not the note itself
return this.apRendererService.addContext(this.apRendererService.renderCreate(await this.apRendererService.renderNote(note, author), note));
return await this.apRendererService.renderNoteOrRenoteActivity(note, author);
} else if (!isPureRenote(note)) {
const apNote = await this.apRendererService.renderNote(note, author);
return this.apRendererService.addContext(apNote);
} else {
return this.apRendererService.renderNote(note, author);
throw new IdentifiableError('732c2633-3395-4d51-a9b7-c7084774e3e7', `Failed to resolve local ${url}: cannot resolve a boost as note`);
}
}) as Promise<IObjectWithId>;
case 'users':
return this.usersRepository.findOneByOrFail({ id: parsed.id })
return this.cacheService.findLocalUserById(parsed.id)
.then(user => this.apRendererService.renderPerson(user as MiLocalUser));
case 'questions':
// Polls are indexed by the note they are attached to.
return Promise.all([
this.notesRepository.findOneByOrFail({ id: parsed.id }),
this.pollsRepository.findOneByOrFail({ noteId: parsed.id }),
this.notesRepository.findOneByOrFail({ id: parsed.id, userHost: IsNull() }),
this.pollsRepository.findOneByOrFail({ noteId: parsed.id, userHost: IsNull() }),
])
.then(([note, poll]) => this.apRendererService.renderQuestion({ id: note.userId }, note, poll)) as Promise<IObjectWithId>;
case 'likes':
return this.noteReactionsRepository.findOneByOrFail({ id: parsed.id }).then(async reaction =>
this.apRendererService.addContext(await this.apRendererService.renderLike(reaction, { uri: null })));
return this.noteReactionsRepository.findOneOrFail({ where: { id: parsed.id }, relations: { user: true } }).then(async reaction => {
if (reaction.user?.host != null) {
throw new IdentifiableError('02b40cd0-fa92-4b0c-acc9-fb2ada952ab8', `failed to resolve local ${url}: not a local reaction`);
}
return this.apRendererService.addContext(await this.apRendererService.renderLike(reaction, { uri: null }));
});
case 'follows':
return this.followRequestsRepository.findOneBy({ id: parsed.id })
.then(async followRequest => {
if (followRequest == null) throw new IdentifiableError('a9d946e5-d276-47f8-95fb-f04230289bb0', `resolveLocal - invalid follow request ID ${parsed.id}: ${url}`);
if (followRequest == null) throw new IdentifiableError('a9d946e5-d276-47f8-95fb-f04230289bb0', `failed to resolve local ${url}: invalid follow request ID`);
const [follower, followee] = await Promise.all([
this.usersRepository.findOneBy({
id: followRequest.followerId,
host: IsNull(),
}),
this.usersRepository.findOneBy({
id: followRequest.followeeId,
host: Not(IsNull()),
}),
this.cacheService.findLocalUserById(followRequest.followerId),
this.cacheService.findLocalUserById(followRequest.followeeId),
]);
if (follower == null || followee == null) {
throw new IdentifiableError('06ae3170-1796-4d93-a697-2611ea6d83b6', `resolveLocal - follower or followee does not exist: ${url}`);
throw new IdentifiableError('06ae3170-1796-4d93-a697-2611ea6d83b6', `failed to resolve local ${url}: follower or followee does not exist`);
}
return this.apRendererService.addContext(this.apRendererService.renderFollow(follower as MiLocalUser | MiRemoteUser, followee as MiLocalUser | MiRemoteUser, url));
});
default:
throw new IdentifiableError('7a5d2fc0-94bc-4db6-b8b8-1bf24a2e23d0', `resolveLocal: type ${parsed.type} unhandled: ${url}`);
throw new IdentifiableError('7a5d2fc0-94bc-4db6-b8b8-1bf24a2e23d0', `failed to resolve local ${url}: unsupported type ${parsed.type}`);
}
}
}
@ -314,6 +439,7 @@ export class ApResolverService {
private loggerService: LoggerService,
private readonly apLogService: ApLogService,
private readonly apUtilityService: ApUtilityService,
private readonly cacheService: CacheService,
) {
}
@ -339,6 +465,7 @@ export class ApResolverService {
this.loggerService,
this.apLogService,
this.apUtilityService,
this.cacheService,
opts?.recursionLimit,
);
}

8
packages/backend/src/core/activitypub/ApUtilityService.ts
View file

@ -24,7 +24,7 @@ export class ApUtilityService {
public assertIdMatchesUrlAuthority(object: IObject, url: string): void {
// This throws if the ID is missing or invalid, but that's ok.
// Anonymous objects are impossible to verify, so we don't allow fetching them.
const id = getApId(object);
const id = getApId(object, url);
// Make sure the object ID matches the final URL (which is where it actually exists).
// The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match.
@ -80,7 +80,6 @@ export class ApUtilityService {
/**
* Verifies that a provided URL is in a format acceptable for federation.
* @throws {IdentifiableError} If URL cannot be parsed
* @throws {IdentifiableError} If URL contains a fragment
* @throws {IdentifiableError} If URL is not HTTPS
*/
public assertApUrl(url: string | URL): void {
@ -93,11 +92,6 @@ export class ApUtilityService {
}
}
// Hash component breaks federation
if (url.hash) {
throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: contains a fragment (#)`);
}
// Must be HTTPS
if (!this.checkHttps(url)) {
throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: unsupported protocol ${url.protocol}`);

83
packages/backend/src/core/activitypub/JsonLdService.ts
View file

@ -8,25 +8,61 @@ import { Injectable } from '@nestjs/common';
import { UnrecoverableError } from 'bullmq';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
import Logger from '@/logger.js';
import { LoggerService } from '@/core/LoggerService.js';
import { StatusError } from '@/misc/status-error.js';
import { CONTEXT, PRELOADED_CONTEXTS } from './misc/contexts.js';
import { validateContentTypeSetAsJsonLD } from './misc/validator.js';
import type { JsonLdDocument } from 'jsonld';
import type { ContextDefinition, JsonLdDocument } from 'jsonld';
import type { JsonLd as JsonLdObject, RemoteDocument } from 'jsonld/jsonld-spec.js';
// https://stackoverflow.com/a/66252656
type RemoveIndex<T> = {
[ K in keyof T as string extends K
? never
: number extends K
? never
: symbol extends K
? never
: K
] : T[K];
};
export type Document = RemoveIndex<JsonLdDocument>;
export type Signature = {
id?: string;
type: string;
creator: string;
domain?: string;
nonce: string;
created: string;
signatureValue: string;
};
export type Signed<T extends Document> = T & {
signature: Signature;
};
export function isSigned<T extends Document>(doc: T): doc is Signed<T> {
return 'signature' in doc && typeof(doc.signature) === 'object';
}
// RsaSignature2017 implementation is based on https://github.com/transmute-industries/RsaSignature2017
class JsonLd {
public debug = false;
public preLoad = true;
public loderTimeout = 5000;
@Injectable()
export class JsonLdService {
private readonly logger: Logger;
constructor(
private httpRequestService: HttpRequestService,
loggerService: LoggerService,
) {
this.logger = loggerService.getLogger('json-ld');
}
@bindThis
public async signRsaSignature2017(data: any, privateKey: string, creator: string, domain?: string, created?: Date): Promise<any> {
public async signRsaSignature2017<T extends Document>(data: T, privateKey: string, creator: string, domain?: string, created?: Date): Promise<Signed<T>> {
const options: {
type: string;
creator: string;
@ -62,7 +98,7 @@ class JsonLd {
}
@bindThis
public async verifyRsaSignature2017(data: any, publicKey: string): Promise<boolean> {
public async verifyRsaSignature2017(data: Signed<Document>, publicKey: string): Promise<boolean> {
const toBeSigned = await this.createVerifyData(data, data.signature);
const verifier = crypto.createVerify('sha256');
verifier.update(toBeSigned);
@ -70,7 +106,7 @@ class JsonLd {
}
@bindThis
public async createVerifyData(data: any, options: any): Promise<string> {
public async createVerifyData<T extends Document>(data: T, options: Partial<Signature>): Promise<string> {
const transformedOptions = {
...options,
'@context': 'https://w3id.org/identity/v1',
@ -80,17 +116,18 @@ class JsonLd {
delete transformedOptions['signatureValue'];
const canonizedOptions = await this.normalize(transformedOptions);
const optionsHash = this.sha256(canonizedOptions.toString());
const transformedData = { ...data };
const transformedData = { ...data } as T & { signature?: unknown };
delete transformedData['signature'];
const cannonidedData = await this.normalize(transformedData);
if (this.debug) console.debug(`cannonidedData: ${cannonidedData}`);
this.logger.debug('cannonidedData', cannonidedData);
const documentHash = this.sha256(cannonidedData.toString());
const verifyData = `${optionsHash}${documentHash}`;
return verifyData;
}
@bindThis
public async compact(data: any, context: any = CONTEXT): Promise<JsonLdDocument> {
// TODO our default CONTEXT isn't valid for the library, is this a bug?
public async compact(data: Document, context: ContextDefinition = CONTEXT as unknown as ContextDefinition): Promise<Document> {
const customLoader = this.getLoader();
// XXX: Importing jsonld dynamically since Jest frequently fails to import it statically
// https://github.com/misskey-dev/misskey/pull/9894#discussion_r1103753595
@ -100,7 +137,7 @@ class JsonLd {
}
@bindThis
public async normalize(data: JsonLdDocument): Promise<string> {
public async normalize(data: Document): Promise<string> {
const customLoader = this.getLoader();
return (await import('jsonld')).default.normalize(data, {
documentLoader: customLoader,
@ -112,9 +149,9 @@ class JsonLd {
return async (url: string): Promise<RemoteDocument> => {
if (!/^https?:\/\//.test(url)) throw new UnrecoverableError(`Invalid URL: ${url}`);
if (this.preLoad) {
{
if (url in PRELOADED_CONTEXTS) {
if (this.debug) console.debug(`HIT: ${url}`);
this.logger.debug(`Preload HIT: ${url}`);
return {
contextUrl: undefined,
document: PRELOADED_CONTEXTS[url],
@ -123,7 +160,7 @@ class JsonLd {
}
}
if (this.debug) console.debug(`MISS: ${url}`);
this.logger.debug(`Preload MISS: ${url}`);
const document = await this.fetchDocument(url);
return {
contextUrl: undefined,
@ -141,7 +178,6 @@ class JsonLd {
headers: {
Accept: 'application/ld+json, application/json',
},
timeout: this.loderTimeout,
},
{
throwErrorWhenResponseNotOk: false,
@ -149,7 +185,7 @@ class JsonLd {
},
).then(res => {
if (!res.ok) {
throw new Error(`JSON-LD fetch failed with ${res.status} ${res.statusText}: ${url}`);
throw new StatusError(`failed to fetch JSON-LD from ${url}`, res.status, res.statusText);
} else {
return res.json();
}
@ -165,16 +201,3 @@ class JsonLd {
return hash.digest('hex');
}
}
@Injectable()
export class JsonLdService {
constructor(
private httpRequestService: HttpRequestService,
) {
}
@bindThis
public use(): JsonLd {
return new JsonLd(this.httpRequestService);
}
}

8
packages/backend/src/core/activitypub/misc/contexts.ts
View file

@ -540,12 +540,20 @@ const extension_context_definition = {
quoteUrl: 'as:quoteUrl',
fedibird: 'http://fedibird.com/ns#',
quoteUri: 'fedibird:quoteUri',
quote: {
'@id': 'https://w3id.org/fep/044f#quote',
'@type': '@id',
},
// Mastodon
toot: 'http://joinmastodon.org/ns#',
Emoji: 'toot:Emoji',
featured: 'toot:featured',
discoverable: 'toot:discoverable',
indexable: 'toot:indexable',
attributionDomains: {
'@id': 'toot:attributionDomains',
'@type': '@id',
},
// schema
schema: 'http://schema.org#',
PropertyValue: 'schema:PropertyValue',

11
packages/backend/src/core/activitypub/misc/validator.ts
View file

@ -3,15 +3,14 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { IdentifiableError } from '@/misc/identifiable-error.js';
import type { Response } from 'node-fetch';
// TODO throw identifiable or unrecoverable errors
export function validateContentTypeSetAsActivityPub(response: Response): void {
const contentType = (response.headers.get('content-type') ?? '').toLowerCase();
if (contentType === '') {
throw new Error(`invalid content type of AP response - no content-type header: ${response.url}`);
throw new IdentifiableError('d09dc850-b76c-4f45-875a-7389339d78b8', `invalid AP response from ${response.url}: no content-type header`, true);
}
if (
contentType.startsWith('application/activity+json') ||
@ -19,7 +18,7 @@ export function validateContentTypeSetAsActivityPub(response: Response): void {
) {
return;
}
throw new Error(`invalid content type of AP response - content type is not application/activity+json or application/ld+json: ${response.url}`);
throw new IdentifiableError('dc110060-a5f2-461d-808b-39c62702ca64', `invalid AP response from ${response.url}: content type "${contentType}" is not application/activity+json or application/ld+json`);
}
const plusJsonSuffixRegex = /^\s*(application|text)\/[a-zA-Z0-9\.\-\+]+\+json\s*(;|$)/;
@ -28,7 +27,7 @@ export function validateContentTypeSetAsJsonLD(response: Response): void {
const contentType = (response.headers.get('content-type') ?? '').toLowerCase();
if (contentType === '') {
throw new Error(`invalid content type of JSON LD - no content-type header: ${response.url}`);
throw new IdentifiableError('45793ab7-7648-4886-b503-429f8a0d0f73', `invalid AP response from ${response.url}: no content-type header`, true);
}
if (
contentType.startsWith('application/ld+json') ||
@ -37,5 +36,5 @@ export function validateContentTypeSetAsJsonLD(response: Response): void {
) {
return;
}
throw new Error(`invalid content type of JSON LD - content type is not application/ld+json or application/json: ${response.url}`);
throw new IdentifiableError('4bf8f36b-4d33-4ac9-ad76-63fa11f354e9', `invalid AP response from ${response.url}: content type "${contentType}" is not application/ld+json or application/json`);
}

4
packages/backend/src/core/activitypub/models/ApImageService.ts
View file

@ -18,7 +18,7 @@ import type { Config } from '@/config.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { ApResolverService } from '../ApResolverService.js';
import { ApLoggerService } from '../ApLoggerService.js';
import { isDocument, type IObject } from '../type.js';
import { getNullableApId, isDocument, type IObject } from '../type.js';
@Injectable()
export class ApImageService {
@ -48,7 +48,7 @@ export class ApImageService {
public async createImage(actor: MiRemoteUser, value: string | IObject): Promise<MiDriveFile | null> {
// 投稿者が凍結されていたらスキップ
if (actor.isSuspended) {
throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `actor has been suspended: ${actor.uri}`);
throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `failed to create image ${getNullableApId(value)}: actor ${actor.id} has been suspended`);
}
const image = await this.apResolverService.createResolver().resolve(value);

129
packages/backend/src/core/activitypub/models/ApNoteService.ts
View file

@ -26,7 +26,8 @@ import { bindThis } from '@/decorators.js';
import { checkHttps } from '@/misc/check-https.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { isRetryableError } from '@/misc/is-retryable-error.js';
import { getOneApId, getApId, validPost, isEmoji, getApType, isApObject, isDocument, IApDocument } from '../type.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import { getOneApId, getApId, validPost, isEmoji, getApType, isApObject, isDocument, IApDocument, isLink } from '../type.js';
import { ApLoggerService } from '../ApLoggerService.js';
import { ApMfmService } from '../ApMfmService.js';
import { ApDbResolverService } from '../ApDbResolverService.js';
@ -100,29 +101,29 @@ export class ApNoteService {
const apType = getApType(object);
if (apType == null || !validPost.includes(apType)) {
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: invalid object type ${apType ?? 'undefined'}`);
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note from ${uri}: invalid object type ${apType ?? 'undefined'}`);
}
if (object.id && this.utilityService.extractDbHost(object.id) !== expectHost) {
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: id has different host. expected: ${expectHost}, actual: ${this.utilityService.extractDbHost(object.id)}`);
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note from ${uri}: id has different host. expected: ${expectHost}, actual: ${this.utilityService.extractDbHost(object.id)}`);
}
const actualHost = object.attributedTo && this.utilityService.extractDbHost(getOneApId(object.attributedTo));
if (object.attributedTo && actualHost !== expectHost) {
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: attributedTo has different host. expected: ${expectHost}, actual: ${actualHost}`);
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note from ${uri}: attributedTo has different host. expected: ${expectHost}, actual: ${actualHost}`);
}
if (object.published && !this.idService.isSafeT(new Date(object.published).valueOf())) {
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', 'invalid Note: published timestamp is malformed');
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', 'invalid Note from ${uri}: published timestamp is malformed');
}
if (actor) {
const attribution = (object.attributedTo) ? getOneApId(object.attributedTo) : actor.uri;
if (attribution !== actor.uri) {
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: attribution does not match the actor that send it. attribution: ${attribution}, actor: ${actor.uri}`);
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note from ${uri}: attribution does not match the actor that send it. attribution: ${attribution}, actor: ${actor.uri}`);
}
if (user && attribution !== user.uri) {
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: updated attribution does not match original attribution. updated attribution: ${user.uri}, original attribution: ${attribution}`);
return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note from ${uri}: updated attribution does not match original attribution. updated attribution: ${user.uri}, original attribution: ${attribution}`);
}
}
@ -161,7 +162,7 @@ export class ApNoteService {
const entryUri = getApId(value);
const err = this.validateNote(object, entryUri, actor);
if (err) {
this.logger.error(err.message, {
this.logger.error(`Error creating note: ${renderInlineError(err)}`, {
resolver: { history: resolver.getHistory() },
value,
object,
@ -174,11 +175,11 @@ export class ApNoteService {
this.logger.debug(`Note fetched: ${JSON.stringify(note, null, 2)}`);
if (note.id == null) {
throw new UnrecoverableError(`Refusing to create note without id: ${entryUri}`);
throw new UnrecoverableError(`failed to create note ${entryUri}: missing ID`);
}
if (!checkHttps(note.id)) {
throw new UnrecoverableError(`unexpected schema of note.id ${note.id} in ${entryUri}`);
throw new UnrecoverableError(`failed to create note ${entryUri}: unexpected schema`);
}
const url = this.apUtilityService.findBestObjectUrl(note);
@ -187,7 +188,7 @@ export class ApNoteService {
// 投稿者をフェッチ
if (note.attributedTo == null) {
throw new UnrecoverableError(`invalid note.attributedTo ${note.attributedTo} in ${entryUri}`);
throw new UnrecoverableError(`failed to create note: ${entryUri}: missing attributedTo`);
}
const uri = getOneApId(note.attributedTo);
@ -196,7 +197,7 @@ export class ApNoteService {
// eslint-disable-next-line no-param-reassign
actor ??= await this.apPersonService.fetchPerson(uri) as MiRemoteUser | undefined;
if (actor && actor.isSuspended) {
throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `actor ${uri} has been suspended: ${entryUri}`);
throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `failed to create note ${entryUri}: actor ${uri} has been suspended`);
}
const apMentions = await this.apMentionService.extractApMentions(note.tag, resolver);
@ -223,7 +224,7 @@ export class ApNoteService {
*/
const hasProhibitedWords = this.noteCreateService.checkProhibitedWordsContain({ cw, text, pollChoices: poll?.choices });
if (hasProhibitedWords) {
throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140af99', `Note contains prohibited words: ${entryUri}`);
throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140af99', `failed to create note ${entryUri}: contains prohibited words`);
}
//#endregion
@ -232,7 +233,7 @@ export class ApNoteService {
// 解決した投稿者が凍結されていたらスキップ
if (actor.isSuspended) {
throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `actor has been suspended: ${entryUri}`);
throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `failed to create note ${entryUri}: actor ${actor.id} has been suspended`);
}
const noteAudience = await this.apAudienceService.parseAudience(actor, note.to, note.cc, resolver);
@ -269,15 +270,15 @@ export class ApNoteService {
? await this.resolveNote(note.inReplyTo, { resolver })
.then(x => {
if (x == null) {
this.logger.warn('Specified inReplyTo, but not found');
throw new Error(`could not fetch inReplyTo ${note.inReplyTo} for note ${entryUri}`);
this.logger.warn(`Specified inReplyTo "${note.inReplyTo}", but not found`);
throw new IdentifiableError('1ebf0a96-2769-4973-a6c2-3dcbad409dff', `failed to create note ${entryUri}: could not fetch inReplyTo ${note.inReplyTo}`, true);
}
return x;
})
.catch(async err => {
this.logger.warn(`error ${err.statusCode ?? err} fetching inReplyTo ${note.inReplyTo} for note ${entryUri}`);
throw err;
this.logger.warn(`error ${renderInlineError(err)} fetching inReplyTo ${note.inReplyTo} for note ${entryUri}`);
throw new IdentifiableError('1ebf0a96-2769-4973-a6c2-3dcbad409dff', `failed to create note ${entryUri}: could not fetch inReplyTo ${note.inReplyTo}`, true, err);
})
: null;
@ -285,6 +286,13 @@ export class ApNoteService {
const quote = await this.getQuote(note, entryUri, resolver);
const processErrors = quote === null ? ['quoteUnavailable'] : null;
if (reply && reply.userHost == null && reply.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot reply to local-only note');
}
if (quote && quote.userHost == null && quote.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot quote a local-only note');
}
// vote
if (reply && reply.hasPoll) {
const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id });
@ -341,7 +349,7 @@ export class ApNoteService {
this.logger.info('The note is already inserted while creating itself, reading again');
const duplicate = await this.fetchNote(value);
if (!duplicate) {
throw new Error(`The note creation failed with duplication error even when there is no duplication: ${entryUri}`);
throw new IdentifiableError('39c328e1-e829-458b-bfc9-65dcd513d1f8', `failed to create note ${entryUri}: the note creation failed with duplication error even when there is no duplication. This is likely a bug.`);
}
return duplicate;
}
@ -355,45 +363,39 @@ export class ApNoteService {
const noteUri = getApId(value);
// URIがこのサーバーを指しているならスキップ
if (noteUri.startsWith(this.config.url + '/')) throw new UnrecoverableError(`uri points local: ${noteUri}`);
if (this.utilityService.isUriLocal(noteUri)) {
throw new UnrecoverableError(`failed to update note ${noteUri}: uri is local`);
}
//#region このサーバーに既に登録されているか
const updatedNote = await this.notesRepository.findOneBy({ uri: noteUri });
if (updatedNote == null) throw new Error(`Note is not registered (no note): ${noteUri}`);
if (updatedNote == null) throw new UnrecoverableError(`failed to update note ${noteUri}: note does not exist`);
const user = await this.usersRepository.findOneBy({ id: updatedNote.userId }) as MiRemoteUser | null;
if (user == null) throw new Error(`Note is not registered (no user): ${noteUri}`);
if (user == null) throw new UnrecoverableError(`failed to update note ${noteUri}: user does not exist`);
// eslint-disable-next-line no-param-reassign
if (resolver == null) resolver = this.apResolverService.createResolver();
resolver ??= this.apResolverService.createResolver();
const object = await resolver.resolve(value);
const entryUri = getApId(value);
const err = this.validateNote(object, entryUri, actor, user);
if (err) {
this.logger.error(err.message, {
resolver: { history: resolver.getHistory() },
value,
object,
});
this.logger.error(`Failed to update note ${noteUri}: ${renderInlineError(err)}`);
throw err;
}
// `validateNote` checks that the actor and user are one and the same
// eslint-disable-next-line no-param-reassign
actor ??= user;
const note = object as IPost;
this.logger.debug(`Note fetched: ${JSON.stringify(note, null, 2)}`);
if (note.id == null) {
throw new UnrecoverableError(`Refusing to update note without id: ${noteUri}`);
throw new UnrecoverableError(`failed to update note ${entryUri}: missing ID`);
}
if (!checkHttps(note.id)) {
throw new UnrecoverableError(`unexpected schema of note.id ${note.id} in ${noteUri}`);
throw new UnrecoverableError(`failed to update note ${entryUri}: unexpected schema`);
}
const url = this.apUtilityService.findBestObjectUrl(note);
@ -401,7 +403,7 @@ export class ApNoteService {
this.logger.info(`Creating the Note: ${note.id}`);
if (actor.isSuspended) {
throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `actor ${actor.id} has been suspended: ${noteUri}`);
throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `failed to update note ${entryUri}: actor ${actor.id} has been suspended`);
}
const apMentions = await this.apMentionService.extractApMentions(note.tag, resolver);
@ -428,7 +430,7 @@ export class ApNoteService {
*/
const hasProhibitedWords = this.noteCreateService.checkProhibitedWordsContain({ cw, text, pollChoices: poll?.choices });
if (hasProhibitedWords) {
throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140af99', `Note contains prohibited words: ${noteUri}`);
throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140af99', `failed to update note ${noteUri}: contains prohibited words`);
}
//#endregion
@ -466,15 +468,15 @@ export class ApNoteService {
? await this.resolveNote(note.inReplyTo, { resolver })
.then(x => {
if (x == null) {
this.logger.warn('Specified inReplyTo, but not found');
throw new Error(`could not fetch inReplyTo ${note.inReplyTo} for note ${entryUri}`);
this.logger.warn(`Specified inReplyTo "${note.inReplyTo}", but not found`);
throw new IdentifiableError('1ebf0a96-2769-4973-a6c2-3dcbad409dff', `failed to update note ${entryUri}: could not fetch inReplyTo ${note.inReplyTo}`, true);
}
return x;
})
.catch(async err => {
this.logger.warn(`error ${err.statusCode ?? err} fetching inReplyTo ${note.inReplyTo} for note ${entryUri}`);
throw err;
this.logger.warn(`error ${renderInlineError(err)} fetching inReplyTo ${note.inReplyTo} for note ${entryUri}`);
throw new IdentifiableError('1ebf0a96-2769-4973-a6c2-3dcbad409dff', `failed to update note ${entryUri}: could not fetch inReplyTo ${note.inReplyTo}`, true, err);
})
: null;
@ -482,6 +484,10 @@ export class ApNoteService {
const quote = await this.getQuote(note, entryUri, resolver);
const processErrors = quote === null ? ['quoteUnavailable'] : null;
if (quote && quote.userHost == null && quote.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot quote a local-only note');
}
// vote
if (reply && reply.hasPoll) {
const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id });
@ -538,7 +544,7 @@ export class ApNoteService {
this.logger.info('The note is already inserted while creating itself, reading again');
const duplicate = await this.fetchNote(value);
if (!duplicate) {
throw new Error(`The note creation failed with duplication error even when there is no duplication: ${noteUri}`);
throw new IdentifiableError('39c328e1-e829-458b-bfc9-65dcd513d1f8', `failed to update note ${entryUri}: the note update failed with duplication error even when there is no duplication. This is likely a bug.`);
}
return duplicate;
}
@ -555,8 +561,7 @@ export class ApNoteService {
const uri = getApId(value);
if (!this.utilityService.isFederationAllowedUri(uri)) {
// TODO convert to identifiable error
throw new StatusError(`blocked host: ${uri}`, 451, 'blocked host');
throw new IdentifiableError('04620a7e-044e-45ce-b72c-10e1bdc22e69', `failed to resolve note ${uri}: host is blocked`);
}
//#region このサーバーに既に登録されていたらそれを返す
@ -566,8 +571,7 @@ export class ApNoteService {
// Bail if local URI doesn't exist
if (this.utilityService.isUriLocal(uri)) {
// TODO convert to identifiable error
throw new StatusError(`cannot resolve local note: ${uri}`, 400, 'cannot resolve local note');
throw new IdentifiableError('cbac7358-23f2-4c70-833e-cffb4bf77913', `failed to resolve note ${uri}: URL is local and does not exist`);
}
const unlock = await this.appLockService.getApLock(uri);
@ -653,9 +657,29 @@ export class ApNoteService {
*/
private async getQuote(note: IPost, entryUri: string, resolver: Resolver): Promise<MiNote | null | undefined> {
const quoteUris = new Set<string>();
if (note._misskey_quote) quoteUris.add(note._misskey_quote);
if (note.quoteUrl) quoteUris.add(note.quoteUrl);
if (note.quoteUri) quoteUris.add(note.quoteUri);
if (note._misskey_quote && typeof(note._misskey_quote as unknown) === 'string') quoteUris.add(note._misskey_quote);
if (note.quoteUrl && typeof(note.quoteUrl as unknown) === 'string') quoteUris.add(note.quoteUrl);
if (note.quoteUri && typeof(note.quoteUri as unknown) === 'string') quoteUris.add(note.quoteUri);
// https://codeberg.org/fediverse/fep/src/branch/main/fep/044f/fep-044f.md
if (note.quote && typeof(note.quote as unknown) === 'string') quoteUris.add(note.quote);
// https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md
const tags = toArray(note.tag).filter(tag => typeof(tag) === 'object' && isLink(tag));
for (const tag of tags) {
if (!tag.href || typeof (tag.href as unknown) !== 'string') continue;
const mediaTypes = toArray(tag.mediaType);
if (
!mediaTypes.includes('application/ld+json; profile="https://www.w3.org/ns/activitystreams"') &&
!mediaTypes.includes('application/activity+json')
) continue;
const rels = toArray(tag.rel);
if (!rels.includes('https://misskey-hub.net/ns#_misskey_quote')) continue;
quoteUris.add(tag.href);
}
// No quote, return undefined
if (quoteUris.size < 1) return undefined;
@ -674,18 +698,13 @@ export class ApNoteService {
const quote = await this.resolveNote(uri, { resolver });
if (quote == null) {
this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}": request error`);
this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}": fetch failed`);
return false;
}
return quote;
} catch (e) {
if (e instanceof Error) {
this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}":`, e);
} else {
this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}": ${e}`);
}
this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}": ${renderInlineError(e)}`);
return isRetryableError(e);
}
};

185
packages/backend/src/core/activitypub/models/ApPersonService.ts
View file

@ -7,7 +7,6 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import promiseLimit from 'promise-limit';
import { DataSource } from 'typeorm';
import { ModuleRef } from '@nestjs/core';
import { AbortError } from 'node-fetch';
import { UnrecoverableError } from 'bullmq';
import { DI } from '@/di-symbols.js';
import type { FollowingsRepository, InstancesRepository, MiMeta, UserProfilesRepository, UserPublickeysRepository, UsersRepository } from '@/models/_.js';
@ -44,6 +43,8 @@ import { AppLockService } from '@/core/AppLockService.js';
import { MemoryKVCache } from '@/misc/cache.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { verifyFieldLinks } from '@/misc/verify-field-link.js';
import { isRetryableError } from '@/misc/is-retryable-error.js';
import { renderInlineError } from '@/misc/render-inline-error.js';
import { getApId, getApType, isActor, isCollection, isCollectionOrOrderedCollection, isPropertyValue } from '../type.js';
import { extractApHashtags } from './tag.js';
import type { OnModuleInit } from '@nestjs/common';
@ -54,6 +55,7 @@ import type { ApLoggerService } from '../ApLoggerService.js';
import type { ApImageService } from './ApImageService.js';
import type { IActor, ICollection, IObject, IOrderedCollection } from '../type.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
const nameLength = 128;
const summaryLength = 2048;
@ -157,21 +159,21 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const expectHost = this.utilityService.punyHostPSLDomain(uri);
if (!isActor(x)) {
throw new UnrecoverableError(`invalid Actor type '${x.type}' in ${uri}`);
throw new UnrecoverableError(`invalid Actor ${uri}: unknown type '${x.type}'`);
}
if (!(typeof x.id === 'string' && x.id.length > 0)) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong id type`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong id type`);
}
if (!(typeof x.inbox === 'string' && x.inbox.length > 0)) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox type`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong inbox type`);
}
this.apUtilityService.assertApUrl(x.inbox);
const inboxHost = this.utilityService.punyHostPSLDomain(x.inbox);
if (inboxHost !== expectHost) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox ${inboxHost}`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong inbox host ${inboxHost}`);
}
const sharedInboxObject = x.sharedInbox ?? (x.endpoints ? x.endpoints.sharedInbox : undefined);
@ -179,7 +181,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const sharedInbox = getApId(sharedInboxObject);
this.apUtilityService.assertApUrl(sharedInbox);
if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHostPSLDomain(sharedInbox) === expectHost)) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong shared inbox ${sharedInbox}`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong shared inbox ${sharedInbox}`);
}
}
@ -190,7 +192,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
if (typeof collectionUri === 'string' && collectionUri.length > 0) {
this.apUtilityService.assertApUrl(collectionUri);
if (this.utilityService.punyHostPSLDomain(collectionUri) !== expectHost) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong ${collection} ${collectionUri}`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong ${collection} host ${collectionUri}`);
}
} else if (collectionUri != null) {
throw new UnrecoverableError(`invalid Actor ${uri}: wrong ${collection} type`);
@ -199,7 +201,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
}
if (!(typeof x.preferredUsername === 'string' && x.preferredUsername.length > 0 && x.preferredUsername.length <= 128 && /^\w([\w-.]*\w)?$/.test(x.preferredUsername))) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong username`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong username`);
}
// These fields are only informational, and some AP software allows these
@ -207,7 +209,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
// we can at least see these users and their activities.
if (x.name) {
if (!(typeof x.name === 'string' && x.name.length > 0)) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong name`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong name`);
}
x.name = truncate(x.name, nameLength);
} else if (x.name === '') {
@ -216,24 +218,24 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
}
if (x.summary) {
if (!(typeof x.summary === 'string' && x.summary.length > 0)) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong summary`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong summary`);
}
x.summary = truncate(x.summary, summaryLength);
}
const idHost = this.utilityService.punyHostPSLDomain(x.id);
if (idHost !== expectHost) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong id ${x.id}`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong id ${x.id}`);
}
if (x.publicKey) {
if (typeof x.publicKey.id !== 'string') {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong publicKey.id type`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong publicKey.id type`);
}
const publicKeyIdHost = this.utilityService.punyHostPSLDomain(x.publicKey.id);
if (publicKeyIdHost !== expectHost) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong publicKey.id ${x.publicKey.id}`);
throw new UnrecoverableError(`invalid Actor ${uri}: wrong publicKey.id ${x.publicKey.id}`);
}
}
@ -271,8 +273,6 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
}
private async resolveAvatarAndBanner(user: MiRemoteUser, icon: any, image: any, bgimg: any): Promise<Partial<Pick<MiRemoteUser, 'avatarId' | 'bannerId' | 'backgroundId' | 'avatarUrl' | 'bannerUrl' | 'backgroundUrl' | 'avatarBlurhash' | 'bannerBlurhash' | 'backgroundBlurhash'>>> {
if (user == null) throw new Error('failed to create user: user is null');
const [avatar, banner, background] = await Promise.all([icon, image, bgimg].map(img => {
// icon and image may be arrays
// see https://www.w3.org/TR/activitystreams-vocabulary/#dfn-icon
@ -325,12 +325,11 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
*/
@bindThis
public async createPerson(uri: string, resolver?: Resolver): Promise<MiRemoteUser> {
if (typeof uri !== 'string') throw new UnrecoverableError(`uri is not string: ${uri}`);
if (typeof uri !== 'string') throw new UnrecoverableError(`failed to create user ${uri}: input is not string`);
const host = this.utilityService.punyHost(uri);
if (host === this.utilityService.toPuny(this.config.host)) {
// TODO convert to unrecoverable error
throw new StatusError(`cannot resolve local user: ${uri}`, 400, 'cannot resolve local user');
throw new UnrecoverableError(`failed to create user ${uri}: URI is local`);
}
return await this._createPerson(uri, resolver);
@ -340,8 +339,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const uri = getApId(value);
const host = this.utilityService.punyHost(uri);
// eslint-disable-next-line no-param-reassign
if (resolver == null) resolver = this.apResolverService.createResolver();
resolver ??= this.apResolverService.createResolver();
const object = await resolver.resolve(value);
const person = this.validateActor(object, uri);
@ -356,14 +354,16 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const [followingVisibility, followersVisibility] = await Promise.all(
[
this.isPublicCollection(person.following, resolver),
this.isPublicCollection(person.followers, resolver),
this.isPublicCollection(person.following, resolver, uri),
this.isPublicCollection(person.followers, resolver, uri),
].map((p): Promise<'public' | 'private'> => p
.then(isPublic => isPublic ? 'public' : 'private')
.catch(err => {
if (!(err instanceof StatusError) || err.isRetryable) {
this.logger.error('error occurred while fetching following/followers collection', { stack: err });
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.error(`error occurred while fetching following/followers collection: ${renderInlineError(err)}`);
}
return 'private';
}),
),
@ -372,7 +372,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const bday = person['vcard:bday']?.match(/^\d{4}-\d{2}-\d{2}/);
if (person.id == null) {
throw new UnrecoverableError(`Refusing to create person without id: ${uri}`);
throw new UnrecoverableError(`failed to create user ${uri}: missing ID`);
}
const url = this.apUtilityService.findBestObjectUrl(person);
@ -387,16 +387,27 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const emojis = await this.apNoteService.extractEmojis(person.tag ?? [], host)
.then(_emojis => _emojis.map(emoji => emoji.name))
.catch(err => {
this.logger.error('error occurred while fetching user emojis', { stack: err });
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.error(`error occurred while fetching user emojis: ${renderInlineError(err)}`);
}
return [];
});
//#endregion
//#region resolve counts
const _resolver = resolver ?? this.apResolverService.createResolver();
const outboxcollection = await _resolver.resolveCollection(person.outbox).catch(() => { return null; });
const followerscollection = await _resolver.resolveCollection(person.followers!).catch(() => { return null; });
const followingcollection = await _resolver.resolveCollection(person.following!).catch(() => { return null; });
const outboxCollection = person.outbox
? await resolver.resolveCollection(person.outbox, true, uri).catch(() => { return null; })
: null;
const followersCollection = person.followers
? await resolver.resolveCollection(person.followers, true, uri).catch(() => { return null; })
: null;
const followingCollection = person.following
? await resolver.resolveCollection(person.following, true, uri).catch(() => { return null; })
: null;
// Register the instance first, to avoid FK errors
await this.federatedInstanceService.fetchOrRegister(host);
try {
// Start transaction
@ -423,9 +434,9 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
host,
inbox: person.inbox,
sharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null,
notesCount: outboxcollection?.totalItems ?? 0,
followersCount: followerscollection?.totalItems ?? 0,
followingCount: followingcollection?.totalItems ?? 0,
notesCount: outboxCollection?.totalItems ?? 0,
followersCount: followersCollection?.totalItems ?? 0,
followingCount: followingCollection?.totalItems ?? 0,
followersUri: person.followers ? getApId(person.followers) : undefined,
featured: person.featured ? getApId(person.featured) : undefined,
uri: person.id,
@ -437,6 +448,11 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
makeNotesFollowersOnlyBefore: (person as any).makeNotesFollowersOnlyBefore ?? null,
makeNotesHiddenBefore: (person as any).makeNotesHiddenBefore ?? null,
emojis,
attributionDomains: Array.isArray(person.attributionDomains)
? person.attributionDomains
.filter((a: unknown) => typeof(a) === 'string' && a.length > 0 && a.length <= 128)
.slice(0, 32)
: [],
})) as MiRemoteUser;
let _description: string | null = null;
@ -480,7 +496,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
user = u as MiRemoteUser;
publicKey = await this.userPublickeysRepository.findOneBy({ userId: user.id });
} else {
this.logger.error(e instanceof Error ? e : new Error(e as string));
this.logger.error('Error creating Person:', e instanceof Error ? e : new Error(e as string));
throw e;
}
}
@ -520,11 +536,19 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
// Register to the cache
this.cacheService.uriPersonCache.set(user.uri, user);
} catch (err) {
this.logger.error('error occurred while fetching user avatar/banner', { stack: err });
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.error(`error occurred while fetching user avatar/banner: ${renderInlineError(err)}`);
}
}
//#endregion
await this.updateFeatured(user.id, resolver).catch(err => this.logger.error(err));
await this.updateFeatured(user.id, resolver).catch(err => {
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.error(`Error updating featured notes: ${renderInlineError(err)}`);
}
});
return user;
}
@ -541,7 +565,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
*/
@bindThis
public async updatePerson(uri: string, resolver?: Resolver | null, hint?: IObject, movePreventUris: string[] = []): Promise<string | void> {
if (typeof uri !== 'string') throw new UnrecoverableError('uri is not string');
if (typeof uri !== 'string') throw new UnrecoverableError(`failed to update user ${uri}: input is not string`);
// URIがこのサーバーを指しているならスキップ
if (this.utilityService.isUriLocal(uri)) return;
@ -561,8 +585,11 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
this.logger.info(`Updating the Person: ${person.id}`);
// カスタム絵文字取得
const emojis = await this.apNoteService.extractEmojis(person.tag ?? [], exist.host).catch(e => {
this.logger.info(`extractEmojis: ${e}`);
const emojis = await this.apNoteService.extractEmojis(person.tag ?? [], exist.host).catch(err => {
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.error(`error occurred while fetching user emojis: ${renderInlineError(err)}`);
}
return [];
});
@ -574,16 +601,18 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const [followingVisibility, followersVisibility] = await Promise.all(
[
this.isPublicCollection(person.following, resolver),
this.isPublicCollection(person.followers, resolver),
this.isPublicCollection(person.following, resolver, exist.uri),
this.isPublicCollection(person.followers, resolver, exist.uri),
].map((p): Promise<'public' | 'private' | undefined> => p
.then(isPublic => isPublic ? 'public' : 'private')
.catch(err => {
if (!(err instanceof StatusError) || err.isRetryable) {
this.logger.error('error occurred while fetching following/followers collection', { stack: err });
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.error(`error occurred while fetching following/followers collection: ${renderInlineError(err)}`);
// Do not update the visibility on transient errors.
return undefined;
}
return 'private';
}),
),
@ -592,7 +621,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const bday = person['vcard:bday']?.match(/^\d{4}-\d{2}-\d{2}/);
if (person.id == null) {
throw new UnrecoverableError(`Refusing to update person without id: ${uri}`);
throw new UnrecoverableError(`failed to update user ${uri}: missing ID`);
}
const url = this.apUtilityService.findBestObjectUrl(person);
@ -620,7 +649,20 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
// We use "!== false" to handle incorrect types, missing / null values, and "default to true" logic.
hideOnlineStatus: person.hideOnlineStatus !== false,
isExplorable: person.discoverable !== false,
...(await this.resolveAvatarAndBanner(exist, person.icon, person.image, person.backgroundUrl).catch(() => ({}))),
attributionDomains: Array.isArray(person.attributionDomains)
? person.attributionDomains
.filter((a: unknown) => typeof(a) === 'string' && a.length > 0 && a.length <= 128)
.slice(0, 32)
: [],
...(await this.resolveAvatarAndBanner(exist, person.icon, person.image, person.backgroundUrl).catch(err => {
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.error(`error occurred while fetching user avatar/banner: ${renderInlineError(err)}`);
}
// Can't return null or destructuring operator will break
return {};
})),
} as Partial<MiRemoteUser> & Pick<MiRemoteUser, 'isBot' | 'isCat' | 'speakAsCat' | 'isLocked' | 'movedToUri' | 'alsoKnownAs' | 'isExplorable'>;
const moving = ((): boolean => {
@ -699,12 +741,24 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
this.hashtagService.updateUsertags(exist, tags);
// 該当ユーザーが既にフォロワーになっていた場合はFollowingもアップデートする
await this.followingsRepository.update(
{ followerId: exist.id },
{ followerSharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null },
);
if (exist.inbox !== person.inbox || exist.sharedInbox !== (person.sharedInbox ?? person.endpoints?.sharedInbox)) {
await this.followingsRepository.update(
{ followerId: exist.id },
{
followerInbox: person.inbox,
followerSharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null,
},
);
await this.updateFeatured(exist.id, resolver).catch(err => this.logger.error(err));
await this.cacheService.refreshFollowRelationsFor(exist.id);
}
await this.updateFeatured(exist.id, resolver).catch(err => {
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.error(`Error updating featured notes: ${renderInlineError(err)}`);
}
});
const updated = { ...exist, ...updates };
@ -743,8 +797,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const uri = getApId(value);
if (!this.utilityService.isFederationAllowedUri(uri)) {
// TODO convert to identifiable error
throw new StatusError(`blocked host: ${uri}`, 451, 'blocked host');
throw new IdentifiableError('590719b3-f51f-48a9-8e7d-6f559ad00e5d', `failed to resolve person ${uri}: host is blocked`);
}
//#region このサーバーに既に登録されていたらそれを返す
@ -754,8 +807,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
// Bail if local URI doesn't exist
if (this.utilityService.isUriLocal(uri)) {
// TODO convert to identifiable error
throw new StatusError(`cannot resolve local person: ${uri}`, 400, 'cannot resolve local person');
throw new IdentifiableError('efb573fd-6b9e-4912-9348-a02f5603df4f', `failed to resolve person ${uri}: URL is local and does not exist`);
}
const unlock = await this.appLockService.getApLock(uri);
@ -799,16 +851,17 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const _resolver = resolver ?? this.apResolverService.createResolver();
// Resolve to (Ordered)Collection Object
const collection = await _resolver.resolveCollection(user.featured).catch(err => {
if (err instanceof AbortError || err instanceof StatusError) {
this.logger.warn(`Failed to update featured notes: ${err.name}: ${err.message}`);
} else {
this.logger.error('Failed to update featured notes:', err);
const collection = user.featured ? await _resolver.resolveCollection(user.featured, true, user.uri).catch(err => {
// Permanent error implies hidden or inaccessible, which is a normal thing.
if (isRetryableError(err)) {
this.logger.warn(`Failed to update featured notes: ${renderInlineError(err)}`);
}
});
return null;
}) : null;
if (!collection) return;
if (!isCollectionOrOrderedCollection(collection)) throw new UnrecoverableError(`featured ${user.featured} is not Collection or OrderedCollection in ${user.uri}`);
if (!isCollectionOrOrderedCollection(collection)) throw new UnrecoverableError(`failed to update user ${user.uri}: featured ${user.featured} is not Collection or OrderedCollection`);
// Resolve to Object(may be Note) arrays
const unresolvedItems = isCollection(collection) ? collection.items : collection.orderedItems;
@ -891,11 +944,13 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
}
@bindThis
private async isPublicCollection(collection: string | ICollection | IOrderedCollection | undefined, resolver: Resolver): Promise<boolean> {
private async isPublicCollection(collection: string | ICollection | IOrderedCollection | undefined, resolver: Resolver, sentFrom: string): Promise<boolean> {
if (collection) {
const resolved = await resolver.resolveCollection(collection);
if (resolved.first || (resolved as ICollection).items || (resolved as IOrderedCollection).orderedItems) {
return true;
const resolved = await resolver.resolveCollection(collection, true, sentFrom).catch(() => null);
if (resolved) {
if (resolved.first || (resolved as ICollection).items || (resolved as IOrderedCollection).orderedItems) {
return true;
}
}
}

1
packages/backend/src/core/activitypub/models/ApQuestionService.ts
View file

@ -93,7 +93,6 @@ export class ApQuestionService {
// eslint-disable-next-line no-param-reassign
if (resolver == null) resolver = this.apResolverService.createResolver();
const question = await resolver.resolve(value);
this.logger.debug(`fetched question: ${JSON.stringify(question, null, 2)}`);
if (!isQuestion(question)) throw new UnrecoverableError(`object ${getApType(question)} is not a Question: ${uri}`);

108
packages/backend/src/core/activitypub/type.ts
View file

@ -35,6 +35,7 @@ export interface IObject {
mediaType?: string;
url?: ApObject | string;
href?: string;
rel?: string | string[];
tag?: IObject | IObject[];
sensitive?: boolean;
}
@ -43,6 +44,28 @@ export interface IObjectWithId extends IObject {
id: string;
}
export function isObjectWithId(object: IObject): object is IObjectWithId {
return typeof(object.id) === 'string';
}
export interface IAnonymousObject extends IObject {
id: undefined;
}
export function isAnonymousObject(object: IObject): object is IAnonymousObject {
return object.id === undefined;
}
export interface ILink extends IObject {
'@context'?: string | string[] | Obj | Obj[];
type: 'Link' | 'Mention';
href: string;
}
export const isLink = (object: IObject): object is ILink =>
(getApType(object) === 'Link' || getApType(object) === 'Link') &&
typeof object.href === 'string';
/**
* Get array of ActivityStreams Objects id
*/
@ -63,24 +86,34 @@ export function getOneApId(value: ApObject): string {
/**
* Get ActivityStreams Object id
*/
export function getApId(value: string | IObject | [string | IObject]): string {
// eslint-disable-next-line no-param-reassign
value = fromTuple(value);
export function getApId(value: string | IObject | [string | IObject], sourceForLogs?: string): string {
const id = getNullableApId(value);
if (typeof value === 'string') return value;
if (typeof value.id === 'string') return value.id;
throw new IdentifiableError('ad2dc287-75c1-44c4-839d-3d2e64576675', `invalid AP object ${value}: missing id`);
if (id == null) {
const message = sourceForLogs
? `invalid AP object ${value} (sent from ${sourceForLogs}): missing id`
: `invalid AP object ${value}: missing id`;
throw new IdentifiableError('ad2dc287-75c1-44c4-839d-3d2e64576675', message);
}
return id;
}
/**
* Get ActivityStreams Object id, or null if not present
*/
export function getNullableApId(value: string | IObject | [string | IObject]): string | null {
// eslint-disable-next-line no-param-reassign
value = fromTuple(value);
export function getNullableApId(source: string | IObject | [string | IObject]): string | null {
const value: unknown = fromTuple(source);
if (value != null) {
if (typeof value === 'string') {
return value;
}
if (typeof (value) === 'object' && 'id' in value && typeof (value.id) === 'string') {
return value.id;
}
}
if (typeof value === 'string') return value;
if (typeof value.id === 'string') return value.id;
return null;
}
@ -125,48 +158,46 @@ export interface IActivity extends IObject {
};
}
export interface ICollection extends IObject {
export interface CollectionBase extends IObject {
totalItems?: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
partOf?: IObject | string;
next?: IObject | string;
prev?: IObject | string;
items?: ApObject;
orderedItems?: ApObject;
}
export interface ICollection extends CollectionBase {
type: 'Collection';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
items?: ApObject;
orderedItems?: undefined;
}
export interface IOrderedCollection extends IObject {
export interface IOrderedCollection extends CollectionBase {
type: 'OrderedCollection';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
items?: undefined;
orderedItems?: ApObject;
}
export interface ICollectionPage extends IObject {
export interface ICollectionPage extends CollectionBase {
type: 'CollectionPage';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
partOf?: IObject | string;
next?: IObject | string;
prev?: IObject | string;
items?: ApObject;
orderedItems?: undefined;
}
export interface IOrderedCollectionPage extends IObject {
export interface IOrderedCollectionPage extends CollectionBase {
type: 'OrderedCollectionPage';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
partOf?: IObject | string;
next?: IObject | string;
prev?: IObject | string;
items?: undefined;
orderedItems?: ApObject;
}
export type AnyCollection = ICollection | IOrderedCollection | ICollectionPage | IOrderedCollectionPage;
export const validPost = ['Note', 'Question', 'Article', 'Audio', 'Document', 'Image', 'Page', 'Video', 'Event'];
export const isPost = (object: IObject): object is IPost => {
@ -184,6 +215,7 @@ export interface IPost extends IObject {
_misskey_content?: string;
quoteUrl?: string;
quoteUri?: string;
quote?: string;
updated?: string;
}
@ -255,6 +287,7 @@ export interface IActor extends IObject {
enableRss?: boolean;
listenbrainz?: string;
backgroundUrl?: string;
attributionDomains?: string[];
}
export const isCollection = (object: IObject): object is ICollection =>
@ -269,7 +302,7 @@ export const isCollectionPage = (object: IObject): object is ICollectionPage =>
export const isOrderedCollectionPage = (object: IObject): object is IOrderedCollectionPage =>
getApType(object) === 'OrderedCollectionPage';
export const isCollectionOrOrderedCollection = (object: IObject): object is ICollection | IOrderedCollection =>
export const isCollectionOrOrderedCollection = (object: IObject): object is AnyCollection =>
isCollection(object) || isOrderedCollection(object) || isCollectionPage(object) || isOrderedCollectionPage(object);
export interface IApPropertyValue extends IObject {
@ -285,9 +318,8 @@ export const isPropertyValue = (object: IObject): object is IApPropertyValue =>
'value' in object &&
typeof object.value === 'string';
export interface IApMention extends IObject {
export interface IApMention extends ILink {
type: 'Mention';
href: string;
name: string;
}

24
packages/backend/src/core/chart/charts/federation.ts
View file

@ -44,10 +44,7 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
const suspendedInstancesQuery = this.instancesRepository.createQueryBuilder('instance')
.select('instance.host')
.where('instance.suspensionState != \'none\'');
// TODO optimization: replace these with exists()
const pubsubSubQuery = this.followingsRepository.createQueryBuilder('f')
.select('f.followerHost')
.where('f.followerHost IS NOT NULL');
@ -64,22 +61,25 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followeeHost)')
.where('following.followeeHost IS NOT NULL')
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followeeHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.innerJoin('following.followeeInstance', 'followeeInstance')
.andWhere('followeeInstance.suspensionState = \'none\'')
.andWhere('followeeInstance.isBlocked = false')
.getRawOne()
.then(x => parseInt(x.count, 10)),
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followerHost)')
.where('following.followerHost IS NOT NULL')
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followerHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere(`following.followerHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.innerJoin('following.followerInstance', 'followerInstance')
.andWhere('followerInstance.isBlocked = false')
.andWhere('followerInstance.suspensionState = \'none\'')
.getRawOne()
.then(x => parseInt(x.count, 10)),
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followeeHost)')
.where('following.followeeHost IS NOT NULL')
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followeeHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.innerJoin('following.followeeInstance', 'followeeInstance')
.andWhere('followeeInstance.isBlocked = false')
.andWhere('followeeInstance.suspensionState = \'none\'')
.andWhere(`following.followeeHost IN (${ pubsubSubQuery.getQuery() })`)
.setParameters(pubsubSubQuery.getParameters())
.getRawOne()
@ -87,7 +87,7 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
this.instancesRepository.createQueryBuilder('instance')
.select('COUNT(instance.id)')
.where(`instance.host IN (${ subInstancesQuery.getQuery() })`)
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || instance.host) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere('instance.isBlocked = false')
.andWhere('instance.suspensionState = \'none\'')
.andWhere('instance.isNotResponding = false')
.getRawOne()
@ -95,7 +95,7 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
this.instancesRepository.createQueryBuilder('instance')
.select('COUNT(instance.id)')
.where(`instance.host IN (${ pubInstancesQuery.getQuery() })`)
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || instance.host) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere('instance.isBlocked = false')
.andWhere('instance.suspensionState = \'none\'')
.andWhere('instance.isNotResponding = false')
.getRawOne()

19
packages/backend/src/core/chart/charts/per-user-following.ts
View file

@ -15,6 +15,7 @@ import Chart from '../core.js';
import { ChartLoggerService } from '../ChartLoggerService.js';
import { name, schema } from './entities/per-user-following.js';
import type { KVs } from '../core.js';
import { CacheService } from '@/core/CacheService.js';
/**
* ユーザーごとのフォローに関するチャート
@ -31,23 +32,25 @@ export default class PerUserFollowingChart extends Chart<typeof schema> { // esl
private appLockService: AppLockService,
private userEntityService: UserEntityService,
private chartLoggerService: ChartLoggerService,
private readonly cacheService: CacheService,
) {
super(db, (k) => appLockService.getChartInsertLock(k), chartLoggerService.logger, name, schema, true);
}
protected async tickMajor(group: string): Promise<Partial<KVs<typeof schema>>> {
const [
localFollowingsCount,
localFollowersCount,
remoteFollowingsCount,
remoteFollowersCount,
followees,
followers,
] = await Promise.all([
this.followingsRepository.countBy({ followerId: group, followeeHost: IsNull() }),
this.followingsRepository.countBy({ followeeId: group, followerHost: IsNull() }),
this.followingsRepository.countBy({ followerId: group, followeeHost: Not(IsNull()) }),
this.followingsRepository.countBy({ followeeId: group, followerHost: Not(IsNull()) }),
this.cacheService.userFollowingsCache.fetch(group).then(fs => Array.from(fs.values())),
this.cacheService.userFollowersCache.fetch(group).then(fs => Array.from(fs.values())),
]);
const localFollowingsCount = followees.reduce((sum, f) => sum + (f.followeeHost == null ? 1 : 0), 0);
const localFollowersCount = followers.reduce((sum, f) => sum + (f.followerHost == null ? 1 : 0), 0);
const remoteFollowingsCount = followees.reduce((sum, f) => sum + (f.followeeHost == null ? 0 : 1), 0);
const remoteFollowersCount = followers.reduce((sum, f) => sum + (f.followerHost == null ? 0 : 1), 0);
return {
'local.followings.total': localFollowingsCount,
'local.followers.total': localFollowersCount,

36
packages/backend/src/core/entities/AbuseUserReportEntityService.ts
View file

@ -5,13 +5,14 @@
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { AbuseUserReportsRepository } from '@/models/_.js';
import type { AbuseUserReportsRepository, InstancesRepository, MiInstance, MiUser } from '@/models/_.js';
import { awaitAll } from '@/misc/prelude/await-all.js';
import type { MiAbuseUserReport } from '@/models/AbuseUserReport.js';
import { bindThis } from '@/decorators.js';
import { IdService } from '@/core/IdService.js';
import type { Packed } from '@/misc/json-schema.js';
import { UserEntityService } from './UserEntityService.js';
import { InstanceEntityService } from './InstanceEntityService.js';
@Injectable()
export class AbuseUserReportEntityService {
@ -19,6 +20,10 @@ export class AbuseUserReportEntityService {
@Inject(DI.abuseUserReportsRepository)
private abuseUserReportsRepository: AbuseUserReportsRepository,
@Inject(DI.instancesRepository)
private instancesRepository: InstancesRepository,
private readonly instanceEntityService: InstanceEntityService,
private userEntityService: UserEntityService,
private idService: IdService,
) {
@ -30,11 +35,14 @@ export class AbuseUserReportEntityService {
hint?: {
packedReporter?: Packed<'UserDetailedNotMe'>,
packedTargetUser?: Packed<'UserDetailedNotMe'>,
packedTargetInstance?: Packed<'FederationInstance'>,
packedAssignee?: Packed<'UserDetailedNotMe'>,
},
me?: MiUser | null,
) {
const report = typeof src === 'object' ? src : await this.abuseUserReportsRepository.findOneByOrFail({ id: src });
// noinspection ES6MissingAwait
return await awaitAll({
id: report.id,
createdAt: this.idService.parse(report.id).date.toISOString(),
@ -43,13 +51,22 @@ export class AbuseUserReportEntityService {
reporterId: report.reporterId,
targetUserId: report.targetUserId,
assigneeId: report.assigneeId,
reporter: hint?.packedReporter ?? this.userEntityService.pack(report.reporter ?? report.reporterId, null, {
reporter: hint?.packedReporter ?? this.userEntityService.pack(report.reporter ?? report.reporterId, me, {
schema: 'UserDetailedNotMe',
}),
targetUser: hint?.packedTargetUser ?? this.userEntityService.pack(report.targetUser ?? report.targetUserId, null, {
targetUser: hint?.packedTargetUser ?? this.userEntityService.pack(report.targetUser ?? report.targetUserId, me, {
schema: 'UserDetailedNotMe',
}),
assignee: report.assigneeId ? hint?.packedAssignee ?? this.userEntityService.pack(report.assignee ?? report.assigneeId, null, {
// return hint, or pack by relation, or fetch and pack by id, or null
targetInstance: hint?.packedTargetInstance ?? (
report.targetUserInstance
? this.instanceEntityService.pack(report.targetUserInstance, me)
: report.targetUserHost
? this.instancesRepository.findOneBy({ host: report.targetUserHost }).then(instance => instance
? this.instanceEntityService.pack(instance, me)
: null)
: null),
assignee: report.assigneeId ? hint?.packedAssignee ?? this.userEntityService.pack(report.assignee ?? report.assigneeId, me, {
schema: 'UserDetailedNotMe',
}) : null,
forwarded: report.forwarded,
@ -61,21 +78,28 @@ export class AbuseUserReportEntityService {
@bindThis
public async packMany(
reports: MiAbuseUserReport[],
me?: MiUser | null,
) {
const _reporters = reports.map(({ reporter, reporterId }) => reporter ?? reporterId);
const _targetUsers = reports.map(({ targetUser, targetUserId }) => targetUser ?? targetUserId);
const _assignees = reports.map(({ assignee, assigneeId }) => assignee ?? assigneeId).filter(x => x != null);
const _userMap = await this.userEntityService.packMany(
[..._reporters, ..._targetUsers, ..._assignees],
null,
me,
{ schema: 'UserDetailedNotMe' },
).then(users => new Map(users.map(u => [u.id, u])));
const _targetInstances = reports
.map(({ targetUserInstance, targetUserHost }) => targetUserInstance ?? targetUserHost)
.filter((i): i is MiInstance | string => i != null);
const _instanceMap = await this.instanceEntityService.packMany(await this.instanceEntityService.fetchInstancesByHost(_targetInstances), me)
.then(instances => new Map(instances.map(i => [i.host, i])));
return Promise.all(
reports.map(report => {
const packedReporter = _userMap.get(report.reporterId);
const packedTargetUser = _userMap.get(report.targetUserId);
const packedTargetInstance = report.targetUserHost ? _instanceMap.get(report.targetUserHost) : undefined;
const packedAssignee = report.assigneeId != null ? _userMap.get(report.assigneeId) : undefined;
return this.pack(report, { packedReporter, packedTargetUser, packedAssignee });
return this.pack(report, { packedReporter, packedTargetUser, packedAssignee, packedTargetInstance }, me);
}),
);
}

36
packages/backend/src/core/entities/InstanceEntityService.ts
View file

@ -4,6 +4,7 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { In } from 'typeorm';
import type { Packed } from '@/misc/json-schema.js';
import type { MiInstance } from '@/models/Instance.js';
import { bindThis } from '@/decorators.js';
@ -11,7 +12,7 @@ import { UtilityService } from '@/core/UtilityService.js';
import { RoleService } from '@/core/RoleService.js';
import { MiUser } from '@/models/User.js';
import { DI } from '@/di-symbols.js';
import { MiMeta } from '@/models/_.js';
import type { InstancesRepository, MiMeta } from '@/models/_.js';
@Injectable()
export class InstanceEntityService {
@ -19,6 +20,9 @@ export class InstanceEntityService {
@Inject(DI.meta)
private meta: MiMeta,
@Inject(DI.instancesRepository)
private readonly instancesRepository: InstancesRepository,
private roleService: RoleService,
private utilityService: UtilityService,
@ -43,7 +47,7 @@ export class InstanceEntityService {
isNotResponding: instance.isNotResponding,
isSuspended: instance.suspensionState !== 'none',
suspensionState: instance.suspensionState,
isBlocked: this.utilityService.isBlockedHost(this.meta.blockedHosts, instance.host),
isBlocked: instance.isBlocked,
softwareName: instance.softwareName,
softwareVersion: instance.softwareVersion,
openRegistrations: instance.openRegistrations,
@ -51,8 +55,8 @@ export class InstanceEntityService {
description: instance.description,
maintainerName: instance.maintainerName,
maintainerEmail: instance.maintainerEmail,
isSilenced: this.utilityService.isSilencedHost(this.meta.silencedHosts, instance.host),
isMediaSilenced: this.utilityService.isMediaSilencedHost(this.meta.mediaSilencedHosts, instance.host),
isSilenced: instance.isSilenced,
isMediaSilenced: instance.isMediaSilenced,
iconUrl: instance.iconUrl,
faviconUrl: instance.faviconUrl,
themeColor: instance.themeColor,
@ -62,6 +66,7 @@ export class InstanceEntityService {
rejectReports: instance.rejectReports,
rejectQuotes: instance.rejectQuotes,
moderationNote: iAmModerator ? instance.moderationNote : null,
isBubbled: this.utilityService.isBubbledHost(instance.host),
};
}
@ -72,5 +77,28 @@ export class InstanceEntityService {
) {
return Promise.all(instances.map(x => this.pack(x, me)));
}
@bindThis
public async fetchInstancesByHost(instances: (MiInstance | MiInstance['host'])[]): Promise<MiInstance[]> {
const result: MiInstance[] = [];
const toFetch: string[] = [];
for (const instance of instances) {
if (typeof(instance) === 'string') {
toFetch.push(instance);
} else {
result.push(instance);
}
}
if (toFetch.length > 0) {
const fetched = await this.instancesRepository.findBy({
host: In(toFetch),
});
result.push(...fetched);
}
return result;
}
}

349
packages/backend/src/core/entities/NoteEntityService.ts
View file

@ -11,12 +11,13 @@ import type { Packed } from '@/misc/json-schema.js';
import { awaitAll } from '@/misc/prelude/await-all.js';
import type { MiUser } from '@/models/User.js';
import type { MiNote } from '@/models/Note.js';
import type { UsersRepository, NotesRepository, FollowingsRepository, PollsRepository, PollVotesRepository, NoteReactionsRepository, ChannelsRepository, MiMeta } from '@/models/_.js';
import type { UsersRepository, NotesRepository, FollowingsRepository, PollsRepository, PollVotesRepository, NoteReactionsRepository, ChannelsRepository, MiMeta, MiPollVote, MiPoll, MiChannel, MiFollowing } from '@/models/_.js';
import { bindThis } from '@/decorators.js';
import { DebounceLoader } from '@/misc/loader.js';
import { IdService } from '@/core/IdService.js';
import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js';
import { isPackedPureRenote } from '@/misc/is-renote.js';
import type { Config } from '@/config.js';
import type { OnModuleInit } from '@nestjs/common';
import type { CacheService } from '../CacheService.js';
import type { CustomEmojiService } from '../CustomEmojiService.js';
@ -25,13 +26,13 @@ import type { UserEntityService } from './UserEntityService.js';
import type { DriveFileEntityService } from './DriveFileEntityService.js';
// is-renote.tsとよしなにリンク
function isPureRenote(note: MiNote): note is MiNote & { renoteId: MiNote['id']; renote: MiNote } {
function isPureRenote(note: MiNote): note is MiNote & { renoteId: MiNote['id'] } {
return (
note.renote != null &&
note.reply == null &&
note.renoteId != null &&
note.replyId == null &&
note.text == null &&
note.cw == null &&
(note.fileIds == null || note.fileIds.length === 0) &&
note.fileIds.length === 0 &&
!note.hasPoll
);
}
@ -92,6 +93,9 @@ export class NoteEntityService implements OnModuleInit {
@Inject(DI.channelsRepository)
private channelsRepository: ChannelsRepository,
@Inject(DI.config)
private readonly config: Config,
//private userEntityService: UserEntityService,
//private driveFileEntityService: DriveFileEntityService,
//private customEmojiService: CustomEmojiService,
@ -128,7 +132,10 @@ export class NoteEntityService implements OnModuleInit {
}
@bindThis
public async hideNote(packedNote: Packed<'Note'>, meId: MiUser['id'] | null): Promise<void> {
public async hideNote(packedNote: Packed<'Note'>, meId: MiUser['id'] | null, hint?: {
myFollowing?: ReadonlyMap<string, unknown>,
myBlockers?: ReadonlySet<string>,
}): Promise<void> {
if (meId === packedNote.userId) return;
// TODO: isVisibleForMe を使うようにしても良さそう(型違うけど)
@ -184,14 +191,9 @@ export class NoteEntityService implements OnModuleInit {
} else if (packedNote.renote && (meId === packedNote.renote.userId)) {
hide = false;
} else {
// フォロワーかどうか
// TODO: 当関数呼び出しごとにクエリが走るのは重そうだからなんとかする
const isFollowing = await this.followingsRepository.exists({
where: {
followeeId: packedNote.userId,
followerId: meId,
},
});
const isFollowing = hint?.myFollowing
? hint.myFollowing.has(packedNote.userId)
: (await this.cacheService.userFollowingsCache.fetch(meId)).has(packedNote.userId);
hide = !isFollowing;
}
@ -207,7 +209,8 @@ export class NoteEntityService implements OnModuleInit {
}
if (!hide && meId && packedNote.userId !== meId) {
const isBlocked = (await this.cacheService.userBlockedCache.fetch(meId)).has(packedNote.userId);
const blockers = hint?.myBlockers ?? await this.cacheService.userBlockedCache.fetch(meId);
const isBlocked = blockers.has(packedNote.userId);
if (isBlocked) hide = true;
}
@ -231,8 +234,11 @@ export class NoteEntityService implements OnModuleInit {
}
@bindThis
private async populatePoll(note: MiNote, meId: MiUser['id'] | null) {
const poll = await this.pollsRepository.findOneByOrFail({ noteId: note.id });
private async populatePoll(note: MiNote, meId: MiUser['id'] | null, hint?: {
poll?: MiPoll,
myVotes?: MiPollVote[],
}) {
const poll = hint?.poll ?? await this.pollsRepository.findOneByOrFail({ noteId: note.id });
const choices = poll.choices.map(c => ({
text: c,
votes: poll.votes[poll.choices.indexOf(c)],
@ -241,7 +247,7 @@ export class NoteEntityService implements OnModuleInit {
if (meId) {
if (poll.multiple) {
const votes = await this.pollVotesRepository.findBy({
const votes = hint?.myVotes ?? await this.pollVotesRepository.findBy({
userId: meId,
noteId: note.id,
});
@ -251,7 +257,7 @@ export class NoteEntityService implements OnModuleInit {
choices[myChoice].isVoted = true;
}
} else {
const vote = await this.pollVotesRepository.findOneBy({
const vote = hint?.myVotes ? hint.myVotes[0] : await this.pollVotesRepository.findOneBy({
userId: meId,
noteId: note.id,
});
@ -313,7 +319,12 @@ export class NoteEntityService implements OnModuleInit {
}
@bindThis
public async isVisibleForMe(note: MiNote, meId: MiUser['id'] | null): Promise<boolean> {
public async isVisibleForMe(note: MiNote, meId: MiUser['id'] | null, hint?: {
myFollowing?: ReadonlySet<string>,
myBlocking?: ReadonlySet<string>,
myBlockers?: ReadonlySet<string>,
me?: Pick<MiUser, 'host'> | null,
}): Promise<boolean> {
// This code must always be synchronized with the checks in generateVisibilityQuery.
// visibility が specified かつ自分が指定されていなかったら非表示
if (note.visibility === 'specified') {
@ -341,16 +352,16 @@ export class NoteEntityService implements OnModuleInit {
return true;
} else {
// フォロワーかどうか
const [blocked, following, user] = await Promise.all([
this.cacheService.userBlockingCache.fetch(meId).then((ids) => ids.has(note.userId)),
this.followingsRepository.count({
where: {
followeeId: note.userId,
followerId: meId,
},
take: 1,
}),
this.usersRepository.findOneByOrFail({ id: meId }),
const [blocked, following, userHost] = await Promise.all([
hint?.myBlocking
? hint.myBlocking.has(note.userId)
: this.cacheService.userBlockingCache.fetch(meId).then((ids) => ids.has(note.userId)),
hint?.myFollowing
? hint.myFollowing.has(note.userId)
: this.cacheService.userFollowingsCache.fetch(meId).then(ids => ids.has(note.userId)),
hint?.me !== undefined
? (hint.me?.host ?? null)
: this.cacheService.findUserById(meId).then(me => me.host),
]);
if (blocked) return false;
@ -362,12 +373,13 @@ export class NoteEntityService implements OnModuleInit {
in which case we can never know the following. Instead we have
to assume that the users are following each other.
*/
return following > 0 || (note.userHost != null && user.host != null);
return following || (note.userHost != null && userHost != null);
}
}
if (meId != null) {
const isBlocked = (await this.cacheService.userBlockedCache.fetch(meId)).has(note.userId);
const blockers = hint?.myBlockers ?? await this.cacheService.userBlockedCache.fetch(meId);
const isBlocked = blockers.has(note.userId);
if (isBlocked) return false;
}
@ -404,6 +416,12 @@ export class NoteEntityService implements OnModuleInit {
packedFiles: Map<MiNote['fileIds'][number], Packed<'DriveFile'> | null>;
packedUsers: Map<MiUser['id'], Packed<'UserLite'>>;
mentionHandles: Record<string, string | undefined>;
userFollowings: Map<string, Map<string, Omit<MiFollowing, 'isFollowerHibernated'>>>;
userBlockers: Map<string, Set<string>>;
polls: Map<string, MiPoll>;
pollVotes: Map<string, Map<string, MiPollVote[]>>;
channels: Map<string, MiChannel>;
notes: Map<string, MiNote>;
};
},
): Promise<Packed<'Note'>> {
@ -433,9 +451,7 @@ export class NoteEntityService implements OnModuleInit {
}
const channel = note.channelId
? note.channel
? note.channel
: await this.channelsRepository.findOneBy({ id: note.channelId })
? (opts._hint_?.channels.get(note.channelId) ?? note.channel ?? await this.channelsRepository.findOneBy({ id: note.channelId }))
: null;
const reactionEmojiNames = Object.keys(reactions)
@ -481,7 +497,10 @@ export class NoteEntityService implements OnModuleInit {
mentionHandles: note.mentions.length > 0 ? this.getUserHandles(note.mentions, options?._hint_?.mentionHandles) : undefined,
uri: note.uri ?? undefined,
url: note.url ?? undefined,
poll: note.hasPoll ? this.populatePoll(note, meId) : undefined,
poll: note.hasPoll ? this.populatePoll(note, meId, {
poll: opts._hint_?.polls.get(note.id),
myVotes: opts._hint_?.pollVotes.get(note.id)?.get(note.userId),
}) : undefined,
...(meId && Object.keys(reactions).length > 0 ? {
myReaction: this.populateMyReaction({
@ -495,14 +514,14 @@ export class NoteEntityService implements OnModuleInit {
clippedCount: note.clippedCount,
processErrors: note.processErrors,
reply: note.replyId ? this.pack(note.reply ?? note.replyId, me, {
reply: note.replyId ? this.pack(note.reply ?? opts._hint_?.notes.get(note.replyId) ?? note.replyId, me, {
detail: false,
skipHide: opts.skipHide,
withReactionAndUserPairCache: opts.withReactionAndUserPairCache,
_hint_: options?._hint_,
}) : undefined,
renote: note.renoteId ? this.pack(note.renote ?? note.renoteId, me, {
renote: note.renoteId ? this.pack(note.renote ?? opts._hint_?.notes.get(note.renoteId) ?? note.renoteId, me, {
detail: true,
skipHide: opts.skipHide,
withReactionAndUserPairCache: opts.withReactionAndUserPairCache,
@ -514,7 +533,10 @@ export class NoteEntityService implements OnModuleInit {
this.treatVisibility(packed);
if (!opts.skipHide) {
await this.hideNote(packed, meId);
await this.hideNote(packed, meId, meId == null ? undefined : {
myFollowing: opts._hint_?.userFollowings.get(meId),
myBlockers: opts._hint_?.userBlockers.get(meId),
});
}
return packed;
@ -531,79 +553,139 @@ export class NoteEntityService implements OnModuleInit {
) {
if (notes.length === 0) return [];
const targetNotes: MiNote[] = [];
const targetNotesMap = new Map<string, MiNote>();
const targetNotesToFetch : string[] = [];
for (const note of notes) {
if (isPureRenote(note)) {
// we may need to fetch 'my reaction' for renote target.
targetNotes.push(note.renote);
if (note.renote.reply) {
// idem if the renote is also a reply.
targetNotes.push(note.renote.reply);
if (note.renote) {
targetNotesMap.set(note.renote.id, note.renote);
if (note.renote.reply) {
// idem if the renote is also a reply.
targetNotesMap.set(note.renote.reply.id, note.renote.reply);
}
} else if (options?.detail) {
targetNotesToFetch.push(note.renoteId);
}
} else {
if (note.reply) {
// idem for OP of a regular reply.
targetNotes.push(note.reply);
targetNotesMap.set(note.reply.id, note.reply);
} else if (note.replyId && options?.detail) {
targetNotesToFetch.push(note.replyId);
}
targetNotes.push(note);
targetNotesMap.set(note.id, note);
}
}
const bufferedReactions = this.meta.enableReactionsBuffering ? await this.reactionsBufferingService.getMany([...getAppearNoteIds(notes)]) : null;
// Don't fetch notes that were added by ID and then found inline in another note.
for (let i = targetNotesToFetch.length - 1; i >= 0; i--) {
if (targetNotesMap.has(targetNotesToFetch[i])) {
targetNotesToFetch.splice(i, 1);
}
}
const meId = me ? me.id : null;
const myReactionsMap = new Map<MiNote['id'], string | null>();
if (meId) {
const idsNeedFetchMyReaction = new Set<MiNote['id']>();
// Populate any relations that weren't included in the source
if (targetNotesToFetch.length > 0) {
const newNotes = await this.notesRepository.find({
where: {
id: In(targetNotesToFetch),
},
relations: {
user: {
userProfile: true,
},
reply: {
user: {
userProfile: true,
},
},
renote: {
user: {
userProfile: true,
},
reply: {
user: {
userProfile: true,
},
},
},
channel: true,
},
});
for (const note of targetNotes) {
const reactionsCount = Object.values(this.reactionsBufferingService.mergeReactions(note.reactions, bufferedReactions?.get(note.id)?.deltas ?? {})).reduce((a, b) => a + b, 0);
if (reactionsCount === 0) {
myReactionsMap.set(note.id, null);
} else if (reactionsCount <= note.reactionAndUserPairCache.length + (bufferedReactions?.get(note.id)?.pairs.length ?? 0)) {
const pairInBuffer = bufferedReactions?.get(note.id)?.pairs.find(p => p[0] === meId);
if (pairInBuffer) {
myReactionsMap.set(note.id, pairInBuffer[1]);
} else {
const pair = note.reactionAndUserPairCache.find(p => p.startsWith(meId));
myReactionsMap.set(note.id, pair ? pair.split('/')[1] : null);
for (const note of newNotes) {
targetNotesMap.set(note.id, note);
}
}
const targetNotes = Array.from(targetNotesMap.values());
const noteIds = Array.from(targetNotesMap.keys());
const usersMap = new Map<string, MiUser | string>();
const allUsers = notes.flatMap(note => [
note.user ?? note.userId,
note.reply?.user ?? note.replyUserId,
note.renote?.user ?? note.renoteUserId,
]);
for (const user of allUsers) {
if (!user) continue;
if (typeof(user) === 'object') {
// ID -> Entity
usersMap.set(user.id, user);
} else if (!usersMap.has(user)) {
// ID -> ID
usersMap.set(user, user);
}
}
const users = Array.from(usersMap.values());
const userIds = Array.from(usersMap.keys());
const fileIds = new Set(targetNotes.flatMap(n => n.fileIds));
const mentionedUsers = new Set(targetNotes.flatMap(note => note.mentions));
const [{ bufferedReactions, myReactionsMap }, packedFiles, packedUsers, mentionHandles, userFollowings, userBlockers, polls, pollVotes, channels] = await Promise.all([
// bufferedReactions & myReactionsMap
this.getReactions(targetNotes, me),
// packedFiles
this.driveFileEntityService.packManyByIdsMap(Array.from(fileIds)),
// packedUsers
this.userEntityService.packMany(users, me)
.then(users => new Map(users.map(u => [u.id, u]))),
// mentionHandles
this.getUserHandles(Array.from(mentionedUsers)),
// userFollowings
this.cacheService.userFollowingsCache.fetchMany(userIds).then(fs => new Map(fs)),
// userBlockers
this.cacheService.userBlockedCache.fetchMany(userIds).then(bs => new Map(bs)),
// polls
this.pollsRepository.findBy({ noteId: In(noteIds) })
.then(polls => new Map(polls.map(p => [p.noteId, p]))),
// pollVotes
this.pollVotesRepository.findBy({ noteId: In(noteIds), userId: In(userIds) })
.then(votes => votes.reduce((noteMap, vote) => {
let userMap = noteMap.get(vote.noteId);
if (!userMap) {
userMap = new Map<string, MiPollVote[]>();
noteMap.set(vote.noteId, userMap);
}
} else {
idsNeedFetchMyReaction.add(note.id);
}
}
const myReactions = idsNeedFetchMyReaction.size > 0 ? await this.noteReactionsRepository.findBy({
userId: meId,
noteId: In(Array.from(idsNeedFetchMyReaction)),
}) : [];
for (const id of idsNeedFetchMyReaction) {
myReactionsMap.set(id, myReactions.find(reaction => reaction.noteId === id)?.reaction ?? null);
}
}
await this.customEmojiService.prefetchEmojis(this.aggregateNoteEmojis(notes));
// TODO: 本当は renote とか reply がないのに renoteId とか replyId があったらここで解決しておく
const fileIds = notes.map(n => [n.fileIds, n.renote?.fileIds, n.reply?.fileIds]).flat(2).filter(x => x != null);
const packedFiles = fileIds.length > 0 ? await this.driveFileEntityService.packManyByIdsMap(fileIds) : new Map();
const users = [
...notes.map(({ user, userId }) => user ?? userId),
...notes.map(({ replyUserId }) => replyUserId).filter(x => x != null),
...notes.map(({ renoteUserId }) => renoteUserId).filter(x => x != null),
];
const packedUsers = await this.userEntityService.packMany(users, me)
.then(users => new Map(users.map(u => [u.id, u])));
// Recursively add all mentioned users from all notes + replies + renotes
const allMentionedUsers = targetNotes.reduce((users, note) => {
for (const user of note.mentions) {
users.add(user);
}
return users;
}, new Set<string>());
const mentionHandles = await this.getUserHandles(Array.from(allMentionedUsers));
let voteList = userMap.get(vote.userId);
if (!voteList) {
voteList = [];
userMap.set(vote.userId, voteList);
}
voteList.push(vote);
return noteMap;
}, new Map<string, Map<string, MiPollVote[]>>)),
// channels
this.getChannels(targetNotes),
// (not returned)
this.customEmojiService.prefetchEmojis(this.aggregateNoteEmojis(notes)),
]);
return await Promise.all(notes.map(n => this.pack(n, me, {
...options,
@ -613,6 +695,12 @@ export class NoteEntityService implements OnModuleInit {
packedFiles,
packedUsers,
mentionHandles,
userFollowings,
userBlockers,
polls,
pollVotes,
channels,
notes: new Map(targetNotes.map(n => [n.id, n])),
},
})));
}
@ -680,4 +768,71 @@ export class NoteEntityService implements OnModuleInit {
return map;
}, {} as Record<string, string | undefined>);
}
private async getChannels(notes: MiNote[]): Promise<Map<string, MiChannel>> {
const channels = new Map<string, MiChannel>();
const channelsToFetch = new Set<string>();
for (const note of notes) {
if (note.channel) {
channels.set(note.channel.id, note.channel);
} else if (note.channelId) {
channelsToFetch.add(note.channelId);
}
}
if (channelsToFetch.size > 0) {
const newChannels = await this.channelsRepository.findBy({
id: In(Array.from(channelsToFetch)),
});
for (const channel of newChannels) {
channels.set(channel.id, channel);
}
}
return channels;
}
private async getReactions(notes: MiNote[], me: { id: string } | null | undefined) {
const bufferedReactions = this.meta.enableReactionsBuffering ? await this.reactionsBufferingService.getMany([...getAppearNoteIds(notes)]) : null;
const meId = me ? me.id : null;
const myReactionsMap = new Map<MiNote['id'], string | null>();
if (meId) {
const idsNeedFetchMyReaction = new Set<MiNote['id']>();
for (const note of notes) {
const reactionsCount = Object.values(this.reactionsBufferingService.mergeReactions(note.reactions, bufferedReactions?.get(note.id)?.deltas ?? {})).reduce((a, b) => a + b, 0);
if (reactionsCount === 0) {
myReactionsMap.set(note.id, null);
} else if (reactionsCount <= note.reactionAndUserPairCache.length + (bufferedReactions?.get(note.id)?.pairs.length ?? 0)) {
const pairInBuffer = bufferedReactions?.get(note.id)?.pairs.find(p => p[0] === meId);
if (pairInBuffer) {
myReactionsMap.set(note.id, pairInBuffer[1]);
} else {
const pair = note.reactionAndUserPairCache.find(p => p.startsWith(meId));
myReactionsMap.set(note.id, pair ? pair.split('/')[1] : null);
}
} else {
idsNeedFetchMyReaction.add(note.id);
}
}
const myReactions = idsNeedFetchMyReaction.size > 0 ? await this.noteReactionsRepository.findBy({
userId: meId,
noteId: In(Array.from(idsNeedFetchMyReaction)),
}) : [];
for (const id of idsNeedFetchMyReaction) {
myReactionsMap.set(id, myReactions.find(reaction => reaction.noteId === id)?.reaction ?? null);
}
}
return { bufferedReactions, myReactionsMap };
}
@bindThis
public genLocalNoteUri(noteId: string): string {
return `${this.config.url}/notes/${noteId}`;
}
}

304
packages/backend/src/core/entities/UserEntityService.ts
View file

@ -30,6 +30,7 @@ import type {
FollowingsRepository,
FollowRequestsRepository,
MiFollowing,
MiInstance,
MiMeta,
MiUserNotePining,
MiUserProfile,
@ -42,7 +43,7 @@ import type {
UsersRepository,
} from '@/models/_.js';
import { bindThis } from '@/decorators.js';
import { RoleService } from '@/core/RoleService.js';
import { RolePolicies, RoleService } from '@/core/RoleService.js';
import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import { IdService } from '@/core/IdService.js';
@ -52,6 +53,7 @@ import { AvatarDecorationService } from '@/core/AvatarDecorationService.js';
import { ChatService } from '@/core/ChatService.js';
import { isSystemAccount } from '@/misc/is-system-account.js';
import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
import type { CacheService } from '@/core/CacheService.js';
import type { OnModuleInit } from '@nestjs/common';
import type { NoteEntityService } from './NoteEntityService.js';
import type { PageEntityService } from './PageEntityService.js';
@ -77,7 +79,7 @@ function isRemoteUser(user: MiUser | { host: MiUser['host'] }): boolean {
export type UserRelation = {
id: MiUser['id']
following: MiFollowing | null,
following: Omit<MiFollowing, 'isFollowerHibernated'> | null,
isFollowing: boolean
isFollowed: boolean
hasPendingFollowRequestFromYou: boolean
@ -103,6 +105,7 @@ export class UserEntityService implements OnModuleInit {
private idService: IdService;
private avatarDecorationService: AvatarDecorationService;
private chatService: ChatService;
private cacheService: CacheService;
constructor(
private moduleRef: ModuleRef,
@ -163,6 +166,7 @@ export class UserEntityService implements OnModuleInit {
this.idService = this.moduleRef.get('IdService');
this.avatarDecorationService = this.moduleRef.get('AvatarDecorationService');
this.chatService = this.moduleRef.get('ChatService');
this.cacheService = this.moduleRef.get('CacheService');
}
//#region Validators
@ -193,16 +197,8 @@ export class UserEntityService implements OnModuleInit {
memo,
mutedInstances,
] = await Promise.all([
this.followingsRepository.findOneBy({
followerId: me,
followeeId: target,
}),
this.followingsRepository.exists({
where: {
followerId: target,
followeeId: me,
},
}),
this.cacheService.userFollowingsCache.fetch(me).then(f => f.get(target) ?? null),
this.cacheService.userFollowingsCache.fetch(target).then(f => f.has(me)),
this.followRequestsRepository.exists({
where: {
followerId: me,
@ -215,45 +211,22 @@ export class UserEntityService implements OnModuleInit {
followeeId: me,
},
}),
this.blockingsRepository.exists({
where: {
blockerId: me,
blockeeId: target,
},
}),
this.blockingsRepository.exists({
where: {
blockerId: target,
blockeeId: me,
},
}),
this.mutingsRepository.exists({
where: {
muterId: me,
muteeId: target,
},
}),
this.renoteMutingsRepository.exists({
where: {
muterId: me,
muteeId: target,
},
}),
this.usersRepository.createQueryBuilder('u')
.select('u.host')
.where({ id: target })
.getRawOne<{ u_host: string }>()
.then(it => it?.u_host ?? null),
this.cacheService.userBlockingCache.fetch(me)
.then(blockees => blockees.has(target)),
this.cacheService.userBlockedCache.fetch(me)
.then(blockers => blockers.has(target)),
this.cacheService.userMutingsCache.fetch(me)
.then(mutings => mutings.has(target)),
this.cacheService.renoteMutingsCache.fetch(me)
.then(mutings => mutings.has(target)),
this.cacheService.findUserById(target).then(u => u.host),
this.userMemosRepository.createQueryBuilder('m')
.select('m.memo')
.where({ userId: me, targetUserId: target })
.getRawOne<{ m_memo: string | null }>()
.then(it => it?.m_memo ?? null),
this.userProfilesRepository.createQueryBuilder('p')
.select('p.mutedInstances')
.where({ userId: me })
.getRawOne<{ p_mutedInstances: string[] }>()
.then(it => it?.p_mutedInstances ?? []),
this.cacheService.userProfileCache.fetch(me)
.then(profile => profile.mutedInstances),
]);
const isInstanceMuted = !!host && mutedInstances.includes(host);
@ -277,8 +250,8 @@ export class UserEntityService implements OnModuleInit {
@bindThis
public async getRelations(me: MiUser['id'], targets: MiUser['id'][]): Promise<Map<MiUser['id'], UserRelation>> {
const [
followers,
followees,
myFollowing,
myFollowers,
followersRequests,
followeesRequests,
blockers,
@ -289,13 +262,8 @@ export class UserEntityService implements OnModuleInit {
memos,
mutedInstances,
] = await Promise.all([
this.followingsRepository.findBy({ followerId: me })
.then(f => new Map(f.map(it => [it.followeeId, it]))),
this.followingsRepository.createQueryBuilder('f')
.select('f.followerId')
.where('f.followeeId = :me', { me })
.getRawMany<{ f_followerId: string }>()
.then(it => it.map(it => it.f_followerId)),
this.cacheService.userFollowingsCache.fetch(me),
this.cacheService.userFollowersCache.fetch(me),
this.followRequestsRepository.createQueryBuilder('f')
.select('f.followeeId')
.where('f.followerId = :me', { me })
@ -306,34 +274,18 @@ export class UserEntityService implements OnModuleInit {
.where('f.followeeId = :me', { me })
.getRawMany<{ f_followerId: string }>()
.then(it => it.map(it => it.f_followerId)),
this.blockingsRepository.createQueryBuilder('b')
.select('b.blockeeId')
.where('b.blockerId = :me', { me })
.getRawMany<{ b_blockeeId: string }>()
.then(it => it.map(it => it.b_blockeeId)),
this.blockingsRepository.createQueryBuilder('b')
.select('b.blockerId')
.where('b.blockeeId = :me', { me })
.getRawMany<{ b_blockerId: string }>()
.then(it => it.map(it => it.b_blockerId)),
this.mutingsRepository.createQueryBuilder('m')
.select('m.muteeId')
.where('m.muterId = :me', { me })
.getRawMany<{ m_muteeId: string }>()
.then(it => it.map(it => it.m_muteeId)),
this.renoteMutingsRepository.createQueryBuilder('m')
.select('m.muteeId')
.where('m.muterId = :me', { me })
.getRawMany<{ m_muteeId: string }>()
.then(it => it.map(it => it.m_muteeId)),
this.usersRepository.createQueryBuilder('u')
.select(['u.id', 'u.host'])
.where({ id: In(targets) } )
.getRawMany<{ m_id: string, m_host: string }>()
.then(it => it.reduce((map, it) => {
map[it.m_id] = it.m_host;
return map;
}, {} as Record<string, string>)),
this.cacheService.userBlockedCache.fetch(me),
this.cacheService.userBlockingCache.fetch(me),
this.cacheService.userMutingsCache.fetch(me),
this.cacheService.renoteMutingsCache.fetch(me),
this.cacheService.getUsers(targets)
.then(users => {
const record: Record<string, string | null> = {};
for (const [id, user] of users) {
record[id] = user.host;
}
return record;
}),
this.userMemosRepository.createQueryBuilder('m')
.select(['m.targetUserId', 'm.memo'])
.where({ userId: me, targetUserId: In(targets) })
@ -342,16 +294,13 @@ export class UserEntityService implements OnModuleInit {
map[it.m_targetUserId] = it.m_memo;
return map;
}, {} as Record<string, string | null>)),
this.userProfilesRepository.createQueryBuilder('p')
.select('p.mutedInstances')
.where({ userId: me })
.getRawOne<{ p_mutedInstances: string[] }>()
.then(it => it?.p_mutedInstances ?? []),
this.cacheService.userProfileCache.fetch(me)
.then(p => p.mutedInstances),
]);
return new Map(
targets.map(target => {
const following = followers.get(target) ?? null;
const following = myFollowing.get(target) ?? null;
return [
target,
@ -359,14 +308,14 @@ export class UserEntityService implements OnModuleInit {
id: target,
following: following,
isFollowing: following != null,
isFollowed: followees.includes(target),
isFollowed: myFollowers.has(target),
hasPendingFollowRequestFromYou: followersRequests.includes(target),
hasPendingFollowRequestToYou: followeesRequests.includes(target),
isBlocking: blockers.includes(target),
isBlocked: blockees.includes(target),
isMuted: muters.includes(target),
isRenoteMuted: renoteMuters.includes(target),
isInstanceMuted: mutedInstances.includes(hosts[target]),
isBlocking: blockees.has(target),
isBlocked: blockers.has(target),
isMuted: muters.has(target),
isRenoteMuted: renoteMuters.has(target),
isInstanceMuted: hosts[target] != null && mutedInstances.includes(hosts[target]),
memo: memos[target] ?? null,
},
];
@ -391,6 +340,7 @@ export class UserEntityService implements OnModuleInit {
return false; // TODO
}
// TODO optimization: make redis calls in MULTI
@bindThis
public async getNotificationsInfo(userId: MiUser['id']): Promise<{
hasUnread: boolean;
@ -424,16 +374,14 @@ export class UserEntityService implements OnModuleInit {
@bindThis
public async getHasPendingReceivedFollowRequest(userId: MiUser['id']): Promise<boolean> {
const count = await this.followRequestsRepository.countBy({
return await this.followRequestsRepository.existsBy({
followeeId: userId,
});
return count > 0;
}
@bindThis
public async getHasPendingSentFollowRequest(userId: MiUser['id']): Promise<boolean> {
return this.followRequestsRepository.existsBy({
return await this.followRequestsRepository.existsBy({
followerId: userId,
});
}
@ -480,6 +428,10 @@ export class UserEntityService implements OnModuleInit {
userRelations?: Map<MiUser['id'], UserRelation>,
userMemos?: Map<MiUser['id'], string | null>,
pinNotes?: Map<MiUser['id'], MiUserNotePining[]>,
iAmModerator?: boolean,
userIdsByUri?: Map<string, string>,
instances?: Map<string, MiInstance | null>,
securityKeyCounts?: Map<string, number>,
},
): Promise<Packed<S>> {
const opts = Object.assign({
@ -487,7 +439,10 @@ export class UserEntityService implements OnModuleInit {
includeSecrets: false,
}, options);
const user = typeof src === 'object' ? src : await this.usersRepository.findOneByOrFail({ id: src });
const user = typeof src === 'object' ? src : await this.usersRepository.findOneOrFail({
where: { id: src },
relations: { userProfile: true },
});
// migration
if (user.avatarId != null && user.avatarUrl === null) {
@ -518,10 +473,10 @@ export class UserEntityService implements OnModuleInit {
const isDetailed = opts.schema !== 'UserLite';
const meId = me ? me.id : null;
const isMe = meId === user.id;
const iAmModerator = me ? await this.roleService.isModerator(me as MiUser) : false;
const iAmModerator = opts.iAmModerator ?? (me ? await this.roleService.isModerator(me as MiUser) : false);
const profile = isDetailed
? (opts.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id }))
? (opts.userProfile ?? user.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id }))
: null;
let relation: UserRelation | null = null;
@ -556,7 +511,7 @@ export class UserEntityService implements OnModuleInit {
}
}
const mastoapi = !isDetailed ? opts.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id }) : null;
const mastoapi = !isDetailed ? opts.userProfile ?? user.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id }) : null;
const followingCount = profile == null ? null :
(profile.followingVisibility === 'public') || isMe || iAmModerator ? user.followingCount :
@ -579,6 +534,9 @@ export class UserEntityService implements OnModuleInit {
const checkHost = user.host == null ? this.config.host : user.host;
const notificationsInfo = isMe && isDetailed ? await this.getNotificationsInfo(user.id) : null;
let fetchPoliciesPromise: Promise<RolePolicies> | null = null;
const fetchPolicies = () => fetchPoliciesPromise ??= this.roleService.getUserPolicies(user);
const packed = {
id: user.id,
name: user.name,
@ -603,19 +561,21 @@ export class UserEntityService implements OnModuleInit {
enableRss: user.enableRss,
mandatoryCW: user.mandatoryCW,
rejectQuotes: user.rejectQuotes,
isSilenced: user.isSilenced || this.roleService.getUserPolicies(user.id).then(r => !r.canPublicNote),
attributionDomains: user.attributionDomains,
isSilenced: user.isSilenced || fetchPolicies().then(r => !r.canPublicNote),
speakAsCat: user.speakAsCat ?? false,
approved: user.approved,
requireSigninToViewContents: user.requireSigninToViewContents === false ? undefined : true,
makeNotesFollowersOnlyBefore: user.makeNotesFollowersOnlyBefore ?? undefined,
makeNotesHiddenBefore: user.makeNotesHiddenBefore ?? undefined,
instance: user.host ? this.federatedInstanceService.federatedInstanceCache.fetch(user.host).then(instance => instance ? {
instance: user.host ? Promise.resolve(opts.instances?.has(user.host) ? opts.instances.get(user.host) : this.federatedInstanceService.fetch(user.host)).then(instance => instance ? {
name: instance.name,
softwareName: instance.softwareName,
softwareVersion: instance.softwareVersion,
iconUrl: instance.iconUrl,
faviconUrl: instance.faviconUrl,
themeColor: instance.themeColor,
isSilenced: instance.isSilenced,
} : undefined) : undefined,
followersCount: followersCount ?? 0,
followingCount: followingCount ?? 0,
@ -623,7 +583,7 @@ export class UserEntityService implements OnModuleInit {
emojis: this.customEmojiService.populateEmojis(user.emojis, checkHost),
onlineStatus: this.getOnlineStatus(user),
// パフォーマンス上の理由でローカルユーザーのみ
badgeRoles: user.host == null ? this.roleService.getUserBadgeRoles(user.id).then((rs) => rs
badgeRoles: user.host == null ? this.roleService.getUserBadgeRoles(user).then((rs) => rs
.filter((r) => r.isPublic || iAmModerator)
.sort((a, b) => b.displayOrder - a.displayOrder)
.map((r) => ({
@ -636,9 +596,9 @@ export class UserEntityService implements OnModuleInit {
...(isDetailed ? {
url: profile!.url,
uri: user.uri,
movedTo: user.movedToUri ? this.apPersonService.resolvePerson(user.movedToUri).then(user => user.id).catch(() => null) : null,
movedTo: user.movedToUri ? Promise.resolve(opts.userIdsByUri?.get(user.movedToUri) ?? this.apPersonService.resolvePerson(user.movedToUri).then(user => user.id).catch(() => null)) : null,
alsoKnownAs: user.alsoKnownAs
? Promise.all(user.alsoKnownAs.map(uri => this.apPersonService.fetchPerson(uri).then(user => user?.id).catch(() => null)))
? Promise.all(user.alsoKnownAs.map(uri => Promise.resolve(opts.userIdsByUri?.get(uri) ?? this.apPersonService.fetchPerson(uri).then(user => user?.id).catch(() => null))))
.then(xs => xs.length === 0 ? null : xs.filter(x => x != null))
: null,
updatedAt: user.updatedAt ? user.updatedAt.toISOString() : null,
@ -665,8 +625,8 @@ export class UserEntityService implements OnModuleInit {
followersVisibility: profile!.followersVisibility,
followingVisibility: profile!.followingVisibility,
chatScope: user.chatScope,
canChat: this.roleService.getUserPolicies(user.id).then(r => r.chatAvailability === 'available'),
roles: this.roleService.getUserRoles(user.id).then(roles => roles.filter(role => role.isPublic).sort((a, b) => b.displayOrder - a.displayOrder).map(role => ({
canChat: fetchPolicies().then(r => r.chatAvailability === 'available'),
roles: this.roleService.getUserRoles(user).then(roles => roles.filter(role => role.isPublic).sort((a, b) => b.displayOrder - a.displayOrder).map(role => ({
id: role.id,
name: role.name,
color: role.color,
@ -684,7 +644,7 @@ export class UserEntityService implements OnModuleInit {
twoFactorEnabled: profile!.twoFactorEnabled,
usePasswordLessLogin: profile!.usePasswordLessLogin,
securityKeys: profile!.twoFactorEnabled
? this.userSecurityKeysRepository.countBy({ userId: user.id }).then(result => result >= 1)
? Promise.resolve(opts.securityKeyCounts?.get(user.id) ?? this.userSecurityKeysRepository.countBy({ userId: user.id })).then(result => result >= 1)
: false,
} : {}),
@ -728,7 +688,7 @@ export class UserEntityService implements OnModuleInit {
emailNotificationTypes: profile!.emailNotificationTypes,
achievements: profile!.achievements,
loggedInDays: profile!.loggedInDates.length,
policies: this.roleService.getUserPolicies(user.id),
policies: fetchPolicies(),
defaultCW: profile!.defaultCW,
defaultCWPriority: profile!.defaultCWPriority,
allowUnsignedFetch: user.allowUnsignedFetch,
@ -778,57 +738,103 @@ export class UserEntityService implements OnModuleInit {
includeSecrets?: boolean,
},
): Promise<Packed<S>[]> {
if (users.length === 0) return [];
// -- IDのみの要素を補完して完全なエンティティ一覧を作る
const _users = users.filter((user): user is MiUser => typeof user !== 'string');
if (_users.length !== users.length) {
_users.push(
...await this.usersRepository.findBy({
id: In(users.filter((user): user is string => typeof user === 'string')),
...await this.usersRepository.find({
where: {
id: In(users.filter((user): user is string => typeof user === 'string')),
},
relations: {
userProfile: true,
},
}),
);
}
const _userIds = _users.map(u => u.id);
// -- 実行者の有無や指定スキーマの種別によって要否が異なる値群を取得
const iAmModerator = await this.roleService.isModerator(me as MiUser);
const meId = me ? me.id : null;
const isDetailed = options && options.schema !== 'UserLite';
const isDetailedAndMod = isDetailed && iAmModerator;
let profilesMap: Map<MiUser['id'], MiUserProfile> = new Map();
let userRelations: Map<MiUser['id'], UserRelation> = new Map();
let userMemos: Map<MiUser['id'], string | null> = new Map();
let pinNotes: Map<MiUser['id'], MiUserNotePining[]> = new Map();
const userUris = new Set(_users
.flatMap(user => [user.uri, user.movedToUri])
.filter((uri): uri is string => uri != null));
if (options?.schema !== 'UserLite') {
profilesMap = await this.userProfilesRepository.findBy({ userId: In(_userIds) })
.then(profiles => new Map(profiles.map(p => [p.userId, p])));
const userHosts = new Set(_users
.map(user => user.host)
.filter((host): host is string => host != null));
const meId = me ? me.id : null;
if (meId) {
userMemos = await this.userMemosRepository.findBy({ userId: meId })
.then(memos => new Map(memos.map(memo => [memo.targetUserId, memo.memo])));
if (_userIds.length > 0) {
userRelations = await this.getRelations(meId, _userIds);
pinNotes = await this.userNotePiningsRepository.createQueryBuilder('pin')
.where('pin.userId IN (:...userIds)', { userIds: _userIds })
.innerJoinAndSelect('pin.note', 'note')
.getMany()
.then(pinsNotes => {
const map = new Map<MiUser['id'], MiUserNotePining[]>();
for (const note of pinsNotes) {
const notes = map.get(note.userId) ?? [];
notes.push(note);
map.set(note.userId, notes);
}
for (const [, notes] of map.entries()) {
// pack側ではDESCで取得しているので、それに合わせて降順に並び替えておく
notes.sort((a, b) => b.id.localeCompare(a.id));
}
return map;
});
}
const _profilesFromUsers: [string, MiUserProfile][] = [];
const _profilesToFetch: string[] = [];
for (const user of _users) {
if (user.userProfile) {
_profilesFromUsers.push([user.id, user.userProfile]);
} else {
_profilesToFetch.push(user.id);
}
}
// -- 実行者の有無や指定スキーマの種別によって要否が異なる値群を取得
const [profilesMap, userMemos, userRelations, pinNotes, userIdsByUri, instances, securityKeyCounts] = await Promise.all([
// profilesMap
this.cacheService.userProfileCache.fetchMany(_profilesToFetch).then(profiles => new Map(profiles.concat(_profilesFromUsers))),
// userMemos
isDetailed && meId ? this.userMemosRepository.findBy({ userId: meId })
.then(memos => new Map(memos.map(memo => [memo.targetUserId, memo.memo]))) : new Map(),
// userRelations
isDetailed && meId ? this.getRelations(meId, _userIds) : new Map(),
// pinNotes
isDetailed ? this.userNotePiningsRepository.createQueryBuilder('pin')
.where('pin.userId IN (:...userIds)', { userIds: _userIds })
.innerJoinAndSelect('pin.note', 'note')
.getMany()
.then(pinsNotes => {
const map = new Map<MiUser['id'], MiUserNotePining[]>();
for (const note of pinsNotes) {
const notes = map.get(note.userId) ?? [];
notes.push(note);
map.set(note.userId, notes);
}
for (const [, notes] of map.entries()) {
// pack側ではDESCで取得しているので、それに合わせて降順に並び替えておく
notes.sort((a, b) => b.id.localeCompare(a.id));
}
return map;
}) : new Map(),
// userIdsByUrl
isDetailed ? this.usersRepository.createQueryBuilder('user')
.select([
'user.id',
'user.uri',
])
.where({
uri: In(Array.from(userUris)),
})
.getRawMany<{ user_uri: string, user_id: string }>()
.then(users => new Map(users.map(u => [u.user_uri, u.user_id]))) : new Map(),
// instances
Promise.all(Array.from(userHosts).map(async host => [host, await this.federatedInstanceService.fetch(host)] as const))
.then(hosts => new Map(hosts)),
// securityKeyCounts
isDetailedAndMod ? this.userSecurityKeysRepository.createQueryBuilder('key')
.select('key.userId', 'userId')
.addSelect('count(key.id)', 'userCount')
.where({
userId: In(_userIds),
})
.groupBy('key.userId')
.getRawMany<{ userId: string, userCount: number }>()
.then(counts => new Map(counts.map(c => [c.userId, c.userCount])))
: undefined, // .pack will fetch the keys for the requesting user if it's in the _userIds
]);
return Promise.all(
_users.map(u => this.pack(
u,
@ -839,6 +845,10 @@ export class UserEntityService implements OnModuleInit {
userRelations: userRelations,
userMemos: userMemos,
pinNotes: pinNotes,
iAmModerator,
userIdsByUri,
instances,
securityKeyCounts,
},
)),
);

1
packages/backend/src/env.ts
View file

@ -11,6 +11,7 @@ const envOption = {
verbose: false,
withLogTime: false,
quiet: false,
hideWorkerId: false,
};
for (const key of Object.keys(envOption) as (keyof typeof envOption)[]) {

14
packages/backend/src/logger.ts
View file

@ -23,6 +23,14 @@ export type DataElement = DataObject | Error | string | null;
// https://stackoverflow.com/questions/61148466/typescript-type-that-matches-any-object-but-not-arrays
export type DataObject = Record<string, unknown> | (object & { length?: never; });
const levelFuncs = {
error: 'error',
warning: 'warn',
success: 'info',
info: 'log',
debug: 'debug',
} as const satisfies Record<Level, keyof typeof console>;
// eslint-disable-next-line import/no-default-export
export default class Logger {
private context: Context;
@ -71,7 +79,9 @@ export default class Logger {
level === 'info' ? message :
null;
let log = `${l} ${worker}\t[${contexts.join(' ')}]\t${m}`;
let log = envOption.hideWorkerId
? `${l}\t[${contexts.join(' ')}]\t\t${m}`
: `${l} ${worker}\t[${contexts.join(' ')}]\t\t${m}`;
if (envOption.withLogTime) log = chalk.gray(time) + ' ' + log;
const args: unknown[] = [important ? chalk.bold(log) : log];
@ -84,7 +94,7 @@ export default class Logger {
} else if (data != null) {
args.push(data);
}
console.log(...args);
console[levelFuncs[level]](...args);
}
@bindThis

2
packages/backend/src/misc/FileWriterStream.ts
View file

@ -21,7 +21,7 @@ export class FileWriterStream extends WritableStream<Uint8Array> {
write: async (chunk, controller) => {
if (file === null) {
controller.error();
throw new Error();
throw new Error('file is null');
}
await file.write(chunk);

385
packages/backend/src/misc/QuantumKVCache.ts Normal file
View file

@ -0,0 +1,385 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { InternalEventService } from '@/core/InternalEventService.js';
import { bindThis } from '@/decorators.js';
import { InternalEventTypes } from '@/core/GlobalEventService.js';
import { MemoryKVCache } from '@/misc/cache.js';
export interface QuantumKVOpts<T> {
/**
* Memory cache lifetime in milliseconds.
*/
lifetime: number;
/**
* Callback to fetch the value for a key that wasn't found in the cache.
* May be synchronous or async.
*/
fetcher: (key: string, cache: QuantumKVCache<T>) => T | Promise<T>;
/**
* Optional callback to fetch the value for multiple keys that weren't found in the cache.
* May be synchronous or async.
* If not provided, then the implementation will fall back on repeated calls to fetcher().
*/
bulkFetcher?: (keys: string[], cache: QuantumKVCache<T>) => Iterable<[key: string, value: T]> | Promise<Iterable<[key: string, value: T]>>;
/**
* Optional callback when one or more values are changed (created, updated, or deleted) in the cache, either locally or elsewhere in the cluster.
* This is called *after* the cache state is updated.
* Implementations may be synchronous or async.
*/
onChanged?: (keys: string[], cache: QuantumKVCache<T>) => void | Promise<void>;
}
/**
* QuantumKVCache is a lifetime-bounded memory cache (like MemoryKVCache) with automatic cross-cluster synchronization via Redis.
* All nodes in the cluster are guaranteed to have a *subset* view of the current accurate state, though individual processes may have different items in their local cache.
* This ensures that a call to get() will never return stale data.
*/
export class QuantumKVCache<T> implements Iterable<[key: string, value: T]> {
private readonly memoryCache: MemoryKVCache<T>;
public readonly fetcher: QuantumKVOpts<T>['fetcher'];
public readonly bulkFetcher: QuantumKVOpts<T>['bulkFetcher'];
public readonly onChanged: QuantumKVOpts<T>['onChanged'];
/**
* @param internalEventService Service bus to synchronize events.
* @param name Unique name of the cache - must be the same in all processes.
* @param opts Cache options
*/
constructor(
private readonly internalEventService: InternalEventService,
private readonly name: string,
opts: QuantumKVOpts<T>,
) {
this.memoryCache = new MemoryKVCache(opts.lifetime);
this.fetcher = opts.fetcher;
this.bulkFetcher = opts.bulkFetcher;
this.onChanged = opts.onChanged;
this.internalEventService.on('quantumCacheUpdated', this.onQuantumCacheUpdated, {
// Ignore our own events, otherwise we'll immediately erase any set value.
ignoreLocal: true,
});
}
/**
* The number of items currently in memory.
* This applies to the local subset view, not the cross-cluster cache state.
*/
public get size() {
return this.memoryCache.size;
}
/**
* Iterates all [key, value] pairs in memory.
* This applies to the local subset view, not the cross-cluster cache state.
*/
@bindThis
public *entries(): Generator<[key: string, value: T]> {
for (const entry of this.memoryCache.entries) {
yield [entry[0], entry[1].value];
}
}
/**
* Iterates all keys in memory.
* This applies to the local subset view, not the cross-cluster cache state.
*/
@bindThis
public *keys() {
for (const entry of this.memoryCache.entries) {
yield entry[0];
}
}
/**
* Iterates all values pairs in memory.
* This applies to the local subset view, not the cross-cluster cache state.
*/
@bindThis
public *values() {
for (const entry of this.memoryCache.entries) {
yield entry[1].value;
}
}
/**
* Creates or updates a value in the cache, and erases any stale caches across the cluster.
* Fires an onSet event after the cache has been updated in all processes.
* Skips if the value is unchanged.
*/
@bindThis
public async set(key: string, value: T): Promise<void> {
if (this.memoryCache.get(key) === value) {
return;
}
this.memoryCache.set(key, value);
await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: [key] });
if (this.onChanged) {
await this.onChanged([key], this);
}
}
/**
* Creates or updates multiple value in the cache, and erases any stale caches across the cluster.
* Fires an onSet for each changed item event after the cache has been updated in all processes.
* Skips if all values are unchanged.
*/
@bindThis
public async setMany(items: Iterable<[key: string, value: T]>): Promise<void> {
const changedKeys: string[] = [];
for (const item of items) {
if (this.memoryCache.get(item[0]) !== item[1]) {
changedKeys.push(item[0]);
this.memoryCache.set(item[0], item[1]);
}
}
if (changedKeys.length > 0) {
await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: changedKeys });
if (this.onChanged) {
await this.onChanged(changedKeys, this);
}
}
}
/**
* Adds a value to the local memory cache without notifying other process.
* Neither a Redis event nor onSet callback will be fired, as the value has not actually changed.
* This should only be used when the value is known to be current, like after fetching from the database.
*/
@bindThis
public add(key: string, value: T): void {
this.memoryCache.set(key, value);
}
/**
* Adds multiple values to the local memory cache without notifying other process.
* Neither a Redis event nor onSet callback will be fired, as the value has not actually changed.
* This should only be used when the value is known to be current, like after fetching from the database.
*/
@bindThis
public addMany(items: Iterable<[key: string, value: T]>): void {
for (const [key, value] of items) {
this.memoryCache.set(key, value);
}
}
/**
* Gets a value from the local memory cache, or returns undefined if not found.
* Returns cached data only - does not make any fetches.
*/
@bindThis
public get(key: string): T | undefined {
return this.memoryCache.get(key);
}
/**
* Gets multiple values from the local memory cache; returning undefined for any missing keys.
* Returns cached data only - does not make any fetches.
*/
@bindThis
public getMany(keys: Iterable<string>): [key: string, value: T | undefined][] {
const results: [key: string, value: T | undefined][] = [];
for (const key of keys) {
results.push([key, this.get(key)]);
}
return results;
}
/**
* Gets or fetches a value from the cache.
* Fires an onSet event, but does not emit an update event to other processes.
*/
@bindThis
public async fetch(key: string): Promise<T> {
let value = this.memoryCache.get(key);
if (value === undefined) {
value = await this.fetcher(key, this);
this.memoryCache.set(key, value);
if (this.onChanged) {
await this.onChanged([key], this);
}
}
return value;
}
/**
* Gets or fetches multiple values from the cache.
* Fires onSet events, but does not emit any update events to other processes.
*/
@bindThis
public async fetchMany(keys: Iterable<string>): Promise<[key: string, value: T][]> {
const results: [key: string, value: T][] = [];
const toFetch: string[] = [];
// Spliterate into cached results / uncached keys.
for (const key of keys) {
const fromCache = this.get(key);
if (fromCache) {
results.push([key, fromCache]);
} else {
toFetch.push(key);
}
}
// Fetch any uncached keys
if (toFetch.length > 0) {
const fetched = await this.bulkFetch(toFetch);
// Add to cache and return set
this.addMany(fetched);
results.push(...fetched);
// Emit event
if (this.onChanged) {
await this.onChanged(toFetch, this);
}
}
return results;
}
/**
* Returns true is a key exists in memory.
* This applies to the local subset view, not the cross-cluster cache state.
*/
@bindThis
public has(key: string): boolean {
return this.memoryCache.get(key) !== undefined;
}
/**
* Deletes a value from the cache, and erases any stale caches across the cluster.
* Fires an onDelete event after the cache has been updated in all processes.
*/
@bindThis
public async delete(key: string): Promise<void> {
this.memoryCache.delete(key);
await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: [key] });
if (this.onChanged) {
await this.onChanged([key], this);
}
}
/**
* Deletes multiple values from the cache, and erases any stale caches across the cluster.
* Fires an onDelete event for each key after the cache has been updated in all processes.
* Skips if the input is empty.
*/
@bindThis
public async deleteMany(keys: Iterable<string>): Promise<void> {
const deleted: string[] = [];
for (const key of keys) {
this.memoryCache.delete(key);
deleted.push(key);
}
if (deleted.length === 0) {
return;
}
await this.internalEventService.emit('quantumCacheUpdated', { name: this.name, keys: deleted });
if (this.onChanged) {
await this.onChanged(deleted, this);
}
}
/**
* Refreshes the value of a key from the fetcher, and erases any stale caches across the cluster.
* Fires an onSet event after the cache has been updated in all processes.
*/
@bindThis
public async refresh(key: string): Promise<T> {
const value = await this.fetcher(key, this);
await this.set(key, value);
return value;
}
@bindThis
public async refreshMany(keys: Iterable<string>): Promise<[key: string, value: T][]> {
const values = await this.bulkFetch(keys);
await this.setMany(values);
return values;
}
/**
* Erases all entries from the local memory cache.
* Does not send any events or update other processes.
*/
@bindThis
public clear() {
this.memoryCache.clear();
}
/**
* Removes expired cache entries from the local view.
* Does not send any events or update other processes.
*/
@bindThis
public gc() {
this.memoryCache.gc();
}
/**
* Erases all data and disconnects from the cluster.
* This *must* be called when shutting down to prevent memory leaks!
*/
@bindThis
public dispose() {
this.internalEventService.off('quantumCacheUpdated', this.onQuantumCacheUpdated);
this.memoryCache.dispose();
}
@bindThis
private async bulkFetch(keys: Iterable<string>): Promise<[key: string, value: T][]> {
if (this.bulkFetcher) {
const results = await this.bulkFetcher(Array.from(keys), this);
return Array.from(results);
}
const results: [key: string, value: T][] = [];
for (const key of keys) {
const value = await this.fetcher(key, this);
results.push([key, value]);
}
return results;
}
@bindThis
private async onQuantumCacheUpdated(data: InternalEventTypes['quantumCacheUpdated']): Promise<void> {
if (data.name === this.name) {
for (const key of data.keys) {
this.memoryCache.delete(key);
}
if (this.onChanged) {
await this.onChanged(data.keys, this);
}
}
}
/**
* Iterates all [key, value] pairs in memory.
* This applies to the local subset view, not the cross-cluster cache state.
*/
[Symbol.iterator](): Iterator<[key: string, value: T]> {
return this.entries();
}
}

53
packages/backend/src/misc/cache.ts
View file

@ -9,9 +9,9 @@ import { bindThis } from '@/decorators.js';
export class RedisKVCache<T> {
private readonly lifetime: number;
private readonly memoryCache: MemoryKVCache<T>;
private readonly fetcher: (key: string) => Promise<T>;
private readonly toRedisConverter: (value: T) => string;
private readonly fromRedisConverter: (value: string) => T | undefined;
public readonly fetcher: (key: string) => Promise<T>;
public readonly toRedisConverter: (value: T) => string;
public readonly fromRedisConverter: (value: string) => T | undefined;
constructor(
private redisClient: Redis.Redis,
@ -99,6 +99,11 @@ export class RedisKVCache<T> {
// TODO: イベント発行して他プロセスのメモリキャッシュも更新できるようにする
}
@bindThis
public clear() {
this.memoryCache.clear();
}
@bindThis
public gc() {
this.memoryCache.gc();
@ -113,9 +118,9 @@ export class RedisKVCache<T> {
export class RedisSingleCache<T> {
private readonly lifetime: number;
private readonly memoryCache: MemorySingleCache<T>;
private readonly fetcher: () => Promise<T>;
private readonly toRedisConverter: (value: T) => string;
private readonly fromRedisConverter: (value: string) => T | undefined;
public readonly fetcher: () => Promise<T>;
public readonly toRedisConverter: (value: T) => string;
public readonly fromRedisConverter: (value: string) => T | undefined;
constructor(
private redisClient: Redis.Redis,
@ -123,16 +128,17 @@ export class RedisSingleCache<T> {
opts: {
lifetime: number;
memoryCacheLifetime: number;
fetcher: RedisSingleCache<T>['fetcher'];
toRedisConverter: RedisSingleCache<T>['toRedisConverter'];
fromRedisConverter: RedisSingleCache<T>['fromRedisConverter'];
fetcher?: RedisSingleCache<T>['fetcher'];
toRedisConverter?: RedisSingleCache<T>['toRedisConverter'];
fromRedisConverter?: RedisSingleCache<T>['fromRedisConverter'];
},
) {
this.lifetime = opts.lifetime;
this.memoryCache = new MemorySingleCache(opts.memoryCacheLifetime);
this.fetcher = opts.fetcher;
this.toRedisConverter = opts.toRedisConverter;
this.fromRedisConverter = opts.fromRedisConverter;
this.fetcher = opts.fetcher ?? (() => { throw new Error('fetch not supported - use get/set directly'); });
this.toRedisConverter = opts.toRedisConverter ?? ((value) => JSON.stringify(value));
this.fromRedisConverter = opts.fromRedisConverter ?? ((value) => JSON.parse(value));
}
@bindThis
@ -237,6 +243,16 @@ export class MemoryKVCache<T> {
return cached.value;
}
public has(key: string): boolean {
const cached = this.cache.get(key);
if (cached == null) return false;
if ((Date.now() - cached.date) > this.lifetime) {
this.cache.delete(key);
return false;
}
return true;
}
@bindThis
public delete(key: string): void {
this.cache.delete(key);
@ -308,11 +324,24 @@ export class MemoryKVCache<T> {
}
}
/**
* Removes all entries from the cache, but does not dispose it.
*/
@bindThis
public clear(): void {
this.cache.clear();
}
@bindThis
public dispose(): void {
this.clear();
clearInterval(this.gcIntervalHandle);
}
public get size() {
return this.cache.size;
}
public get entries() {
return this.cache.entries();
}

Some files were not shown because too many files have changed in this diff Show more