mirrors/barkey
1
0
Fork 0
mirror of https://codeberg.org/yeentown/barkey.git synced 2025-07-07 04:26:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

merge develop and fix conflicts.

Browse source
This commit is contained in:
piuvas 2025-06-03 10:56:10 -03:00
commit 1120ad19ae
No known key found for this signature in database
166 changed files with 2933 additions and 1079 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
.config/ci.yml
View file

@ -115,8 +115,14 @@ db:
user: postgres
pass: ci
# Whether disable Caching queries
#disableCache: true
## Log a warning to the server console if any query takes longer than this to complete.
## Measured in milliseconds; set to 0 to disable. (default: 300)
#slowQueryThreshold: 300
# If false, then query results will be cached in redis.
# If true (default), then queries will not be cached.
# This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior.
#disableCache: false
# Extra Connection options
#extra:
@ -297,6 +303,10 @@ proxyBypassHosts:
#proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4
#proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5
# Path to the directory that uploaded media will be saved to
# Defaults to a folder called "files" in the Sharkey directory
#mediaDirectory: /var/lib/sharkey
# Media Proxy
#mediaProxy: https://example.com/proxy

14
.config/cypress-devcontainer.yml
View file

@ -57,8 +57,14 @@ db:
user: postgres
pass: postgres
# Whether disable Caching queries
#disableCache: true
## Log a warning to the server console if any query takes longer than this to complete.
## Measured in milliseconds; set to 0 to disable. (default: 300)
#slowQueryThreshold: 300
# If false, then query results will be cached in redis.
# If true (default), then queries will not be cached.
# This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior.
#disableCache: false
# Extra Connection options
#extra:
@ -260,6 +266,10 @@ proxyBypassHosts:
#proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4
#proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5
# Path to the directory that uploaded media will be saved to
# Defaults to a folder called "files" in the Sharkey directory
#mediaDirectory: /var/lib/sharkey
# Media Proxy
#mediaProxy: https://example.com/proxy

14
.config/docker_example.yml
View file

@ -118,8 +118,14 @@ db:
user: example-misskey-user
pass: example-misskey-pass
# Whether disable Caching queries
#disableCache: true
## Log a warning to the server console if any query takes longer than this to complete.
## Measured in milliseconds; set to 0 to disable. (default: 300)
#slowQueryThreshold: 300
# If false, then query results will be cached in redis.
# If true (default), then queries will not be cached.
# This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior.
#disableCache: false
# Extra Connection options
#extra:
@ -351,6 +357,10 @@ proxyBypassHosts:
#proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4
#proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5
# Path to the directory that uploaded media will be saved to
# Defaults to a folder called "files" in the Sharkey directory
#mediaDirectory: /var/lib/sharkey
# Media Proxy
# Reference Implementation: https://github.com/misskey-dev/media-proxy
# * Deliver a common cache between instances

14
.config/example.yml
View file

@ -121,8 +121,14 @@ db:
user: sharkey
pass: example-misskey-pass
# Whether disable Caching queries
#disableCache: true
## Log a warning to the server console if any query takes longer than this to complete.
## Measured in milliseconds; set to 0 to disable. (default: 300)
#slowQueryThreshold: 300
# If false, then query results will be cached in redis.
# If true (default), then queries will not be cached.
# This will reduce database load at the cost of increased Redis traffic and risk of bugs and unpredictable behavior.
#disableCache: false
# Extra Connection options
#extra:
@ -354,6 +360,10 @@ proxyBypassHosts:
#proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4
#proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5
# Path to the directory that uploaded media will be saved to
# Defaults to a folder called "files" in the Sharkey directory
#mediaDirectory: /var/lib/sharkey
# Media Proxy
# Reference Implementation: https://github.com/misskey-dev/media-proxy
# * Deliver a common cache between instances

4
.devcontainer/devcontainer.yml
View file

@ -199,6 +199,10 @@ proxyBypassHosts:
#proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4
#proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5
# Path to the directory that uploaded media will be saved to
# Defaults to a folder called "files" in the Sharkey directory
#mediaDirectory: /var/lib/sharkey
# Media Proxy
#mediaProxy: https://example.com/proxy

4
.gitignore vendored
View file

@ -87,3 +87,7 @@ vite.config.local-dev.ts.timestamp-*
# Sharkey
/packages/megalodon/lib
# TypeScript
.tsbuildinfo
*.tsbuildinfo

2
CONTRIBUTING.md
View file

@ -690,7 +690,7 @@ seems to do a decent job)
* re-generate locales (`pnpm run build-assets`) and commit
* build the frontend: `rm -rf built/; NODE_ENV=development pnpm --filter=frontend --filter=frontend-embed --filter=frontend-shared build` (the `development` tells it to keep some of the original filenames in the built files)
* make sure there aren't any new `ti-*` classes (Tabler Icons), and replace them with appropriate `ph-*` ones (Phosphor Icons) in [`vite.replaceicons.ts`](packages/frontend/vite.replaceIcons.ts).
* This command should show you want to change: `grep -ohrP '(?<=["'\'']ti )(ti-(?!fw)[\w\-]+)' --exclude \*.map -- built/ | sort -u`.
* This command should show you want to change: `grep -ohrP '(?<=["'\''](ti )?)(ti-(?!fw)[\w\-]+)' --exclude \*.map -- built/ | sort -u`.
* NOTE: `ti-fw` is a special class that's defined by Misskey, leave it alone.
* After every change, re-build the frontend and check again, until there are no more `ti-*` classes in the built files.
* Commit!

3
cypress/tsconfig.json
View file

@ -2,7 +2,8 @@
"compilerOptions": {
"lib": ["dom", "es5"],
"target": "es5",
"types": ["cypress", "node"]
"types": ["cypress", "node"],
"incremental": true
},
"include": ["./**/*.ts"]
}

76
locales/index.d.ts vendored
View file

@ -12492,6 +12492,14 @@ export interface Locale extends ILocale {
* Displays content centered.
*/
"centerDescription": string;
/**
* Unix Time
*/
"unixtime": string;
/**
* Displays a timestamp in the viewer's current timezone.
*/
"unixtimeDescription": string;
/**
* Code (Inline)
*/
@ -13069,6 +13077,26 @@ export interface Locale extends ILocale {
* Users popular on {name}
*/
"popularUsersLocal": ParameterizedString<"name">;
/**
* Polls trending on {name}
*/
"pollsOnLocal": ParameterizedString<"name">;
/**
* Polls trending on the global network
*/
"pollsOnRemote": string;
/**
* Polls that have ended recently
*/
"pollsExpired": string;
/**
* Trending polls are disabled on this instance.
*/
"trendingPollsDisabled": string;
/**
* Please log in to view trending polls.
*/
"trendingPollsDisabledLogIn": string;
/**
* Silenced
*/
@ -13141,6 +13169,54 @@ export interface Locale extends ILocale {
* Written by {user}
*/
"writtenBy": ParameterizedString<"user">;
/**
* Following (Pub)
*/
"followingPub": string;
/**
* Followers (Sub)
*/
"followersSub": string;
/**
* Well-known resources
*/
"wellKnownResources": string;
/**
* Last posted: {at}
*/
"lastPosted": ParameterizedString<"at">;
/**
* NSFW
*/
"nsfw": string;
/**
* Raw
*/
"raw": string;
/**
* CW
*/
"cw": string;
/**
* Media Silenced
*/
"mediaSilenced": string;
/**
* Bubble
*/
"bubble": string;
/**
* Verified
*/
"verified": string;
/**
* Not Verified
*/
"notVerified": string;
/**
* Hibernated
*/
"hibernated": string;
}
declare const locales: {
[lang: string]: Locale;

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "sharkey",
"version": "2025.4.2-rc",
"version": "2025.5.0-dev",
"codename": "shonk",
"repository": {
"type": "git",

42
packages/backend/migration/1747938628395-add-missing-indexes.js
View file

@ -4,19 +4,35 @@
*/
export class AddMissingIndexes1747938628395 {
name = 'AddMissingIndexes1747938628395'
name = 'AddMissingIndexes1747938628395'
async up(queryRunner) {
await queryRunner.query(`CREATE INDEX "IDX_58699f75b9cf904f5f007909cb" ON "user_profile" ("birthday") `);
await queryRunner.query(`CREATE INDEX "IDX_021015e6683570ae9f6b0c62be" ON "user_list_membership" ("userId") `);
await queryRunner.query(`CREATE INDEX "IDX_cddcaf418dc4d392ecfcca842a" ON "user_list_membership" ("userListId") `);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_e4f3094c43f2d665e6030b0337" ON "user_list_membership" ("userId", "userListId") `);
}
async up(queryRunner) {
// Some instances have duplicate list entries
await queryRunner.query(`
DELETE FROM "user_list_membership"
WHERE "id" NOT IN (
SELECT MIN("id")
FROM "user_list_membership"
GROUP BY "userId", "userListId"
)`);
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_e4f3094c43f2d665e6030b0337"`);
await queryRunner.query(`DROP INDEX "public"."IDX_cddcaf418dc4d392ecfcca842a"`);
await queryRunner.query(`DROP INDEX "public"."IDX_021015e6683570ae9f6b0c62be"`);
await queryRunner.query(`DROP INDEX "public"."IDX_58699f75b9cf904f5f007909cb"`);
}
// Some instances already have these indexes, for an unknown reason
await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_e4f3094c43f2d665e6030b0337"`);
await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_cddcaf418dc4d392ecfcca842a"`);
await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_021015e6683570ae9f6b0c62be"`);
await queryRunner.query(`DROP INDEX IF EXISTS "public"."IDX_58699f75b9cf904f5f007909cb"`);
// Now the actual migration
await queryRunner.query(`CREATE INDEX "IDX_58699f75b9cf904f5f007909cb" ON "user_profile" ("birthday") `);
await queryRunner.query(`CREATE INDEX "IDX_021015e6683570ae9f6b0c62be" ON "user_list_membership" ("userId") `);
await queryRunner.query(`CREATE INDEX "IDX_cddcaf418dc4d392ecfcca842a" ON "user_list_membership" ("userListId") `);
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_e4f3094c43f2d665e6030b0337" ON "user_list_membership" ("userId", "userListId") `);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_e4f3094c43f2d665e6030b0337"`);
await queryRunner.query(`DROP INDEX "public"."IDX_cddcaf418dc4d392ecfcca842a"`);
await queryRunner.query(`DROP INDEX "public"."IDX_021015e6683570ae9f6b0c62be"`);
await queryRunner.query(`DROP INDEX "public"."IDX_58699f75b9cf904f5f007909cb"`);
}
}

16
packages/backend/migration/1748104955717-index_IDX_instance_host_key.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class IndexIDXInstanceHostKey1748104955717 {
name = 'IndexIDXInstanceHostKey1748104955717'
async up(queryRunner) {
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_instance_host_key" ON "instance" (((lower(reverse("host")) || '.')::text) text_pattern_ops)`);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "IDX_instance_host_key"`);
}
}

85
packages/backend/migration/1748105111513-add_instance_block_columns.js Normal file
View file

@ -0,0 +1,85 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
* @typedef {{ blockedHosts: string[], silencedHosts: string[], mediaSilencedHosts: string[], federationHosts: string[], bubbleInstances: string[] }} Meta
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AddInstanceBlockColumns1748105111513 {
name = 'AddInstanceBlockColumns1748105111513'
async up(queryRunner) {
// Schema migration
await queryRunner.query(`ALTER TABLE "instance" ADD "isBlocked" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isBlocked" IS 'True if this instance is blocked from federation.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isAllowListed" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isAllowListed" IS 'True if this instance is allow-listed.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isBubbled" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isBubbled" IS 'True if this instance is part of the local bubble.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isSilenced" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isSilenced" IS 'True if this instance is silenced.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isMediaSilenced" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isMediaSilenced" IS 'True if this instance is media-silenced.'`);
// Data migration
/** @type {Meta[]} */
const metas = await queryRunner.query(`SELECT "blockedHosts", "silencedHosts", "mediaSilencedHosts", "federationHosts", "bubbleInstances" FROM "meta"`);
if (metas.length > 0) {
/** @type {Meta} */
const meta = metas[0];
// Blocked hosts
if (meta.blockedHosts.length > 0) {
const patterns = buildPatterns(meta.blockedHosts);
await queryRunner.query(`UPDATE "instance" SET "isBlocked" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Silenced hosts
if (meta.silencedHosts.length > 0) {
const patterns = buildPatterns(meta.silencedHosts);
await queryRunner.query(`UPDATE "instance" SET "isSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Media silenced hosts
if (meta.mediaSilencedHosts.length > 0) {
const patterns = buildPatterns(meta.mediaSilencedHosts);
await queryRunner.query(`UPDATE "instance" SET "isMediaSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Allow-listed hosts
if (meta.federationHosts.length > 0) {
const patterns = buildPatterns(meta.federationHosts);
await queryRunner.query(`UPDATE "instance" SET "isAllowListed" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Bubbled hosts
if (meta.bubbleInstances.length > 0) {
const patterns = buildPatterns(meta.bubbleInstances);
await queryRunner.query(`UPDATE "instance" SET "isBubbled" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
}
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isMediaSilenced"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isSilenced"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isBubbled"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isAllowListed"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isBlocked"`);
}
}
/**
* @param {string[]} input
* @returns {string[]}
*/
function buildPatterns(input) {
return input.map(i => i.toLowerCase().split('').reverse().join('') + '.%');
}

44
packages/backend/migration/1748128176881-add_instance_foreign_keys.js Normal file
View file

@ -0,0 +1,44 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AddInstanceForeignKeys1748128176881 {
name = 'AddInstanceForeignKeys1748128176881'
async up(queryRunner) {
// Fix-up: Some older instances have users without a matching instance entry
await queryRunner.query(`
INSERT INTO "instance" ("id", "host", "firstRetrievedAt")
SELECT
MIN("id"),
"host",
COALESCE(MIN("lastFetchedAt"), CURRENT_TIMESTAMP)
FROM "user"
WHERE
"host" IS NOT NULL AND
NOT EXISTS (select 1 from "instance" where "instance"."host" = "user"."host")
GROUP BY "host"
`);
await queryRunner.query(`ALTER TABLE "user" ADD CONSTRAINT "FK_user_host" FOREIGN KEY ("host") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_userHost" FOREIGN KEY ("userHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_replyUserHost" FOREIGN KEY ("replyUserHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_renoteUserHost" FOREIGN KEY ("renoteUserHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_renoteUserHost"`);
await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_replyUserHost"`);
await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_userHost"`);
await queryRunner.query(`ALTER TABLE "user" DROP CONSTRAINT "FK_user_host"`);
}
}

26
packages/backend/migration/1748137683887-add_instance_foreign_keys_to_following.js Normal file
View file

@ -0,0 +1,26 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AddInstanceForeignKeysToFollowing1748137683887 {
name = 'AddInstanceForeignKeysToFollowing1748137683887'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "following" ADD CONSTRAINT "FK_following_followerHost" FOREIGN KEY ("followerHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "following" ADD CONSTRAINT "FK_following_followeeHost" FOREIGN KEY ("followeeHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "following" DROP CONSTRAINT "FK_following_followeeHost"`);
await queryRunner.query(`ALTER TABLE "following" DROP CONSTRAINT "FK_following_followerHost"`);
}
}

25
packages/backend/migration/1748191631151-analyze_instance-user-note-following.js Normal file
View file

@ -0,0 +1,25 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AnalyzeInstanceUserNoteFollowing1748191631151 {
name = 'AnalyzeInstanceUserNoteFollowing1748191631151'
async up(queryRunner) {
// Refresh statistics for tables impacted by new indexes.
// This helps the query planner to efficiently use them without waiting for the next full vacuum.
await queryRunner.query(`ANALYZE "instance", "user", "following", "note"`);
}
async down(queryRunner) {
}
}

3
packages/backend/package.json
View file

@ -10,6 +10,9 @@
"start": "node ./built/boot/entry.js",
"start:test": "cross-env NODE_ENV=test node ./built/boot/entry.js",
"migrate": "pnpm typeorm migration:run -d ormconfig.js",
"migrate:revert": "pnpm typeorm migration:revert -d ormconfig.js",
"migrate:generate": "pnpm typeorm migration:generate -d ormconfig.js",
"migrate:create": "pnpm typeorm migration:create",
"revert": "pnpm typeorm migration:revert -d ormconfig.js",
"check:connect": "node ./scripts/check_connect.js",
"build": "swc src -d built -D --strip-leading-paths",

33
packages/backend/src/config.ts
View file

@ -9,6 +9,7 @@ import { dirname, resolve } from 'node:path';
import * as yaml from 'js-yaml';
import { globSync } from 'glob';
import ipaddr from 'ipaddr.js';
import Logger from './logger.js';
import type * as Sentry from '@sentry/node';
import type * as SentryVue from '@sentry/vue';
import type { RedisOptions } from 'ioredis';
@ -40,6 +41,7 @@ type Source = {
db?: string;
user?: string;
pass?: string;
slowQueryThreshold?: number;
disableCache?: boolean;
extra?: { [x: string]: string };
};
@ -111,6 +113,7 @@ type Source = {
deliverJobMaxAttempts?: number;
inboxJobMaxAttempts?: number;
mediaDirectory?: string;
mediaProxy?: string;
proxyRemoteFiles?: boolean;
videoThumbnailGenerator?: string;
@ -154,6 +157,8 @@ type Source = {
}
};
const configLogger = new Logger('config');
export type PrivateNetworkSource = string | { network?: string, ports?: number[] };
export type PrivateNetwork = {
@ -191,7 +196,7 @@ export function parsePrivateNetworks(patterns: PrivateNetworkSource[] | undefine
}
}
console.warn('[config] Skipping invalid entry in allowedPrivateNetworks: ', e);
configLogger.warn('Skipping invalid entry in allowedPrivateNetworks: ', e);
return null;
})
.filter(p => p != null);
@ -221,6 +226,7 @@ export type Config = {
db: string;
user: string;
pass: string;
slowQueryThreshold?: number;
disableCache?: boolean;
extra?: { [x: string]: string };
};
@ -297,6 +303,7 @@ export type Config = {
frontendManifestExists: boolean;
frontendEmbedEntry: string;
frontendEmbedManifestExists: boolean;
mediaDirectory: string;
mediaProxy: string;
externalMediaProxyEnabled: boolean;
videoThumbnailGenerator: string | null;
@ -346,7 +353,7 @@ const _dirname = dirname(_filename);
/**
* Path of configuration directory
*/
const dir = `${_dirname}/../../../.config`;
const dir = process.env.MISSKEY_CONFIG_DIR ?? `${_dirname}/../../../.config`;
/**
* Path of configuration file
@ -373,11 +380,14 @@ export function loadConfig(): Config {
if (configFiles.length === 0
&& !process.env['MK_WARNED_ABOUT_CONFIG']) {
console.log('No config files loaded, check if this is intentional');
configLogger.warn('No config files loaded, check if this is intentional');
process.env['MK_WARNED_ABOUT_CONFIG'] = '1';
}
const config = configFiles.map(path => fs.readFileSync(path, 'utf-8'))
const config = configFiles.map(path => {
configLogger.info(`Reading configuration from ${path}`);
return fs.readFileSync(path, 'utf-8');
})
.map(contents => yaml.load(contents) as Source)
.reduce(
(acc: Source, cur: Source) => Object.assign(acc, cur),
@ -403,6 +413,10 @@ export function loadConfig(): Config {
const internalMediaProxy = `${scheme}://${host}/proxy`;
const redis = convertRedisOptions(config.redis, host);
// nullish => 300 (default)
// 0 => undefined (disabled)
const slowQueryThreshold = (config.db.slowQueryThreshold ?? 300) || undefined;
return {
version,
publishTarballInsteadOfProvideRepositoryUrl: !!config.publishTarballInsteadOfProvideRepositoryUrl,
@ -421,7 +435,7 @@ export function loadConfig(): Config {
apiUrl: `${scheme}://${host}/api`,
authUrl: `${scheme}://${host}/auth`,
driveUrl: `${scheme}://${host}/files`,
db: { ...config.db, db: dbDb, user: dbUser, pass: dbPass },
db: { ...config.db, db: dbDb, user: dbUser, pass: dbPass, slowQueryThreshold },
dbReplications: config.dbReplications,
dbSlaves: config.dbSlaves,
fulltextSearch: config.fulltextSearch,
@ -463,6 +477,7 @@ export function loadConfig(): Config {
signToActivityPubGet: config.signToActivityPubGet ?? true,
attachLdSignatureForRelays: config.attachLdSignatureForRelays ?? true,
checkActivityPubGetSignature: config.checkActivityPubGetSignature,
mediaDirectory: config.mediaDirectory ?? resolve(_dirname, '../../../files'),
mediaProxy: externalMediaProxy ?? internalMediaProxy,
externalMediaProxyEnabled: externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy,
videoThumbnailGenerator: config.videoThumbnailGenerator ?
@ -493,6 +508,10 @@ export function loadConfig(): Config {
}
function tryCreateUrl(url: string) {
if (!url) {
throw new Error('Failed to load: no "url" property found in config. Please check the value of "MISSKEY_CONFIG_DIR" and "MISSKEY_CONFIG_YML", and verify that all configuration files are correct.');
}
try {
return new URL(url);
} catch (e) {
@ -624,7 +643,7 @@ function applyEnvOverrides(config: Source) {
// these are all the settings that can be overridden
_apply_top([['url', 'port', 'address', 'socket', 'chmodSocket', 'disableHsts', 'id', 'dbReplications', 'websocketCompression']]);
_apply_top(['db', ['host', 'port', 'db', 'user', 'pass', 'disableCache']]);
_apply_top(['db', ['host', 'port', 'db', 'user', 'pass', 'slowQueryThreshold', 'disableCache']]);
_apply_top(['dbSlaves', Array.from((config.dbSlaves ?? []).keys()), ['host', 'port', 'db', 'user', 'pass']]);
_apply_top([
['redis', 'redisForPubsub', 'redisForJobQueue', 'redisForTimelines', 'redisForReactions', 'redisForRateLimit'],
@ -638,7 +657,7 @@ function applyEnvOverrides(config: Source) {
_apply_top(['sentryForFrontend', 'vueIntegration', 'tracingOptions', 'timeout']);
_apply_top(['sentryForFrontend', 'browserTracingIntegration', 'routeLabel']);
_apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]);
_apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'proxyRemoteFiles', 'videoThumbnailGenerator']]);
_apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaDirectory', 'mediaProxy', 'proxyRemoteFiles', 'videoThumbnailGenerator']]);
_apply_top([['maxFileSize', 'maxNoteLength', 'maxRemoteNoteLength', 'maxAltTextLength', 'maxRemoteAltTextLength', 'pidFile', 'filePermissionBits']]);
_apply_top(['import', ['downloadTimeout', 'maxFileSize']]);
_apply_top([['signToActivityPubGet', 'checkActivityPubGetSignature', 'setupPassword', 'disallowExternalApRedirect']]);

8
packages/backend/src/core/DriveService.ts
View file

@ -159,6 +159,14 @@ export class DriveService {
// thunbnail, webpublic を必要なら生成
const alts = await this.generateAlts(path, type, !file.uri);
if (type && type.startsWith('video/')) {
try {
await this.videoProcessingService.webOptimizeVideo(path, type);
} catch (err) {
this.registerLogger.warn(`Video optimization failed: ${err instanceof Error ? err.message : String(err)}`, { error: err });
}
}
if (this.meta.useObjectStorage) {
//#region ObjectStorage params
let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) ?? ['']);

11
packages/backend/src/core/FanoutTimelineEndpointService.ts
View file

@ -136,10 +136,10 @@ export class FanoutTimelineEndpointService {
const parentFilter = filter;
filter = (note) => {
if (!ps.ignoreAuthorFromInstanceBlock) {
if (this.utilityService.isBlockedHost(this.meta.blockedHosts, note.userHost)) return false;
if (note.userInstance?.isBlocked) return false;
}
if (note.userId !== note.renoteUserId && this.utilityService.isBlockedHost(this.meta.blockedHosts, note.renoteUserHost)) return false;
if (note.userId !== note.replyUserId && this.utilityService.isBlockedHost(this.meta.blockedHosts, note.replyUserHost)) return false;
if (note.userId !== note.renoteUserId && note.renoteUserInstance?.isBlocked) return false;
if (note.userId !== note.replyUserId && note.replyUserInstance?.isBlocked) return false;
return parentFilter(note);
};
@ -194,7 +194,10 @@ export class FanoutTimelineEndpointService {
.leftJoinAndSelect('note.renote', 'renote')
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser')
.leftJoinAndSelect('note.channel', 'channel');
.leftJoinAndSelect('note.channel', 'channel')
.leftJoinAndSelect('note.userInstance', 'userInstance')
.leftJoinAndSelect('note.replyUserInstance', 'replyUserInstance')
.leftJoinAndSelect('note.renoteUserInstance', 'renoteUserInstance');
const notes = (await query.getMany()).filter(noteFilter);

103
packages/backend/src/core/FederatedInstanceService.ts
View file

@ -5,23 +5,24 @@
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import * as Redis from 'ioredis';
import { QueryFailedError } from 'typeorm';
import type { InstancesRepository } from '@/models/_.js';
import type { InstancesRepository, MiMeta } from '@/models/_.js';
import type { MiInstance } from '@/models/Instance.js';
import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js';
import { MemoryKVCache } from '@/misc/cache.js';
import { IdService } from '@/core/IdService.js';
import { DI } from '@/di-symbols.js';
import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { Serialized } from '@/types.js';
import { diffArrays, diffArraysSimple } from '@/misc/diff-arrays.js';
@Injectable()
export class FederatedInstanceService implements OnApplicationShutdown {
public federatedInstanceCache: RedisKVCache<MiInstance | null>;
private readonly federatedInstanceCache: MemoryKVCache<MiInstance | null>;
constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis,
@Inject(DI.redisForSub)
private redisForSub: Redis.Redis,
@Inject(DI.instancesRepository)
private instancesRepository: InstancesRepository,
@ -29,67 +30,46 @@ export class FederatedInstanceService implements OnApplicationShutdown {
private utilityService: UtilityService,
private idService: IdService,
) {
this.federatedInstanceCache = new RedisKVCache<MiInstance | null>(this.redisClient, 'federatedInstance', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60 * 3, // 3m
fetcher: (key) => this.instancesRepository.findOneBy({ host: key }),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => {
const parsed = JSON.parse(value);
if (parsed == null) return null;
return {
...parsed,
firstRetrievedAt: new Date(parsed.firstRetrievedAt),
latestRequestReceivedAt: parsed.latestRequestReceivedAt ? new Date(parsed.latestRequestReceivedAt) : null,
infoUpdatedAt: parsed.infoUpdatedAt ? new Date(parsed.infoUpdatedAt) : null,
notRespondingSince: parsed.notRespondingSince ? new Date(parsed.notRespondingSince) : null,
};
},
});
this.federatedInstanceCache = new MemoryKVCache(1000 * 60 * 3); // 3m
this.redisForSub.on('message', this.onMessage);
}
@bindThis
public async fetchOrRegister(host: string): Promise<MiInstance> {
host = this.utilityService.toPuny(host);
const cached = await this.federatedInstanceCache.get(host);
const cached = this.federatedInstanceCache.get(host);
if (cached) return cached;
const index = await this.instancesRepository.findOneBy({ host });
let index = await this.instancesRepository.findOneBy({ host });
if (index == null) {
let i;
try {
i = await this.instancesRepository.insertOne({
await this.instancesRepository.createQueryBuilder('instance')
.insert()
.values({
id: this.idService.gen(),
host,
firstRetrievedAt: new Date(),
});
} catch (e: unknown) {
if (e instanceof QueryFailedError) {
if (isDuplicateKeyValueError(e)) {
i = await this.instancesRepository.findOneBy({ host });
}
}
isBlocked: this.utilityService.isBlockedHost(host),
isSilenced: this.utilityService.isSilencedHost(host),
isMediaSilenced: this.utilityService.isMediaSilencedHost(host),
isAllowListed: this.utilityService.isAllowListedHost(host),
isBubbled: this.utilityService.isBubbledHost(host),
})
.orIgnore()
.execute();
if (i == null) {
throw e;
}
}
this.federatedInstanceCache.set(host, i);
return i;
} else {
this.federatedInstanceCache.set(host, index);
return index;
index = await this.instancesRepository.findOneByOrFail({ host });
}
this.federatedInstanceCache.set(host, index);
return index;
}
@bindThis
public async fetch(host: string): Promise<MiInstance | null> {
host = this.utilityService.toPuny(host);
const cached = await this.federatedInstanceCache.get(host);
const cached = this.federatedInstanceCache.get(host);
if (cached !== undefined) return cached;
const index = await this.instancesRepository.findOneBy({ host });
@ -117,8 +97,35 @@ export class FederatedInstanceService implements OnApplicationShutdown {
this.federatedInstanceCache.set(result.host, result);
}
private syncCache(before: Serialized<MiMeta | undefined>, after: Serialized<MiMeta>): void {
const changed =
diffArraysSimple(before?.blockedHosts, after.blockedHosts) ||
diffArraysSimple(before?.silencedHosts, after.silencedHosts) ||
diffArraysSimple(before?.mediaSilencedHosts, after.mediaSilencedHosts) ||
diffArraysSimple(before?.federationHosts, after.federationHosts) ||
diffArraysSimple(before?.bubbleInstances, after.bubbleInstances);
if (changed) {
// We have to clear the whole thing, otherwise subdomains won't be synced.
this.federatedInstanceCache.clear();
}
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
if (type === 'metaUpdated') {
this.syncCache(body.before, body.after);
}
}
}
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.federatedInstanceCache.dispose();
}

10
packages/backend/src/core/HttpRequestService.ts
View file

@ -235,7 +235,9 @@ export class HttpRequestService {
}
@bindThis
public async getActivityJson(url: string, isLocalAddressAllowed = false): Promise<IObjectWithId> {
public async getActivityJson(url: string, isLocalAddressAllowed = false, allowAnonymous = false): Promise<IObjectWithId> {
this.apUtilityService.assertApUrl(url);
const res = await this.send(url, {
method: 'GET',
headers: {
@ -253,7 +255,11 @@ export class HttpRequestService {
// Make sure the object ID matches the final URL (which is where it actually exists).
// The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match.
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
if (allowAnonymous && activity.id == null) {
activity.id = res.url;
} else {
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
}
return activity as IObjectWithId;
}

11
packages/backend/src/core/InternalStorageService.ts
View file

@ -6,18 +6,11 @@
import * as fs from 'node:fs';
import { copyFile, unlink, writeFile, chmod } from 'node:fs/promises';
import * as Path from 'node:path';
import { fileURLToPath } from 'node:url';
import { dirname } from 'node:path';
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { bindThis } from '@/decorators.js';
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
const path = Path.resolve(_dirname, '../../../../files');
@Injectable()
export class InternalStorageService {
constructor(
@ -25,12 +18,12 @@ export class InternalStorageService {
private config: Config,
) {
// No one should erase the working directory *while the server is running*.
fs.mkdirSync(path, { recursive: true });
fs.mkdirSync(this.config.mediaDirectory, { recursive: true });
}
@bindThis
public resolvePath(key: string) {
return Path.resolve(path, key);
return Path.resolve(this.config.mediaDirectory, key);
}
@bindThis

113
packages/backend/src/core/MetaService.ts
View file

@ -4,7 +4,7 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { DataSource } from 'typeorm';
import { DataSource, EntityManager } from 'typeorm';
import * as Redis from 'ioredis';
import { DI } from '@/di-symbols.js';
import { MiMeta } from '@/models/Meta.js';
@ -12,6 +12,9 @@ import { GlobalEventService } from '@/core/GlobalEventService.js';
import { bindThis } from '@/decorators.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { FeaturedService } from '@/core/FeaturedService.js';
import { MiInstance } from '@/models/Instance.js';
import { diffArrays } from '@/misc/diff-arrays.js';
import type { MetasRepository } from '@/models/_.js';
import type { OnApplicationShutdown } from '@nestjs/common';
@Injectable()
@ -26,6 +29,9 @@ export class MetaService implements OnApplicationShutdown {
@Inject(DI.db)
private db: DataSource,
@Inject(DI.metasRepository)
private readonly metasRepository: MetasRepository,
private featuredService: FeaturedService,
private globalEventService: GlobalEventService,
) {
@ -67,35 +73,35 @@ export class MetaService implements OnApplicationShutdown {
public async fetch(noCache = false): Promise<MiMeta> {
if (!noCache && this.cache) return this.cache;
return await this.db.transaction(async transactionalEntityManager => {
// 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する
const metas = await transactionalEntityManager.find(MiMeta, {
order: {
// 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する
let meta = await this.metasRepository.createQueryBuilder('meta')
.select()
.orderBy({
id: 'DESC',
})
.limit(1)
.getOne();
if (!meta) {
await this.metasRepository.createQueryBuilder('meta')
.insert()
.values({
id: 'x',
})
.orIgnore()
.execute();
meta = await this.metasRepository.createQueryBuilder('meta')
.select()
.orderBy({
id: 'DESC',
},
});
})
.limit(1)
.getOneOrFail();
}
const meta = metas[0];
if (meta) {
this.cache = meta;
return meta;
} else {
// metaが空のときfetchMetaが同時に呼ばれるとここが同時に呼ばれてしまうことがあるのでフェイルセーフなupsertを使う
const saved = await transactionalEntityManager
.upsert(
MiMeta,
{
id: 'x',
},
['id'],
)
.then((x) => transactionalEntityManager.findOneByOrFail(MiMeta, x.identifiers[0]));
this.cache = saved;
return saved;
}
});
this.cache = meta;
return meta;
}
@bindThis
@ -103,7 +109,7 @@ export class MetaService implements OnApplicationShutdown {
let before: MiMeta | undefined;
const updated = await this.db.transaction(async transactionalEntityManager => {
const metas = await transactionalEntityManager.find(MiMeta, {
const metas: (MiMeta | undefined)[] = await transactionalEntityManager.find(MiMeta, {
order: {
id: 'DESC',
},
@ -126,6 +132,10 @@ export class MetaService implements OnApplicationShutdown {
},
});
// Propagate changes to blockedHosts, silencedHosts, mediaSilencedHosts, federationInstances, and bubbleInstances to the relevant instance rows
// Do this inside the transaction to avoid potential race condition (when an instance gets registered while we're updating).
await this.persistBlocks(transactionalEntityManager, before ?? {}, afters[0]);
return afters[0];
});
@ -159,4 +169,49 @@ export class MetaService implements OnApplicationShutdown {
public onApplicationShutdown(signal?: string | undefined): void {
this.dispose();
}
private async persistBlocks(tem: EntityManager, before: Partial<MiMeta>, after: Partial<MiMeta>): Promise<void> {
await this.persistBlock(tem, before.blockedHosts, after.blockedHosts, 'isBlocked');
await this.persistBlock(tem, before.silencedHosts, after.silencedHosts, 'isSilenced');
await this.persistBlock(tem, before.mediaSilencedHosts, after.mediaSilencedHosts, 'isMediaSilenced');
await this.persistBlock(tem, before.federationHosts, after.federationHosts, 'isAllowListed');
await this.persistBlock(tem, before.bubbleInstances, after.bubbleInstances, 'isBubbled');
}
private async persistBlock(tem: EntityManager, before: string[] | undefined, after: string[] | undefined, field: keyof MiInstance): Promise<void> {
const { added, removed } = diffArrays(before, after);
if (removed.length > 0) {
await this.updateInstancesByHost(tem, field, false, removed);
}
if (added.length > 0) {
await this.updateInstancesByHost(tem, field, true, added);
}
}
private async updateInstancesByHost(tem: EntityManager, field: keyof MiInstance, value: boolean, hosts: string[]): Promise<void> {
// Use non-array queries when possible, as they are indexed and can be much faster.
if (hosts.length === 1) {
const pattern = genHostPattern(hosts[0]);
await tem
.createQueryBuilder(MiInstance, 'instance')
.update()
.set({ [field]: value })
.where('(lower(reverse("host")) || \'.\') LIKE :pattern', { pattern })
.execute();
} else if (hosts.length > 1) {
const patterns = hosts.map(host => genHostPattern(host));
await tem
.createQueryBuilder(MiInstance, 'instance')
.update()
.set({ [field]: value })
.where('(lower(reverse("host")) || \'.\') LIKE ANY (:patterns)', { patterns })
.execute();
}
}
}
function genHostPattern(host: string): string {
return host.toLowerCase().split('').reverse().join('') + '.%';
}

61
packages/backend/src/core/QueryService.ts
View file

@ -243,47 +243,44 @@ export class QueryService {
q.andWhere(new Brackets(qb => {
qb
.where(new Brackets(qb => {
qb.where('note.renoteId IS NOT NULL');
qb.andWhere('note.text IS NULL');
qb.andWhere(`note.userId NOT IN (${ mutingQuery.getQuery() })`);
}))
.orWhere('note.renoteId IS NULL')
.orWhere('note.text IS NOT NULL');
.orWhere('note.text IS NOT NULL')
.orWhere('note.cw IS NOT NULL')
.orWhere('note.replyId IS NOT NULL')
.orWhere('note.hasPoll = false')
.orWhere('note.fileIds != \'{}\'')
.orWhere(`note.userId NOT IN (${ mutingQuery.getQuery() })`);
}));
q.setParameters(mutingQuery.getParameters());
}
@bindThis
public generateBlockedHostQueryForNote(q: SelectQueryBuilder<any>, excludeAuthor?: boolean): void {
let nonBlockedHostQuery: (part: string) => string;
if (this.meta.blockedHosts.length === 0) {
nonBlockedHostQuery = () => '1=1';
} else {
nonBlockedHostQuery = (match: string) => `('.' || ${match}) NOT ILIKE ALL(select '%.' || x from (select unnest("blockedHosts") as x from "meta") t)`;
public generateBlockedHostQueryForNote(q: SelectQueryBuilder<any>, excludeAuthor?: boolean, allowSilenced = true): void {
function checkFor(key: 'user' | 'replyUser' | 'renoteUser') {
q.leftJoin(`note.${key}Instance`, `${key}Instance`);
q.andWhere(new Brackets(qb => {
qb.orWhere(`note.${key}Id IS NULL`) // no corresponding user
.orWhere(`note.${key}Host IS NULL`); // local
if (allowSilenced) {
qb.orWhere(`${key}Instance.isBlocked = false`); // not blocked
} else {
qb.orWhere(new Brackets(qbb => qbb
.andWhere(`${key}Instance.isBlocked = false`) // not blocked
.andWhere(`${key}Instance.isSilenced = false`))); // not silenced
}
if (excludeAuthor) {
qb.orWhere(`note.userId = note.${key}Id`); // author
}
}));
}
if (excludeAuthor) {
const instanceSuspension = (user: string) => new Brackets(qb => qb
.where(`note.${user}Id IS NULL`) // no corresponding user
.orWhere(`note.userId = note.${user}Id`)
.orWhere(`note.${user}Host IS NULL`) // local
.orWhere(nonBlockedHostQuery(`note.${user}Host`)));
q
.andWhere(instanceSuspension('replyUser'))
.andWhere(instanceSuspension('renoteUser'));
} else {
const instanceSuspension = (user: string) => new Brackets(qb => qb
.where(`note.${user}Id IS NULL`) // no corresponding user
.orWhere(`note.${user}Host IS NULL`) // local
.orWhere(nonBlockedHostQuery(`note.${user}Host`)));
q
.andWhere(instanceSuspension('user'))
.andWhere(instanceSuspension('replyUser'))
.andWhere(instanceSuspension('renoteUser'));
if (!excludeAuthor) {
checkFor('user');
}
checkFor('replyUser');
checkFor('renoteUser');
}
}

44
packages/backend/src/core/ReactionService.ts
View file

@ -10,7 +10,7 @@ import { IdentifiableError } from '@/misc/identifiable-error.js';
import type { MiRemoteUser, MiUser } from '@/models/User.js';
import type { MiNote } from '@/models/Note.js';
import { IdService } from '@/core/IdService.js';
import type { MiNoteReaction } from '@/models/NoteReaction.js';
import { MiNoteReaction } from '@/models/NoteReaction.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { NotificationService } from '@/core/NotificationService.js';
@ -31,6 +31,7 @@ import { isQuote, isRenote } from '@/misc/is-renote.js';
import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js';
import { PER_NOTE_REACTION_USER_PAIR_CACHE_MAX } from '@/const.js';
import { CacheService } from '@/core/CacheService.js';
import type { DataSource } from 'typeorm';
const FALLBACK = '\u2764';
@ -89,6 +90,9 @@ export class ReactionService {
@Inject(DI.emojisRepository)
private emojisRepository: EmojisRepository,
@Inject(DI.db)
private readonly db: DataSource,
private utilityService: UtilityService,
private customEmojiService: CustomEmojiService,
private roleService: RoleService,
@ -176,26 +180,28 @@ export class ReactionService {
reaction,
};
try {
await this.noteReactionsRepository.insert(record);
} catch (e) {
if (isDuplicateKeyValueError(e)) {
const exists = await this.noteReactionsRepository.findOneByOrFail({
noteId: note.id,
userId: user.id,
});
const result = await this.db.transaction(async tem => {
await tem.createQueryBuilder(MiNoteReaction, 'noteReaction')
.insert()
.values(record)
.orIgnore()
.execute();
if (exists.reaction !== reaction) {
// 別のリアクションがすでにされていたら置き換える
await this.delete(user, note);
await this.noteReactionsRepository.insert(record);
} else {
// 同じリアクションがすでにされていたらエラー
throw new IdentifiableError('51c42bb4-931a-456b-bff7-e5a8a70dd298');
}
} else {
throw e;
return await tem.createQueryBuilder(MiNoteReaction, 'noteReaction')
.select()
.where({ noteId: note.id, userId: user.id })
.getOneOrFail();
});
if (result.id !== record.id) {
// Conflict with the same ID => nothing to do.
if (result.reaction === record.reaction) {
return;
}
// 別のリアクションがすでにされていたら置き換える
await this.delete(user, note);
await this.noteReactionsRepository.insert(record);
}
// Increment reactions count

2
packages/backend/src/core/ReversiService.ts
View file

@ -587,6 +587,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
lastActiveDate: parsed.user1.lastActiveDate != null ? new Date(parsed.user1.lastActiveDate) : null,
lastFetchedAt: parsed.user1.lastFetchedAt != null ? new Date(parsed.user1.lastFetchedAt) : null,
movedAt: parsed.user1.movedAt != null ? new Date(parsed.user1.movedAt) : null,
instance: null,
} : null,
user2: parsed.user2 != null ? {
...parsed.user2,
@ -597,6 +598,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
lastActiveDate: parsed.user2.lastActiveDate != null ? new Date(parsed.user2.lastActiveDate) : null,
lastFetchedAt: parsed.user2.lastFetchedAt != null ? new Date(parsed.user2.lastFetchedAt) : null,
movedAt: parsed.user2.movedAt != null ? new Date(parsed.user2.movedAt) : null,
instance: null,
} : null,
};
} else {

4
packages/backend/src/core/RoleService.ts
View file

@ -86,10 +86,10 @@ export const DEFAULT_POLICIES: RolePolicies = {
canManageCustomEmojis: false,
canManageAvatarDecorations: false,
canSearchNotes: false,
canUseTranslator: true,
canUseTranslator: false,
canHideAds: false,
driveCapacityMb: 100,
maxFileSizeMb: 10,
maxFileSizeMb: 25,
alwaysMarkNsfw: false,
canUpdateBioMedia: true,
pinLimit: 5,

39
packages/backend/src/core/UtilityService.ts
View file

@ -49,22 +49,49 @@ export class UtilityService {
return regexp.test(email);
}
public isBlockedHost(host: string | null): boolean;
public isBlockedHost(blockedHosts: string[], host: string | null): boolean;
@bindThis
public isBlockedHost(blockedHosts: string[], host: string | null): boolean {
public isBlockedHost(blockedHostsOrHost: string[] | string | null, host?: string | null): boolean {
const blockedHosts = Array.isArray(blockedHostsOrHost) ? blockedHostsOrHost : this.meta.blockedHosts;
host = Array.isArray(blockedHostsOrHost) ? host : blockedHostsOrHost;
if (host == null) return false;
return blockedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
public isSilencedHost(host: string | null): boolean;
public isSilencedHost(silencedHosts: string[], host: string | null): boolean;
@bindThis
public isSilencedHost(silencedHosts: string[] | undefined, host: string | null): boolean {
if (!silencedHosts || host == null) return false;
public isSilencedHost(silencedHostsOrHost: string[] | string | null, host?: string | null): boolean {
const silencedHosts = Array.isArray(silencedHostsOrHost) ? silencedHostsOrHost : this.meta.silencedHosts;
host = Array.isArray(silencedHostsOrHost) ? host : silencedHostsOrHost;
if (host == null) return false;
return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
public isMediaSilencedHost(host: string | null): boolean;
public isMediaSilencedHost(silencedHosts: string[], host: string | null): boolean;
@bindThis
public isMediaSilencedHost(silencedHosts: string[] | undefined, host: string | null): boolean {
if (!silencedHosts || host == null) return false;
return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
public isMediaSilencedHost(mediaSilencedHostsOrHost: string[] | string | null, host?: string | null): boolean {
const mediaSilencedHosts = Array.isArray(mediaSilencedHostsOrHost) ? mediaSilencedHostsOrHost : this.meta.mediaSilencedHosts;
host = Array.isArray(mediaSilencedHostsOrHost) ? host : mediaSilencedHostsOrHost;
if (host == null) return false;
return mediaSilencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
@bindThis
public isAllowListedHost(host: string | null): boolean {
if (host == null) return false;
return this.meta.federationHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
@bindThis
public isBubbledHost(host: string | null): boolean {
if (host == null) return false;
return this.meta.bubbleInstances.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
@bindThis

64
packages/backend/src/core/VideoProcessingService.ts
View file

@ -3,24 +3,41 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import fs from 'node:fs/promises';
import { Inject, Injectable } from '@nestjs/common';
import FFmpeg from 'fluent-ffmpeg';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { ImageProcessingService } from '@/core/ImageProcessingService.js';
import type { IImage } from '@/core/ImageProcessingService.js';
import { createTempDir } from '@/misc/create-temp.js';
import { createTemp, createTempDir } from '@/misc/create-temp.js';
import { bindThis } from '@/decorators.js';
import { appendQuery, query } from '@/misc/prelude/url.js';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
// faststart is only supported for MP4, M4A, M4W and MOV files (the MOV family).
// WebM (and Matroska) files always support faststart-like behavior.
const supportedMimeTypes = new Map([
['video/mp4', 'mp4'],
['video/m4a', 'mp4'],
['video/m4v', 'mp4'],
['video/quicktime', 'mov'],
]);
@Injectable()
export class VideoProcessingService {
private readonly logger: Logger;
constructor(
@Inject(DI.config)
private config: Config,
private imageProcessingService: ImageProcessingService,
private loggerService: LoggerService,
) {
this.logger = this.loggerService.getLogger('video-processing');
}
@bindThis
@ -60,5 +77,50 @@ export class VideoProcessingService {
}),
);
}
/**
* Optimize video for web playback by adding faststart flag.
* This allows the video to start playing before it is fully downloaded.
* The original file is modified in-place.
* @param source Path to the video file
* @param mimeType The MIME type of the video
* @returns Promise that resolves when optimization is complete
*/
@bindThis
public async webOptimizeVideo(source: string, mimeType: string): Promise<void> {
const outputFormat = supportedMimeTypes.get(mimeType);
if (!outputFormat) {
this.logger.debug(`Skipping web optimization for unsupported MIME type: ${mimeType}`);
return;
}
const [tempPath, cleanup] = await createTemp();
try {
await new Promise<void>((resolve, reject) => {
FFmpeg(source)
.format(outputFormat) // Specify output format
.addOutputOptions('-c copy') // Copy streams without re-encoding
.addOutputOptions('-movflags +faststart')
.on('error', reject)
.on('end', async () => {
try {
// Replace original file with optimized version
await fs.copyFile(tempPath, source);
this.logger.info(`Web-optimized video: ${source}`);
resolve();
} catch (copyError) {
reject(copyError);
}
})
.save(tempPath);
});
} catch (error) {
this.logger.warn(`Failed to web-optimize video: ${source}`, { error });
throw error;
} finally {
cleanup();
}
}
}

5
packages/backend/src/core/WebhookTestService.ts
View file

@ -63,6 +63,7 @@ function generateDummyUser(override?: Partial<MiUser>): MiUser {
emojis: [],
score: 0,
host: null,
instance: null,
inbox: null,
sharedInbox: null,
featured: null,
@ -115,10 +116,13 @@ function generateDummyNote(override?: Partial<MiNote>): MiNote {
channelId: null,
channel: null,
userHost: null,
userInstance: null,
replyUserId: null,
replyUserHost: null,
replyUserInstance: null,
renoteUserId: null,
renoteUserHost: null,
renoteUserInstance: null,
updatedAt: null,
processErrors: [],
...override,
@ -450,6 +454,7 @@ export class WebhookTestService {
isAdmin: false,
isModerator: false,
isSystem: false,
instance: undefined,
...override,
};
}

37
packages/backend/src/core/activitypub/ApInboxService.ts
View file

@ -36,7 +36,7 @@ import InstanceChart from '@/core/chart/charts/instance.js';
import FederationChart from '@/core/chart/charts/federation.js';
import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js';
import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js';
import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js';
import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js';
import { ApNoteService } from './models/ApNoteService.js';
import { ApLoggerService } from './ApLoggerService.js';
import { ApDbResolverService } from './ApDbResolverService.js';
@ -106,22 +106,25 @@ export class ApInboxService {
let result = undefined as string | void;
if (isCollectionOrOrderedCollection(activity)) {
const results = [] as [string, string | void][];
// eslint-disable-next-line no-param-reassign
resolver ??= this.apResolverService.createResolver();
const items = toArray(isCollection(activity) ? activity.items : activity.orderedItems);
if (items.length >= resolver.getRecursionLimit()) {
throw new Error(`skipping activity: collection would surpass recursion limit: ${this.utilityService.extractDbHost(actor.uri)}`);
}
for (const item of items) {
const act = await resolver.resolve(item);
if (act.id == null || this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) {
this.logger.debug('skipping activity: activity id is null or mismatching');
continue;
const items = await resolver.resolveCollectionItems(activity);
for (let i = 0; i < items.length; i++) {
const act = items[i];
if (act.id != null) {
if (this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) {
this.logger.warn('skipping activity: activity id mismatch');
continue;
}
} else {
// Activity ID should only be string or undefined.
act.id = undefined;
}
try {
results.push([getApId(item), await this.performOneActivity(actor, act, resolver)]);
const id = getNullableApId(act) ?? `${getNullableApId(activity)}#${i}`;
const result = await this.performOneActivity(actor, act, resolver);
results.push([id, result]);
} catch (err) {
if (err instanceof Error || typeof err === 'string') {
this.logger.error(err);
@ -217,6 +220,10 @@ export class ApInboxService {
const note = await this.apNoteService.resolveNote(object, { resolver });
if (!note) return `skip: target note not found ${targetUri}`;
if (note.userHost == null && note.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot react to local-only note');
}
await this.apNoteService.extractEmojis(activity.tag ?? [], actor.host).catch(() => null);
try {
@ -371,6 +378,10 @@ export class ApInboxService {
return 'skip: invalid actor for this activity';
}
if (renote.userHost == null && renote.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot renote a local-only note');
}
this.logger.info(`Creating the (Re)Note: ${uri}`);
const activityAudience = await this.apAudienceService.parseAudience(actor, activity.to, activity.cc, resolver);

17
packages/backend/src/core/activitypub/ApRequestService.ts
View file

@ -155,6 +155,8 @@ export class ApRequestService {
@bindThis
public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, digest?: string): Promise<void> {
this.apUtilityService.assertApUrl(url);
const body = typeof object === 'string' ? object : JSON.stringify(object);
const keypair = await this.userKeypairService.getUserKeypair(user.id);
@ -182,10 +184,13 @@ export class ApRequestService {
* Get AP object with http-signature
* @param user http-signature user
* @param url URL to fetch
* @param followAlternate
* @param allowAnonymous If a fetched object lacks an ID, then it will be auto-generated from the final URL. (default: false)
* @param followAlternate Whether to resolve HTML responses to their referenced canonical AP endpoint. (default: true)
*/
@bindThis
public async signedGet(url: string, user: { id: MiUser['id'] }, followAlternate?: boolean): Promise<IObjectWithId> {
public async signedGet(url: string, user: { id: MiUser['id'] }, allowAnonymous = false, followAlternate?: boolean): Promise<IObjectWithId> {
this.apUtilityService.assertApUrl(url);
const _followAlternate = followAlternate ?? true;
const keypair = await this.userKeypairService.getUserKeypair(user.id);
@ -254,7 +259,7 @@ export class ApRequestService {
if (alternate) {
const href = alternate.getAttribute('href');
if (href && this.apUtilityService.haveSameAuthority(url, href)) {
return await this.signedGet(href, user, false);
return await this.signedGet(href, user, allowAnonymous, false);
}
}
} catch {
@ -271,7 +276,11 @@ export class ApRequestService {
// Make sure the object ID matches the final URL (which is where it actually exists).
// The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match.
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
if (allowAnonymous && activity.id == null) {
activity.id = res.url;
} else {
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
}
return activity as IObjectWithId;
}

161
packages/backend/src/core/activitypub/ApResolverService.ts
View file

@ -5,6 +5,7 @@
import { Inject, Injectable } from '@nestjs/common';
import { IsNull, Not } from 'typeorm';
import promiseLimit from 'promise-limit';
import type { MiLocalUser, MiRemoteUser } from '@/models/User.js';
import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta, SkApFetchLog } from '@/models/_.js';
import type { Config } from '@/config.js';
@ -19,11 +20,12 @@ import { ApLogService, calculateDurationSince, extractObjectContext } from '@/co
import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { getApId, getNullableApId, IObjectWithId, isCollectionOrOrderedCollection } from './type.js';
import { toArray } from '@/misc/prelude/array.js';
import { AnyCollection, getApId, getNullableApId, IObjectWithId, isCollection, isCollectionOrOrderedCollection, isCollectionPage, isOrderedCollection, isOrderedCollectionPage } from './type.js';
import { ApDbResolverService } from './ApDbResolverService.js';
import { ApRendererService } from './ApRendererService.js';
import { ApRequestService } from './ApRequestService.js';
import type { IObject, ICollection, IOrderedCollection, ApObject } from './type.js';
import type { IObject, ApObject, IAnonymousObject } from './type.js';
export class Resolver {
private history: Set<string>;
@ -63,11 +65,16 @@ export class Resolver {
return this.recursionLimit;
}
public async resolveCollection(value: string | IObjectWithId, allowAnonymous?: boolean, sentFromUri?: string): Promise<AnyCollection & IObjectWithId>;
public async resolveCollection(value: string | IObject, allowAnonymous: boolean | undefined, sentFromUri: string): Promise<AnyCollection & IObjectWithId>;
public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise<AnyCollection>;
@bindThis
public async resolveCollection(value: string | IObject): Promise<ICollection | IOrderedCollection> {
public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise<AnyCollection> {
const collection = typeof value === 'string'
? await this.resolve(value)
: value;
? sentFromUri
? await this.secureResolve(value, sentFromUri, allowAnonymous)
: await this.resolve(value, allowAnonymous)
: value; // TODO try and remove this eventually, as it's a major security foot-gun
if (isCollectionOrOrderedCollection(collection)) {
return collection;
@ -76,20 +83,110 @@ export class Resolver {
}
}
public async resolveCollectionItems(collection: IAnonymousObject, limit?: number | null, allowAnonymousItems?: true, concurrency?: number): Promise<IAnonymousObject[]>;
public async resolveCollectionItems(collection: string | IObjectWithId, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise<IObjectWithId[]>;
public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise<IObject[]>;
/**
* Recursively resolves items from a collection.
* Stops when reaching the resolution limit or an optional item limit - whichever is lower.
* This method supports Collection, OrderedCollection, and individual pages of either type.
* Malformed collections (mixing Ordered and un-Ordered types) are also supported.
* @param collection Collection to resolve from - can be a URL or object of any supported collection type.
* @param limit Maximum number of items to resolve. If null or undefined (default), then items will be resolved until reaching the recursion limit.
* @param allowAnonymousItems If true, collection items can be anonymous (lack an ID). If false (default), then an error is thrown when reaching an item without ID.
* @param concurrency Maximum number of items to resolve at once. (default: 4)
*/
@bindThis
public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency = 4): Promise<IObject[]> {
const resolvedItems: IObject[] = [];
// This is pulled up to avoid code duplication below
const iterate = async(items: ApObject, current: AnyCollection) => {
const sentFrom = current.id;
const itemArr = toArray(items);
const itemLimit = limit ?? Number.MAX_SAFE_INTEGER;
const allowAnonymous = allowAnonymousItems ?? false;
await this.resolveItemArray(itemArr, sentFrom, itemLimit, concurrency, allowAnonymous, resolvedItems);
};
let current: AnyCollection | null = await this.resolveCollection(collection);
do {
// Iterate all items in the current page
if (current.items) {
await iterate(current.items, current);
}
if (current.orderedItems) {
await iterate(current.orderedItems, current);
}
if (this.history.size >= this.recursionLimit) {
// Stop when we reach the fetch limit
current = null;
} else if (limit != null && resolvedItems.length >= limit) {
// Stop when we reach the item limit
current = null;
} else if (isCollection(current) || isOrderedCollection(current)) {
// Continue to first page
current = current.first ? await this.resolveCollection(current.first, true, current.id) : null;
} else if (isCollectionPage(current) || isOrderedCollectionPage(current)) {
// Continue to next page
current = current.next ? await this.resolveCollection(current.next, true, current.id) : null;
} else {
// Stop in all other conditions
current = null;
}
} while (current != null);
return resolvedItems;
}
private async resolveItemArray(source: (string | IObject)[], sentFrom: undefined, itemLimit: number, concurrency: number, allowAnonymousItems: true, destination: IAnonymousObject[]): Promise<void>;
private async resolveItemArray(source: (string | IObject)[], sentFrom: string, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObjectWithId[]): Promise<void>;
private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise<void>;
private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise<void> {
const recursionLimit = this.recursionLimit - this.history.size;
const batchLimit = Math.min(source.length, recursionLimit, itemLimit);
const limiter = promiseLimit<IObject>(concurrency);
const batch = await Promise.all(source
.slice(0, batchLimit)
.map(item => limiter(async () => {
if (sentFrom) {
// Use secureResolve to avoid re-fetching items that were included inline.
return await this.secureResolve(item, sentFrom, allowAnonymousItems);
} else if (allowAnonymousItems) {
return await this.resolveAnonymous(item);
} else {
// ID is required if we have neither sentFrom not allowAnonymousItems
const id = getApId(item);
return await this.resolve(id);
}
})));
destination.push(...batch);
};
/**
* Securely resolves an AP object or URL that has been sent from another instance.
* An input object is trusted if and only if its ID matches the authority of sentFromUri.
* In all other cases, the object is re-fetched from remote by input string or object ID.
* @param input The input object or URL to resolve
* @param sentFromUri The URL where this object originated. This MUST be accurate - all security checks depend on this value!
* @param allowAnonymous If true, anonymous objects are allowed and will have their ID set to sentFromUri. If false (default) then anonymous objects will be rejected with an error.
*/
@bindThis
public async secureResolve(input: ApObject, sentFromUri: string): Promise<IObjectWithId> {
public async secureResolve(input: string | IObject | [string | IObject], sentFromUri: string, allowAnonymous?: boolean): Promise<IObjectWithId> {
// Unpack arrays to get the value element.
const value = fromTuple(input);
if (value == null) {
throw new IdentifiableError('20058164-9de1-4573-8715-425753a21c1d', 'Cannot resolve null input');
// If anonymous input is allowed, then any object is automatically valid if we set the ID.
// We can short-circuit here and avoid un-necessary checks.
if (allowAnonymous && typeof(value) === 'object' && value.id == null) {
value.id = sentFromUri;
return value as IObjectWithId;
}
// This will throw if the input has no ID, which is good because we can't verify an anonymous object anyway.
// This ensures the input has a string ID, protecting against type confusion and rejecting anonymous objects.
const id = getApId(value);
// Check if we can use the provided object as-is.
@ -100,28 +197,52 @@ export class Resolver {
}
// If the checks didn't pass, then we must fetch the object and use that.
return await this.resolve(id);
return await this.resolve(id, allowAnonymous);
}
public async resolve(value: string | [string]): Promise<IObjectWithId>;
public async resolve(value: string | IObject | [string | IObject]): Promise<IObject>;
/**
* Resolves an anonymous object.
* The returned value will not have any ID present.
* If one is provided in the response, it will be removed automatically.
*/
@bindThis
public async resolve(value: string | IObject | [string | IObject]): Promise<IObject> {
public async resolveAnonymous(value: string | IObject | [string | IObject]): Promise<IAnonymousObject> {
value = fromTuple(value);
const object = await this.resolve(value);
object.id = undefined;
return object as IAnonymousObject;
}
public async resolve(value: string | [string], allowAnonymous?: boolean): Promise<IObjectWithId>;
public async resolve(value: string | IObjectWithId | [string | IObjectWithId], allowAnonymous?: boolean): Promise<IObjectWithId>;
public async resolve(value: string | IObject | [string | IObject], allowAnonymous?: boolean): Promise<IObject>;
/**
* Resolves a URL or object to an AP object.
* Tuples are expanded to their first element before anything else, and non-string inputs are returned as-is.
* Otherwise, the string URL is fetched and validated to represent a valid ActivityPub object.
* @param value The input value to resolve
* @param allowAnonymous Determines what to do if a response object lacks an ID field. If false (default), then an exception is thrown. If true, then the ID is populated from the final response URL.
*/
@bindThis
public async resolve(value: string | IObject | [string | IObject], allowAnonymous = false): Promise<IObject> {
value = fromTuple(value);
// TODO try and remove this eventually, as it's a major security foot-gun
if (typeof value !== 'string') {
return value;
}
const host = this.utilityService.extractDbHost(value);
if (this.config.activityLogging.enabled && !this.utilityService.isSelfHost(host)) {
return await this._resolveLogged(value, host);
return await this._resolveLogged(value, host, allowAnonymous);
} else {
return await this._resolve(value, host);
return await this._resolve(value, host, allowAnonymous);
}
}
private async _resolveLogged(requestUri: string, host: string): Promise<IObjectWithId> {
private async _resolveLogged(requestUri: string, host: string, allowAnonymous: boolean): Promise<IObjectWithId> {
const startTime = process.hrtime.bigint();
const log = await this.apLogService.createFetchLog({
@ -130,7 +251,7 @@ export class Resolver {
});
try {
const result = await this._resolve(requestUri, host, log);
const result = await this._resolve(requestUri, host, allowAnonymous, log);
log.accepted = true;
log.result = 'ok';
@ -150,7 +271,7 @@ export class Resolver {
}
}
private async _resolve(value: string, host: string, log?: SkApFetchLog): Promise<IObjectWithId> {
private async _resolve(value: string, host: string, allowAnonymous: boolean, log?: SkApFetchLog): Promise<IObjectWithId> {
if (value.includes('#')) {
// URLs with fragment parts cannot be resolved correctly because
// the fragment part does not get transmitted over HTTP(S).
@ -181,8 +302,8 @@ export class Resolver {
}
const object = (this.user
? await this.apRequestService.signedGet(value, this.user)
: await this.httpRequestService.getActivityJson(value));
? await this.apRequestService.signedGet(value, this.user, allowAnonymous)
: await this.httpRequestService.getActivityJson(value, false, allowAnonymous));
if (log) {
const { object: objectOnly, context, contextHash } = extractObjectContext(object);

32
packages/backend/src/core/activitypub/ApUtilityService.ts
View file

@ -77,16 +77,42 @@ export class ApUtilityService {
return acceptableUrls[0]?.url ?? null;
}
/**
* Verifies that a provided URL is in a format acceptable for federation.
* @throws {IdentifiableError} If URL cannot be parsed
* @throws {IdentifiableError} If URL is not HTTPS
*/
public assertApUrl(url: string | URL): void {
// If string, parse and validate
if (typeof(url) === 'string') {
try {
url = new URL(url);
} catch {
throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: not a valid URL`);
}
}
// Must be HTTPS
if (!this.checkHttps(url)) {
throw new IdentifiableError('0bedd29b-e3bf-4604-af51-d3352e2518af', `invalid AP url ${url}: unsupported protocol ${url.protocol}`);
}
}
/**
* Checks if the URL contains HTTPS.
* Additionally, allows HTTP in non-production environments.
* Based on check-https.ts.
*/
private checkHttps(url: string): boolean {
private checkHttps(url: string | URL): boolean {
const isNonProd = this.envService.env.NODE_ENV !== 'production';
// noinspection HttpUrlsUsage
return url.startsWith('https://') || (url.startsWith('http://') && isNonProd);
try {
const proto = new URL(url).protocol;
return proto === 'https:' || (proto === 'http:' && isNonProd);
} catch {
// Invalid URLs don't "count" as HTTPS
return false;
}
}
}

12
packages/backend/src/core/activitypub/models/ApNoteService.ts
View file

@ -95,6 +95,7 @@ export class ApNoteService {
actor?: MiRemoteUser,
user?: MiRemoteUser,
): Error | null {
this.apUtilityService.assertApUrl(uri);
const expectHost = this.utilityService.extractDbHost(uri);
const apType = getApType(object);
@ -284,6 +285,13 @@ export class ApNoteService {
const quote = await this.getQuote(note, entryUri, resolver);
const processErrors = quote === null ? ['quoteUnavailable'] : null;
if (reply && reply.userHost == null && reply.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot reply to local-only note');
}
if (quote && quote.userHost == null && quote.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot quote a local-only note');
}
// vote
if (reply && reply.hasPoll) {
const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id });
@ -481,6 +489,10 @@ export class ApNoteService {
const quote = await this.getQuote(note, entryUri, resolver);
const processErrors = quote === null ? ['quoteUnavailable'] : null;
if (quote && quote.userHost == null && quote.localOnly) {
throw new IdentifiableError('12e23cec-edd9-442b-aa48-9c21f0c3b215', 'Cannot quote a local-only note');
}
// vote
if (reply && reply.hasPoll) {
const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id });

48
packages/backend/src/core/activitypub/models/ApPersonService.ts
View file

@ -153,6 +153,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
*/
@bindThis
private validateActor(x: IObject, uri: string): IActor {
this.apUtilityService.assertApUrl(uri);
const expectHost = this.utilityService.punyHostPSLDomain(uri);
if (!isActor(x)) {
@ -167,6 +168,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox type`);
}
this.apUtilityService.assertApUrl(x.inbox);
const inboxHost = this.utilityService.punyHostPSLDomain(x.inbox);
if (inboxHost !== expectHost) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox ${inboxHost}`);
@ -175,6 +177,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const sharedInboxObject = x.sharedInbox ?? (x.endpoints ? x.endpoints.sharedInbox : undefined);
if (sharedInboxObject != null) {
const sharedInbox = getApId(sharedInboxObject);
this.apUtilityService.assertApUrl(sharedInbox);
if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHostPSLDomain(sharedInbox) === expectHost)) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong shared inbox ${sharedInbox}`);
}
@ -185,6 +188,7 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
if (xCollection != null) {
const collectionUri = getApId(xCollection);
if (typeof collectionUri === 'string' && collectionUri.length > 0) {
this.apUtilityService.assertApUrl(collectionUri);
if (this.utilityService.punyHostPSLDomain(collectionUri) !== expectHost) {
throw new UnrecoverableError(`invalid Actor ${uri} - wrong ${collection} ${collectionUri}`);
}
@ -352,8 +356,8 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const [followingVisibility, followersVisibility] = await Promise.all(
[
this.isPublicCollection(person.following, resolver),
this.isPublicCollection(person.followers, resolver),
this.isPublicCollection(person.following, resolver, uri),
this.isPublicCollection(person.followers, resolver, uri),
].map((p): Promise<'public' | 'private'> => p
.then(isPublic => isPublic ? 'public' : 'private')
.catch(err => {
@ -389,10 +393,18 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
//#endregion
//#region resolve counts
const _resolver = resolver ?? this.apResolverService.createResolver();
const outboxcollection = await _resolver.resolveCollection(person.outbox).catch(() => { return null; });
const followerscollection = await _resolver.resolveCollection(person.followers!).catch(() => { return null; });
const followingcollection = await _resolver.resolveCollection(person.following!).catch(() => { return null; });
const outboxCollection = person.outbox
? await resolver.resolveCollection(person.outbox, true, uri).catch(() => { return null; })
: null;
const followersCollection = person.followers
? await resolver.resolveCollection(person.followers, true, uri).catch(() => { return null; })
: null;
const followingCollection = person.following
? await resolver.resolveCollection(person.following, true, uri).catch(() => { return null; })
: null;
// Register the instance first, to avoid FK errors
await this.federatedInstanceService.fetchOrRegister(host);
try {
// Start transaction
@ -419,9 +431,9 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
host,
inbox: person.inbox,
sharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null,
notesCount: outboxcollection?.totalItems ?? 0,
followersCount: followerscollection?.totalItems ?? 0,
followingCount: followingcollection?.totalItems ?? 0,
notesCount: outboxCollection?.totalItems ?? 0,
followersCount: followersCollection?.totalItems ?? 0,
followingCount: followingCollection?.totalItems ?? 0,
followersUri: person.followers ? getApId(person.followers) : undefined,
featured: person.featured ? getApId(person.featured) : undefined,
uri: person.id,
@ -571,8 +583,8 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const [followingVisibility, followersVisibility] = await Promise.all(
[
this.isPublicCollection(person.following, resolver),
this.isPublicCollection(person.followers, resolver),
this.isPublicCollection(person.following, resolver, exist.uri),
this.isPublicCollection(person.followers, resolver, exist.uri),
].map((p): Promise<'public' | 'private' | undefined> => p
.then(isPublic => isPublic ? 'public' : 'private')
.catch(err => {
@ -797,13 +809,13 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const _resolver = resolver ?? this.apResolverService.createResolver();
// Resolve to (Ordered)Collection Object
const collection = await _resolver.resolveCollection(user.featured).catch(err => {
const collection = user.featured ? await _resolver.resolveCollection(user.featured, true, user.uri).catch(err => {
if (err instanceof AbortError || err instanceof StatusError) {
this.logger.warn(`Failed to update featured notes: ${err.name}: ${err.message}`);
} else {
this.logger.error('Failed to update featured notes:', err);
}
});
}) : null;
if (!collection) return;
if (!isCollectionOrOrderedCollection(collection)) throw new UnrecoverableError(`featured ${user.featured} is not Collection or OrderedCollection in ${user.uri}`);
@ -889,11 +901,13 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
}
@bindThis
private async isPublicCollection(collection: string | ICollection | IOrderedCollection | undefined, resolver: Resolver): Promise<boolean> {
private async isPublicCollection(collection: string | ICollection | IOrderedCollection | undefined, resolver: Resolver, sentFrom: string): Promise<boolean> {
if (collection) {
const resolved = await resolver.resolveCollection(collection);
if (resolved.first || (resolved as ICollection).items || (resolved as IOrderedCollection).orderedItems) {
return true;
const resolved = await resolver.resolveCollection(collection, true, sentFrom).catch(() => null);
if (resolved) {
if (resolved.first || (resolved as ICollection).items || (resolved as IOrderedCollection).orderedItems) {
return true;
}
}
}

60
packages/backend/src/core/activitypub/type.ts
View file

@ -43,6 +43,18 @@ export interface IObjectWithId extends IObject {
id: string;
}
export function isObjectWithId(object: IObject): object is IObjectWithId {
return typeof(object.id) === 'string';
}
export interface IAnonymousObject extends IObject {
id: undefined;
}
export function isAnonymousObject(object: IObject): object is IAnonymousObject {
return object.id === undefined;
}
/**
* Get array of ActivityStreams Objects id
*/
@ -125,48 +137,46 @@ export interface IActivity extends IObject {
};
}
export interface ICollection extends IObject {
export interface CollectionBase extends IObject {
totalItems?: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
partOf?: IObject | string;
next?: IObject | string;
prev?: IObject | string;
items?: ApObject;
orderedItems?: ApObject;
}
export interface ICollection extends CollectionBase {
type: 'Collection';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
items?: ApObject;
orderedItems?: undefined;
}
export interface IOrderedCollection extends IObject {
export interface IOrderedCollection extends CollectionBase {
type: 'OrderedCollection';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
items?: undefined;
orderedItems?: ApObject;
}
export interface ICollectionPage extends IObject {
export interface ICollectionPage extends CollectionBase {
type: 'CollectionPage';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
partOf?: IObject | string;
next?: IObject | string;
prev?: IObject | string;
items?: ApObject;
orderedItems?: undefined;
}
export interface IOrderedCollectionPage extends IObject {
export interface IOrderedCollectionPage extends CollectionBase {
type: 'OrderedCollectionPage';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
partOf?: IObject | string;
next?: IObject | string;
prev?: IObject | string;
items?: undefined;
orderedItems?: ApObject;
}
export type AnyCollection = ICollection | IOrderedCollection | ICollectionPage | IOrderedCollectionPage;
export const validPost = ['Note', 'Question', 'Article', 'Audio', 'Document', 'Image', 'Page', 'Video', 'Event'];
export const isPost = (object: IObject): object is IPost => {
@ -270,7 +280,7 @@ export const isCollectionPage = (object: IObject): object is ICollectionPage =>
export const isOrderedCollectionPage = (object: IObject): object is IOrderedCollectionPage =>
getApType(object) === 'OrderedCollectionPage';
export const isCollectionOrOrderedCollection = (object: IObject): object is ICollection | IOrderedCollection =>
export const isCollectionOrOrderedCollection = (object: IObject): object is AnyCollection =>
isCollection(object) || isOrderedCollection(object) || isCollectionPage(object) || isOrderedCollectionPage(object);
export interface IApPropertyValue extends IObject {

23
packages/backend/src/core/chart/charts/federation.ts
View file

@ -44,10 +44,6 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
const suspendedInstancesQuery = this.instancesRepository.createQueryBuilder('instance')
.select('instance.host')
.where('instance.suspensionState != \'none\'');
const pubsubSubQuery = this.followingsRepository.createQueryBuilder('f')
.select('f.followerHost')
.where('f.followerHost IS NOT NULL');
@ -64,22 +60,25 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followeeHost)')
.where('following.followeeHost IS NOT NULL')
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followeeHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.innerJoin('following.followeeInstance', 'followeeInstance')
.andWhere('followeeInstance.suspensionState = \'none\'')
.andWhere('followeeInstance.isBlocked = false')
.getRawOne()
.then(x => parseInt(x.count, 10)),
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followerHost)')
.where('following.followerHost IS NOT NULL')
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followerHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere(`following.followerHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.innerJoin('following.followerInstance', 'followerInstance')
.andWhere('followerInstance.isBlocked = false')
.andWhere('followerInstance.suspensionState = \'none\'')
.getRawOne()
.then(x => parseInt(x.count, 10)),
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followeeHost)')
.where('following.followeeHost IS NOT NULL')
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followeeHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.innerJoin('following.followeeInstance', 'followeeInstance')
.andWhere('followeeInstance.isBlocked = false')
.andWhere('followeeInstance.suspensionState = \'none\'')
.andWhere(`following.followeeHost IN (${ pubsubSubQuery.getQuery() })`)
.setParameters(pubsubSubQuery.getParameters())
.getRawOne()
@ -87,7 +86,7 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
this.instancesRepository.createQueryBuilder('instance')
.select('COUNT(instance.id)')
.where(`instance.host IN (${ subInstancesQuery.getQuery() })`)
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || instance.host) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere('instance.isBlocked = false')
.andWhere('instance.suspensionState = \'none\'')
.andWhere('instance.isNotResponding = false')
.getRawOne()
@ -95,7 +94,7 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
this.instancesRepository.createQueryBuilder('instance')
.select('COUNT(instance.id)')
.where(`instance.host IN (${ pubInstancesQuery.getQuery() })`)
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || instance.host) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere('instance.isBlocked = false')
.andWhere('instance.suspensionState = \'none\'')
.andWhere('instance.isNotResponding = false')
.getRawOne()

7
packages/backend/src/core/entities/InstanceEntityService.ts
View file

@ -43,7 +43,7 @@ export class InstanceEntityService {
isNotResponding: instance.isNotResponding,
isSuspended: instance.suspensionState !== 'none',
suspensionState: instance.suspensionState,
isBlocked: this.utilityService.isBlockedHost(this.meta.blockedHosts, instance.host),
isBlocked: instance.isBlocked,
softwareName: instance.softwareName,
softwareVersion: instance.softwareVersion,
openRegistrations: instance.openRegistrations,
@ -51,8 +51,8 @@ export class InstanceEntityService {
description: instance.description,
maintainerName: instance.maintainerName,
maintainerEmail: instance.maintainerEmail,
isSilenced: this.utilityService.isSilencedHost(this.meta.silencedHosts, instance.host),
isMediaSilenced: this.utilityService.isMediaSilencedHost(this.meta.mediaSilencedHosts, instance.host),
isSilenced: instance.isSilenced,
isMediaSilenced: instance.isMediaSilenced,
iconUrl: instance.iconUrl,
faviconUrl: instance.faviconUrl,
themeColor: instance.themeColor,
@ -62,6 +62,7 @@ export class InstanceEntityService {
rejectReports: instance.rejectReports,
rejectQuotes: instance.rejectQuotes,
moderationNote: iAmModerator ? instance.moderationNote : null,
isBubbled: this.utilityService.isBubbledHost(instance.host),
};
}

22
packages/backend/src/core/entities/NotificationEntityService.ts
View file

@ -23,6 +23,13 @@ import type { NoteEntityService } from './NoteEntityService.js';
const NOTE_REQUIRED_NOTIFICATION_TYPES = new Set(['note', 'mention', 'reply', 'renote', 'renote:grouped', 'quote', 'reaction', 'reaction:grouped', 'pollEnded', 'edited', 'scheduledNotePosted'] as (typeof groupedNotificationTypes[number])[]);
function undefOnMissing<T>(packPromise: Promise<T>): Promise<T | undefined> {
return packPromise.catch(err => {
if (err instanceof EntityNotFoundError) return undefined;
throw err;
});
}
@Injectable()
export class NotificationEntityService implements OnModuleInit {
private userEntityService: UserEntityService;
@ -75,9 +82,9 @@ export class NotificationEntityService implements OnModuleInit {
const noteIfNeed = needsNote ? (
hint?.packedNotes != null
? hint.packedNotes.get(notification.noteId)
: this.noteEntityService.pack(notification.noteId, { id: meId }, {
: undefOnMissing(this.noteEntityService.pack(notification.noteId, { id: meId }, {
detail: true,
})
}))
) : undefined;
// if the note has been deleted, don't show this notification
if (needsNote && !noteIfNeed) return null;
@ -86,7 +93,7 @@ export class NotificationEntityService implements OnModuleInit {
const userIfNeed = needsUser ? (
hint?.packedUsers != null
? hint.packedUsers.get(notification.notifierId)
: this.userEntityService.pack(notification.notifierId, { id: meId })
: undefOnMissing(this.userEntityService.pack(notification.notifierId, { id: meId }))
) : undefined;
// if the user has been deleted, don't show this notification
if (needsUser && !userIfNeed) return null;
@ -96,7 +103,7 @@ export class NotificationEntityService implements OnModuleInit {
const reactions = (await Promise.all(notification.reactions.map(async reaction => {
const user = hint?.packedUsers != null
? hint.packedUsers.get(reaction.userId)!
: await this.userEntityService.pack(reaction.userId, { id: meId });
: await undefOnMissing(this.userEntityService.pack(reaction.userId, { id: meId }));
return {
user,
reaction: reaction.reaction,
@ -121,7 +128,7 @@ export class NotificationEntityService implements OnModuleInit {
return packedUser;
}
return this.userEntityService.pack(userId, { id: meId });
return undefOnMissing(this.userEntityService.pack(userId, { id: meId }));
}))).filter(x => x != null);
// if all users have been deleted, don't show this notification
if (users.length === 0) {
@ -140,10 +147,7 @@ export class NotificationEntityService implements OnModuleInit {
const needsRole = notification.type === 'roleAssigned';
const role = needsRole
? await this.roleEntityService.pack(notification.roleId).catch(err => {
if (err instanceof EntityNotFoundError) return undefined;
throw err;
})
? await undefOnMissing(this.roleEntityService.pack(notification.roleId))
: undefined;
// if the role has been deleted, don't show this notification
if (needsRole && !role) {

2
packages/backend/src/core/entities/UserEntityService.ts
View file

@ -610,7 +610,7 @@ export class UserEntityService implements OnModuleInit {
requireSigninToViewContents: user.requireSigninToViewContents === false ? undefined : true,
makeNotesFollowersOnlyBefore: user.makeNotesFollowersOnlyBefore ?? undefined,
makeNotesHiddenBefore: user.makeNotesHiddenBefore ?? undefined,
instance: user.host ? this.federatedInstanceService.federatedInstanceCache.fetch(user.host).then(instance => instance ? {
instance: user.host ? this.federatedInstanceService.fetch(user.host).then(instance => instance ? {
name: instance.name,
softwareName: instance.softwareName,
softwareVersion: instance.softwareVersion,

1
packages/backend/src/env.ts
View file

@ -11,6 +11,7 @@ const envOption = {
verbose: false,
withLogTime: false,
quiet: false,
hideWorkerId: false,
};
for (const key of Object.keys(envOption) as (keyof typeof envOption)[]) {

4
packages/backend/src/logger.ts
View file

@ -71,7 +71,9 @@ export default class Logger {
level === 'info' ? message :
null;
let log = `${l} ${worker}\t[${contexts.join(' ')}]\t${m}`;
let log = envOption.hideWorkerId
? `${l}\t[${contexts.join(' ')}]\t\t${m}`
: `${l} ${worker}\t[${contexts.join(' ')}]\t\t${m}`;
if (envOption.withLogTime) log = chalk.gray(time) + ' ' + log;
const args: unknown[] = [important ? chalk.bold(log) : log];

9
packages/backend/src/misc/cache.ts
View file

@ -308,8 +308,17 @@ export class MemoryKVCache<T> {
}
}
/**
* Removes all entries from the cache, but does not dispose it.
*/
@bindThis
public clear(): void {
this.cache.clear();
}
@bindThis
public dispose(): void {
this.clear();
clearInterval(this.gcIntervalHandle);
}

102
packages/backend/src/misc/diff-arrays.ts Normal file
View file

@ -0,0 +1,102 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export interface DiffResult<T> {
added: T[];
removed: T[];
}
/**
* Calculates the difference between two snapshots of data.
* Null, undefined, and empty arrays are supported, and duplicate values are ignored.
* Result sets are de-duplicated, and will be empty if no data was added or removed (respectively).
* The inputs are treated as un-ordered, so a re-ordering of the same data will NOT be considered a change.
* @param dataBefore Array containing data before the change
* @param dataAfter Array containing data after the change
*/
export function diffArrays<T>(dataBefore: T[] | null | undefined, dataAfter: T[] | null | undefined): DiffResult<T> {
const before = dataBefore ? new Set(dataBefore) : null;
const after = dataAfter ? new Set(dataAfter) : null;
// data before AND after => changed
if (before?.size && after?.size) {
const added: T[] = [];
const removed: T[] = [];
for (const host of before) {
// before and NOT after => removed
// delete operation removes duplicates to speed up the "after" loop
if (!after.delete(host)) {
removed.push(host);
}
}
for (const host of after) {
// after and NOT before => added
if (!before.has(host)) {
added.push(host);
}
}
return { added, removed };
}
// data ONLY before => all removed
if (before?.size) {
return { added: [], removed: Array.from(before) };
}
// data ONLY after => all added
if (after?.size) {
return { added: Array.from(after), removed: [] };
}
// data NEITHER before nor after => no change
return { added: [], removed: [] };
}
/**
* Checks for any difference between two snapshots of data.
* Null, undefined, and empty arrays are supported, and duplicate values are ignored.
* The inputs are treated as un-ordered, so a re-ordering of the same data will NOT be considered a change.
* @param dataBefore Array containing data before the change
* @param dataAfter Array containing data after the change
*/
export function diffArraysSimple<T>(dataBefore: T[] | null | undefined, dataAfter: T[] | null | undefined): boolean {
const before = dataBefore ? new Set(dataBefore) : null;
const after = dataAfter ? new Set(dataAfter) : null;
if (before?.size && after?.size) {
// different size => changed
if (before.size !== after.size) return true;
// removed => changed
for (const host of before) {
// delete operation removes duplicates to speed up the "after" loop
if (!after.delete(host)) {
return true;
}
}
// added => changed
for (const host of after) {
if (!before.has(host)) {
return true;
}
}
// identical values => no change
return false;
}
// before and NOT after => change
if (before?.size) return true;
// after and NOT before => change
if (after?.size) return true;
// NEITHER before nor after => no change
return false;
}

21
packages/backend/src/models/Following.ts
View file

@ -4,6 +4,7 @@
*/
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm';
import { MiInstance } from '@/models/Instance.js';
import { id } from './util/id.js';
import { MiUser } from './User.js';
@ -66,6 +67,16 @@ export class MiFollowing {
})
public followerHost: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'followerHost',
foreignKeyConstraintName: 'FK_following_followerHost',
referencedColumnName: 'host',
})
public followerInstance: MiInstance | null;
@Column('varchar', {
length: 512, nullable: true,
comment: '[Denormalized]',
@ -85,6 +96,16 @@ export class MiFollowing {
})
public followeeHost: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'followeeHost',
foreignKeyConstraintName: 'FK_following_followeeHost',
referencedColumnName: 'host',
})
public followeeInstance: MiInstance | null;
@Column('varchar', {
length: 512, nullable: true,
comment: '[Denormalized]',

51
packages/backend/src/models/Instance.ts
View file

@ -6,6 +6,7 @@
import { Entity, PrimaryColumn, Index, Column } from 'typeorm';
import { id } from './util/id.js';
@Index('IDX_instance_host_key', { synchronize: false })
@Entity('instance')
export class MiInstance {
@PrimaryColumn(id())
@ -98,6 +99,56 @@ export class MiInstance {
})
public suspensionState: 'none' | 'manuallySuspended' | 'goneSuspended' | 'autoSuspendedForNotResponding';
/**
* True if this instance is blocked from federation.
*/
@Column('boolean', {
nullable: false,
default: false,
comment: 'True if this instance is blocked from federation.',
})
public isBlocked: boolean;
/**
* True if this instance is allow-listed.
*/
@Column('boolean', {
nullable: false,
default: false,
comment: 'True if this instance is allow-listed.',
})
public isAllowListed: boolean;
/**
* True if this instance is part of the local bubble.
*/
@Column('boolean', {
nullable: false,
default: false,
comment: 'True if this instance is part of the local bubble.',
})
public isBubbled: boolean;
/**
* True if this instance is silenced.
*/
@Column('boolean', {
nullable: false,
default: false,
comment: 'True if this instance is silenced.',
})
public isSilenced: boolean;
/**
* True if this instance is media-silenced.
*/
@Column('boolean', {
nullable: false,
default: false,
comment: 'True if this instance is media-silenced.',
})
public isMediaSilenced: boolean;
@Column('varchar', {
length: 64, nullable: true,
comment: 'The software of the Instance.',

31
packages/backend/src/models/Note.ts
View file

@ -5,6 +5,7 @@
import { Entity, Index, JoinColumn, Column, PrimaryColumn, ManyToOne } from 'typeorm';
import { noteVisibilities } from '@/types.js';
import { MiInstance } from '@/models/Instance.js';
import { id } from './util/id.js';
import { MiUser } from './User.js';
import { MiChannel } from './Channel.js';
@ -222,6 +223,16 @@ export class MiNote {
})
public userHost: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'userHost',
foreignKeyConstraintName: 'FK_note_userHost',
referencedColumnName: 'host',
})
public userInstance: MiInstance | null;
@Column({
...id(),
nullable: true,
@ -235,6 +246,16 @@ export class MiNote {
})
public replyUserHost: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'replyUserHost',
foreignKeyConstraintName: 'FK_note_replyUserHost',
referencedColumnName: 'host',
})
public replyUserInstance: MiInstance | null;
@Column({
...id(),
nullable: true,
@ -247,6 +268,16 @@ export class MiNote {
comment: '[Denormalized]',
})
public renoteUserHost: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'renoteUserHost',
foreignKeyConstraintName: 'FK_note_renoteUserHost',
referencedColumnName: 'host',
})
public renoteUserInstance: MiInstance | null;
//#endregion
constructor(data: Partial<MiNote>) {

13
packages/backend/src/models/User.ts
View file

@ -3,8 +3,9 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Entity, Column, Index, OneToOne, JoinColumn, PrimaryColumn } from 'typeorm';
import { Entity, Column, Index, OneToOne, JoinColumn, PrimaryColumn, ManyToOne } from 'typeorm';
import { type UserUnsignedFetchOption, userUnsignedFetchOptions } from '@/const.js';
import { MiInstance } from '@/models/Instance.js';
import { id } from './util/id.js';
import { MiDriveFile } from './DriveFile.js';
@ -292,6 +293,16 @@ export class MiUser {
})
public host: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'host',
foreignKeyConstraintName: 'FK_user_host',
referencedColumnName: 'host',
})
public instance: MiInstance | null;
@Column('varchar', {
length: 512, nullable: true,
comment: 'The inbox URL of the User. It will be null if the origin of the user is local.',

4
packages/backend/src/models/json-schema/federation-instance.ts
View file

@ -135,5 +135,9 @@ export const packedFederationInstanceSchema = {
type: 'string',
optional: true, nullable: true,
},
isBubbled: {
type: 'boolean',
optional: false, nullable: false,
},
},
} as const;

49
packages/backend/src/postgres.ts
View file

@ -98,9 +98,12 @@ pg.types.setTypeParser(20, Number);
export const dbLogger = new MisskeyLogger('db');
const sqlLogger = dbLogger.createSubLogger('sql', 'gray');
const sqlMigrateLogger = sqlLogger.createSubLogger('migrate');
const sqlSchemaLogger = sqlLogger.createSubLogger('schema');
export type LoggerProps = {
disableQueryTruncation?: boolean;
enableQueryLogging?: boolean;
enableQueryParamLogging?: boolean;
printReplicationMode?: boolean,
};
@ -112,7 +115,7 @@ function highlightSql(sql: string) {
}
function truncateSql(sql: string) {
return sql.length > 100 ? `${sql.substring(0, 100)}...` : sql;
return sql.length > 100 ? `${sql.substring(0, 100)} [truncated]` : sql;
}
function stringifyParameter(param: any) {
@ -136,7 +139,7 @@ class MyCustomLogger implements Logger {
modded = truncateSql(modded);
}
return highlightSql(modded);
return this.props.enableQueryLogging ? highlightSql(modded) : modded;
}
@bindThis
@ -150,6 +153,8 @@ class MyCustomLogger implements Logger {
@bindThis
public logQuery(query: string, parameters?: any[], queryRunner?: QueryRunner) {
if (!this.props.enableQueryLogging) return;
const prefix = (this.props.printReplicationMode && queryRunner)
? `[${queryRunner.getReplicationMode()}] `
: undefined;
@ -161,7 +166,8 @@ class MyCustomLogger implements Logger {
const prefix = (this.props.printReplicationMode && queryRunner)
? `[${queryRunner.getReplicationMode()}] `
: undefined;
sqlLogger.error(this.transformQueryLog(query, { prefix }), this.transformParameters(parameters));
const transformed = this.transformQueryLog(query, { prefix });
sqlLogger.error(`Query error (${error}): ${transformed}`, this.transformParameters(parameters));
}
@bindThis
@ -169,22 +175,32 @@ class MyCustomLogger implements Logger {
const prefix = (this.props.printReplicationMode && queryRunner)
? `[${queryRunner.getReplicationMode()}] `
: undefined;
sqlLogger.warn(this.transformQueryLog(query, { prefix }), this.transformParameters(parameters));
const transformed = this.transformQueryLog(query, { prefix });
sqlLogger.warn(`Query is slow (${time}ms): ${transformed}`, this.transformParameters(parameters));
}
@bindThis
public logSchemaBuild(message: string) {
sqlLogger.info(message);
sqlSchemaLogger.debug(message);
}
@bindThis
public log(message: string) {
sqlLogger.info(message);
public log(level: 'log' | 'info' | 'warn', message: string) {
switch (level) {
case 'log':
case 'info': {
sqlLogger.info(message);
break;
}
case 'warn': {
sqlLogger.warn(message);
}
}
}
@bindThis
public logMigration(message: string) {
sqlLogger.info(message);
sqlMigrateLogger.debug(message);
}
}
@ -306,7 +322,7 @@ export function createPostgresDataSource(config: Config) {
} : {}),
synchronize: process.env.NODE_ENV === 'test',
dropSchema: process.env.NODE_ENV === 'test',
cache: !config.db.disableCache && process.env.NODE_ENV !== 'test' ? { // dbをcloseしても何故かredisのコネクションが内部的に残り続けるようで、テストの際に支障が出るため無効にする(キャッシュも含めてテストしたいため本当は有効にしたいが...)
cache: config.db.disableCache === false && process.env.NODE_ENV !== 'test' ? { // dbをcloseしても何故かredisのコネクションが内部的に残り続けるようで、テストの際に支障が出るため無効にする(キャッシュも含めてテストしたいため本当は有効にしたいが...)
type: 'ioredis',
options: {
...config.redis,
@ -314,14 +330,13 @@ export function createPostgresDataSource(config: Config) {
},
} : false,
logging: log,
logger: log
? new MyCustomLogger({
disableQueryTruncation: config.logging?.sql?.disableQueryTruncation,
enableQueryParamLogging: config.logging?.sql?.enableQueryParamLogging,
printReplicationMode: !!config.dbReplications,
})
: undefined,
maxQueryExecutionTime: 300,
logger: new MyCustomLogger({
disableQueryTruncation: config.logging?.sql?.disableQueryTruncation,
enableQueryLogging: log,
enableQueryParamLogging: config.logging?.sql?.enableQueryParamLogging,
printReplicationMode: !!config.dbReplications,
}),
maxQueryExecutionTime: config.db.slowQueryThreshold,
entities: entities,
migrations: ['../../migration/*.js'],
});

16
packages/backend/src/server/api/ApiCallService.ts
View file

@ -344,14 +344,14 @@ export class ApiCallService implements OnApplicationShutdown {
}
if (ep.meta.requireCredential || ep.meta.requireModerator || ep.meta.requireAdmin) {
if (user == null) {
if (user == null && ep.meta.requireCredential !== 'optional') {
throw new ApiError({
message: 'Credential required.',
code: 'CREDENTIAL_REQUIRED',
id: '1384574d-a912-4b81-8601-c7b1c4085df1',
httpStatusCode: 401,
});
} else if (user!.isSuspended) {
} else if (user?.isSuspended) {
throw new ApiError({
message: 'Your account has been suspended.',
code: 'YOUR_ACCOUNT_SUSPENDED',
@ -372,8 +372,8 @@ export class ApiCallService implements OnApplicationShutdown {
}
}
if ((ep.meta.requireModerator || ep.meta.requireAdmin) && (this.meta.rootUserId !== user!.id)) {
const myRoles = await this.roleService.getUserRoles(user!.id);
if ((ep.meta.requireModerator || ep.meta.requireAdmin) && (this.meta.rootUserId !== user?.id)) {
const myRoles = user ? await this.roleService.getUserRoles(user) : [];
if (ep.meta.requireModerator && !myRoles.some(r => r.isModerator || r.isAdministrator)) {
throw new ApiError({
message: 'You are not assigned to a moderator role.',
@ -392,9 +392,9 @@ export class ApiCallService implements OnApplicationShutdown {
}
}
if (ep.meta.requiredRolePolicy != null && (this.meta.rootUserId !== user!.id)) {
const myRoles = await this.roleService.getUserRoles(user!.id);
const policies = await this.roleService.getUserPolicies(user!.id);
if (ep.meta.requiredRolePolicy != null && (this.meta.rootUserId !== user?.id)) {
const myRoles = user ? await this.roleService.getUserRoles(user) : [];
const policies = await this.roleService.getUserPolicies(user ?? null);
if (!policies[ep.meta.requiredRolePolicy] && !myRoles.some(r => r.isAdministrator)) {
throw new ApiError({
message: 'You are not assigned to a required role.',
@ -418,7 +418,7 @@ export class ApiCallService implements OnApplicationShutdown {
// Cast non JSON input
if ((ep.meta.requireFile || request.method === 'GET') && ep.params.properties) {
for (const k of Object.keys(ep.params.properties)) {
const param = ep.params.properties![k];
const param = ep.params.properties[k];
if (['boolean', 'number', 'integer'].includes(param.type ?? '') && typeof data[k] === 'string') {
try {
data[k] = JSON.parse(data[k]);

2
packages/backend/src/server/api/endpoints.ts
View file

@ -92,7 +92,7 @@ export type IEndpointMeta = (Omit<IEndpointMetaBase, 'requireCrential' | 'requir
}) | (Omit<IEndpointMetaBase, 'secure'> & {
secure: true,
}) | (Omit<IEndpointMetaBase, 'requireCredential' | 'kind'> & {
requireCredential: true,
requireCredential: true | 'optional',
kind: (typeof permissions)[number],
}) | (Omit<IEndpointMetaBase, 'requireModerator' | 'kind'> & {
requireModerator: true,

6
packages/backend/src/server/api/endpoints/admin/show-user.ts
View file

@ -122,6 +122,10 @@ export const meta = {
type: 'boolean',
optional: false, nullable: false,
},
isAdministrator: {
type: 'boolean',
optional: false, nullable: false,
},
isSystem: {
type: 'boolean',
optional: false, nullable: false,
@ -257,6 +261,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
}
const isModerator = await this.roleService.isModerator(user);
const isAdministrator = await this.roleService.isAdministrator(user);
const isSilenced = user.isSilenced || !(await this.roleService.getUserPolicies(user.id)).canPublicNote;
const _me = await this.usersRepository.findOneByOrFail({ id: me.id });
@ -289,6 +294,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
mutedInstances: profile.mutedInstances,
notificationRecieveConfig: profile.notificationRecieveConfig,
isModerator: isModerator,
isAdministrator: isAdministrator,
isSystem: isSystemAccount(user),
isSilenced: isSilenced,
isSuspended: user.isSuspended,

1
packages/backend/src/server/api/endpoints/antennas/notes.ts
View file

@ -121,6 +121,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
this.queryService.generateVisibilityQuery(query, me);
this.queryService.generateMutedUserQueryForNotes(query, me);
this.queryService.generateBlockedUserQueryForNotes(query, me);
this.queryService.generateMutedUserRenotesQueryForNotes(query, me);
const notes = await query.getMany();
if (sinceId != null && untilId == null) {

17
packages/backend/src/server/api/endpoints/ap/get.ts
View file

@ -7,6 +7,7 @@ import { Injectable } from '@nestjs/common';
import ms from 'ms';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { ApResolverService } from '@/core/activitypub/ApResolverService.js';
import { isCollectionOrOrderedCollection, isOrderedCollection, isOrderedCollectionPage } from '@/core/activitypub/type.js';
export const meta = {
tags: ['federation'],
@ -33,6 +34,9 @@ export const paramDef = {
type: 'object',
properties: {
uri: { type: 'string' },
expandCollectionItems: { type: 'boolean' },
expandCollectionLimit: { type: 'integer', nullable: true },
allowAnonymous: { type: 'boolean' },
},
required: ['uri'],
} as const;
@ -44,7 +48,18 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
) {
super(meta, paramDef, async (ps, me) => {
const resolver = this.apResolverService.createResolver();
const object = await resolver.resolve(ps.uri);
const object = await resolver.resolve(ps.uri, ps.allowAnonymous ?? false);
if (ps.expandCollectionItems && isCollectionOrOrderedCollection(object)) {
const items = await resolver.resolveCollectionItems(object, ps.expandCollectionLimit, ps.allowAnonymous ?? false);
if (isOrderedCollection(object) || isOrderedCollectionPage(object)) {
object.orderedItems = items;
} else {
object.items = items;
}
}
return object;
});
}

2
packages/backend/src/server/api/endpoints/channels/timeline.ts
View file

@ -138,9 +138,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
.leftJoinAndSelect('note.channel', 'channel');
this.queryService.generateBlockedHostQueryForNote(query);
this.queryService.generateVisibilityQuery(query, me);
if (me) {
this.queryService.generateMutedUserQueryForNotes(query, me);
this.queryService.generateBlockedUserQueryForNotes(query, me);
this.queryService.generateMutedUserRenotesQueryForNotes(query, me);
}
if (ps.withRenotes === false) {

6
packages/backend/src/server/api/endpoints/charts/active-users.ts
View file

@ -17,11 +17,11 @@ export const meta = {
allowGet: true,
cacheSec: 60 * 60,
// Burst up to 100, then 2/sec average
// Burst up to 200, then 5/sec average
limit: {
type: 'bucket',
size: 100,
dripRate: 500,
size: 200,
dripRate: 200,
},
} as const;

6
packages/backend/src/server/api/endpoints/charts/ap-request.ts
View file

@ -17,11 +17,11 @@ export const meta = {
allowGet: true,
cacheSec: 60 * 60,
// Burst up to 100, then 2/sec average
// Burst up to 200, then 5/sec average
limit: {
type: 'bucket',
size: 100,
dripRate: 500,
size: 200,
dripRate: 200,
},
} as const;

6
packages/backend/src/server/api/endpoints/charts/drive.ts
View file

@ -17,11 +17,11 @@ export const meta = {
allowGet: true,
cacheSec: 60 * 60,
// Burst up to 100, then 2/sec average
// Burst up to 200, then 5/sec average
limit: {
type: 'bucket',
size: 100,
dripRate: 500,
size: 200,
dripRate: 200,
},
} as const;

6
packages/backend/src/server/api/endpoints/charts/federation.ts
View file

@ -17,11 +17,11 @@ export const meta = {
allowGet: true,
cacheSec: 60 * 60,
// Burst up to 100, then 2/sec average
// Burst up to 200, then 5/sec average
limit: {
type: 'bucket',
size: 100,
dripRate: 500,
size: 200,
dripRate: 200,
},
} as const;

6
packages/backend/src/server/api/endpoints/charts/instance.ts
View file

@ -17,11 +17,11 @@ export const meta = {
allowGet: true,
cacheSec: 60 * 60,
// Burst up to 100, then 2/sec average
// Burst up to 200, then 5/sec average
limit: {
type: 'bucket',
size: 100,
dripRate: 500,
size: 200,
dripRate: 200,
},
} as const;

6
packages/backend/src/server/api/endpoints/charts/notes.ts
View file

@ -17,11 +17,11 @@ export const meta = {
allowGet: true,
cacheSec: 60 * 60,
// Burst up to 100, then 2/sec average
// Burst up to 200, then 5/sec average
limit: {
type: 'bucket',
size: 100,
dripRate: 500,
size: 200,
dripRate: 200,
},
} as const;

6
packages/backend/src/server/api/endpoints/charts/user/drive.ts
View file

@ -17,11 +17,11 @@ export const meta = {
allowGet: true,
cacheSec: 60 * 60,
// Burst up to 100, then 2/sec average
// Burst up to 200, then 5/sec average
limit: {
type: 'bucket',
size: 100,
dripRate: 500,
size: 200,
dripRate: 200,
},
} as const;

6
packages/backend/src/server/api/endpoints/charts/user/following.ts
View file

@ -17,11 +17,11 @@ export const meta = {
allowGet: true,
cacheSec: 60 * 60,
// Burst up to 100, then 2/sec average
// Burst up to 200, then 5/sec average
limit: {
type: 'bucket',
size: 100,
dripRate: 500,
size: 200,
dripRate: 200,
},
} as const;

6
packages/backend/src/server/api/endpoints/charts/user/notes.ts
View file

@ -17,11 +17,11 @@ export const meta = {
allowGet: true,
cacheSec: 60 * 60,
// Burst up to 100, then 2/sec average
// Burst up to 200, then 5/sec average
limit: {
type: 'bucket',
size: 100,
dripRate: 500,
size: 200,
dripRate: 200,
},
} as const;

6
packages/backend/src/server/api/endpoints/charts/user/pv.ts
View file

@ -17,11 +17,11 @@ export const meta = {
allowGet: true,
cacheSec: 60 * 60,
// Burst up to 100, then 2/sec average
// Burst up to 200, then 5/sec average
limit: {
type: 'bucket',
size: 100,
dripRate: 500,
size: 200,
dripRate: 200,
},
} as const;

6
packages/backend/src/server/api/endpoints/charts/user/reactions.ts
View file

@ -17,11 +17,11 @@ export const meta = {
allowGet: true,
cacheSec: 60 * 60,
// Burst up to 100, then 2/sec average
// Burst up to 200, then 5/sec average
limit: {
type: 'bucket',
size: 100,
dripRate: 500,
size: 200,
dripRate: 200,
},
} as const;

6
packages/backend/src/server/api/endpoints/charts/users.ts
View file

@ -17,11 +17,11 @@ export const meta = {
allowGet: true,
cacheSec: 60 * 60,
// Burst up to 100, then 2/sec average
// Burst up to 200, then 5/sec average
limit: {
type: 'bucket',
size: 100,
dripRate: 500,
size: 200,
dripRate: 200,
},
} as const;

3
packages/backend/src/server/api/endpoints/clips/notes.ts
View file

@ -92,10 +92,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
.andWhere('clipNote.clipId = :clipId', { clipId: clip.id });
this.queryService.generateBlockedHostQueryForNote(query);
this.queryService.generateVisibilityQuery(query, me);
if (me) {
this.queryService.generateVisibilityQuery(query, me);
this.queryService.generateMutedUserQueryForNotes(query, me);
this.queryService.generateBlockedUserQueryForNotes(query, me);
this.queryService.generateMutedUserRenotesQueryForNotes(query, me);
}
const notes = await query

34
packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts
View file

@ -74,18 +74,15 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
throw new ApiError(meta.errors.btlDisabled);
}
const [
followings,
] = me ? await Promise.all([
this.cacheService.userFollowingsCache.fetch(me.id),
]) : [undefined];
const followings = me ? await this.cacheService.userFollowingsCache.fetch(me.id) : undefined;
//#region Construct query
const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'),
ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate)
.andWhere('note.visibility = \'public\'')
.andWhere('note.channelId IS NULL')
.andWhere('note.userHost IN (:...hosts)', { hosts: this.serverSettings.bubbleInstances })
.andWhere('note.userHost IS NOT NULL')
.andWhere('userInstance.isBubbled = true') // This comes from generateBlockedHostQueryForNote below
.innerJoinAndSelect('note.user', 'user')
.leftJoinAndSelect('note.reply', 'reply')
.leftJoinAndSelect('note.renote', 'renote')
@ -97,6 +94,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
if (me) this.queryService.generateMutedUserQueryForNotes(query, me);
if (me) this.queryService.generateBlockedUserQueryForNotes(query, me);
if (me) this.queryService.generateMutedUserRenotesQueryForNotes(query, me);
if (!me) query.andWhere('user.requireSigninToViewContents = false');
if (ps.withFiles) {
query.andWhere('note.fileIds != \'{}\'');
@ -104,21 +102,27 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
if (!ps.withBots) query.andWhere('user.isBot = FALSE');
if (ps.withRenotes === false) {
query.andWhere(new Brackets(qb => {
qb.where('note.renoteId IS NULL');
qb.orWhere(new Brackets(qb => {
qb.where('note.text IS NOT NULL');
qb.orWhere('note.fileIds != \'{}\'');
}));
}));
if (!ps.withRenotes) {
query.andWhere(new Brackets(qb => qb
.orWhere('note.renoteId IS NULL')
.orWhere('note.text IS NOT NULL')
.orWhere('note.cw IS NOT NULL')
.orWhere('note.replyId IS NOT NULL')
.orWhere('note.hasPoll = false')
.orWhere('note.fileIds != \'{}\'')));
}
//#endregion
let timeline = await query.limit(ps.limit).getMany();
timeline = timeline.filter(note => {
if (note.user?.isSilenced && me && followings && note.userId !== me.id && !followings[note.userId]) return false;
if (note.user?.isSilenced) {
if (!me) return false;
if (!followings) return false;
if (note.userId !== me.id) {
return followings[note.userId];
}
}
return true;
});

13
packages/backend/src/server/api/endpoints/notes/following.ts
View file

@ -4,7 +4,7 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { ObjectLiteral, SelectQueryBuilder } from 'typeorm';
import { IsNull, ObjectLiteral, SelectQueryBuilder } from 'typeorm';
import { SkLatestNote, MiFollowing } from '@/models/_.js';
import type { NotesRepository } from '@/models/_.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
@ -130,7 +130,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
.leftJoinAndSelect('note.renote', 'renote')
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser')
.leftJoinAndSelect('note.channel', 'channel')
// Exclude channel notes
.andWhere({ channelId: IsNull() })
;
// Limit to files, if requested
@ -145,11 +147,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
// Hide blocked users / instances
query.andWhere('"user"."isSuspended" = false');
query.andWhere('("replyUser" IS NULL OR "replyUser"."isSuspended" = false)');
query.andWhere('("renoteUser" IS NULL OR "renoteUser"."isSuspended" = false)');
this.queryService.generateBlockedHostQueryForNote(query);
// Respect blocks and mutes
// Respect blocks, mutes, and privacy
this.queryService.generateVisibilityQuery(query, me);
this.queryService.generateBlockedUserQueryForNotes(query, me);
this.queryService.generateMutedUserQueryForNotes(query, me);
@ -161,7 +162,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
// Query and return the next page
const notes = await query.getMany();
return await this.noteEntityService.packMany(notes, me);
return await this.noteEntityService.packMany(notes, me, { skipHide: true });
});
}
}

94
packages/backend/src/server/api/endpoints/notes/polls/recommendation.ts
View file

@ -9,13 +9,13 @@ import type { NotesRepository, MutingsRepository, PollsRepository, PollVotesRepo
import { Endpoint } from '@/server/api/endpoint-base.js';
import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
import { DI } from '@/di-symbols.js';
import { QueryService } from '@/core/QueryService.js';
import { RoleService } from '@/core/RoleService.js';
import { ApiError } from '@/server/api/error.js';
export const meta = {
tags: ['notes'],
requireCredential: true,
kind: 'read:account',
res: {
type: 'array',
optional: false, nullable: false,
@ -26,10 +26,24 @@ export const meta = {
},
},
// 2 calls per second
errors: {
ltlDisabled: {
message: 'Local timeline has been disabled.',
code: 'LTL_DISABLED',
id: '45a6eb02-7695-4393-b023-dd3be9aaaefd',
},
gtlDisabled: {
message: 'Global timeline has been disabled.',
code: 'GTL_DISABLED',
id: '0332fc13-6ab2-4427-ae80-a9fadffd1a6b',
},
},
// Up to 10 calls, then 2 per second
limit: {
duration: 1000,
max: 2,
type: 'bucket',
size: 10,
dripRate: 500,
},
} as const;
@ -39,6 +53,8 @@ export const paramDef = {
limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 },
offset: { type: 'integer', default: 0 },
excludeChannels: { type: 'boolean', default: false },
local: { type: 'boolean', nullable: true, default: null },
expired: { type: 'boolean', default: false },
},
required: [],
} as const;
@ -59,18 +75,54 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private mutingsRepository: MutingsRepository,
private noteEntityService: NoteEntityService,
private readonly queryService: QueryService,
private readonly roleService: RoleService,
) {
super(meta, paramDef, async (ps, me) => {
const query = this.pollsRepository.createQueryBuilder('poll')
.where('poll.userHost IS NULL')
.andWhere('poll.userId != :meId', { meId: me.id })
.andWhere('poll.noteVisibility = \'public\'')
.andWhere(new Brackets(qb => {
.innerJoinAndSelect('poll.note', 'note')
.innerJoinAndSelect('note.user', 'user')
.leftJoinAndSelect('note.renote', 'renote')
.leftJoinAndSelect('note.reply', 'reply')
.leftJoinAndSelect('renote.user', 'renoteUser')
.leftJoinAndSelect('reply.user', 'replyUser')
.andWhere('user.isExplorable = TRUE')
;
if (me) {
query.andWhere('poll.userId != :meId', { meId: me.id });
}
if (ps.expired) {
query.andWhere('poll.expiresAt IS NOT NULL');
query.andWhere('poll.expiresAt <= :expiresMax', {
expiresMax: new Date(),
});
query.andWhere('poll.expiresAt >= :expiresMin', {
expiresMin: new Date(Date.now() - (1000 * 60 * 60 * 24 * 7)),
});
} else {
query.andWhere(new Brackets(qb => {
qb
.where('poll.expiresAt IS NULL')
.orWhere('poll.expiresAt > :now', { now: new Date() });
}));
}
const policies = await this.roleService.getUserPolicies(me?.id ?? null);
if (ps.local != null) {
if (ps.local) {
if (!policies.ltlAvailable) throw new ApiError(meta.errors.ltlDisabled);
query.andWhere('poll.userHost IS NULL');
} else {
if (!policies.gtlAvailable) throw new ApiError(meta.errors.gtlDisabled);
query.andWhere('poll.userHost IS NOT NULL');
}
} else {
if (!policies.gtlAvailable) throw new ApiError(meta.errors.gtlDisabled);
}
/*
//#region exclude arleady voted polls
const votedQuery = this.pollVotesRepository.createQueryBuilder('vote')
.select('vote.noteId')
@ -81,16 +133,15 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
query.setParameters(votedQuery.getParameters());
//#endregion
*/
//#region mute
const mutingQuery = this.mutingsRepository.createQueryBuilder('muting')
.select('muting.muteeId')
.where('muting.muterId = :muterId', { muterId: me.id });
query
.andWhere(`poll.userId NOT IN (${ mutingQuery.getQuery() })`);
query.setParameters(mutingQuery.getParameters());
//#region block/mute/vis
this.queryService.generateVisibilityQuery(query, me);
this.queryService.generateBlockedHostQueryForNote(query);
if (me) {
this.queryService.generateBlockedUserQueryForNotes(query, me);
this.queryService.generateMutedUserQueryForNotes(query, me);
}
//#endregion
//#region exclude channels
@ -107,6 +158,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
if (polls.length === 0) return [];
/*
const notes = await this.notesRepository.find({
where: {
id: In(polls.map(poll => poll.noteId)),
@ -115,6 +167,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
id: 'DESC',
},
});
*/
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const notes = polls.map(poll => poll.note!);
return await this.noteEntityService.packMany(notes, me, {
detail: true,

5
packages/backend/src/server/api/endpoints/notes/search-by-tag.ts
View file

@ -96,10 +96,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
if (!this.serverSettings.enableBotTrending) query.andWhere('user.isBot = FALSE');
this.queryService.generateVisibilityQuery(query, me);
this.queryService.generateBlockedHostQueryForNote(query);
this.queryService.generateBlockedHostQueryForNote(query, undefined, false);
if (me) this.queryService.generateMutedUserQueryForNotes(query, me);
if (me) this.queryService.generateBlockedUserQueryForNotes(query, me);
if (me) this.queryService.generateMutedUserRenotesQueryForNotes(query, me);
const followings = me ? await this.cacheService.userFollowingsCache.fetch(me.id) : {};
@ -160,7 +160,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
if (note.user?.isSuspended) return false;
if (note.userHost) {
if (!this.utilityService.isFederationAllowedHost(note.userHost)) return false;
if (this.utilityService.isSilencedHost(this.serverSettings.silencedHosts, note.userHost)) return false;
}
return true;
});

15
packages/backend/src/server/api/endpoints/notes/translate.ts
View file

@ -20,11 +20,9 @@ import { ApiError } from '../../error.js';
export const meta = {
tags: ['notes'],
// TODO allow unauthenticated if default template allows?
// Maybe a value 'optional' that allows unauthenticated OR a token w/ appropriate role.
// This will allow unauthenticated requests without leaking post data to restricted clients.
requireCredential: true,
requireCredential: 'optional',
kind: 'read:account',
requiredRolePolicy: 'canUseTranslator',
res: {
type: 'object',
@ -88,17 +86,12 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
private readonly loggerService: ApiLoggerService,
) {
super(meta, paramDef, async (ps, me) => {
const policies = await this.roleService.getUserPolicies(me.id);
if (!policies.canUseTranslator) {
throw new ApiError(meta.errors.unavailable);
}
const note = await this.getterService.getNote(ps.noteId).catch(err => {
if (err.id === '9725d0ce-ba28-4dde-95a7-2cbb2c15de24') throw new ApiError(meta.errors.noSuchNote);
throw err;
});
if (!(await this.noteEntityService.isVisibleForMe(note, me.id))) {
if (!(await this.noteEntityService.isVisibleForMe(note, me?.id ?? null))) {
throw new ApiError(meta.errors.cannotTranslateInvisibleNote);
}
@ -140,7 +133,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
if (this.serverSettings.deeplAuthKey) params.append('auth_key', this.serverSettings.deeplAuthKey);
params.append('text', note.text);
params.append('target_lang', targetLang);
const endpoint = deeplFreeInstance ?? this.serverSettings.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate';
const endpoint = deeplFreeInstance ?? ( this.serverSettings.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate' );
const res = await this.httpRequestService.send(endpoint, {
method: 'POST',

2
packages/backend/src/server/api/endpoints/roles/notes.ts
View file

@ -107,10 +107,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser');
this.queryService.generateVisibilityQuery(query, me);
this.queryService.generateBlockedHostQueryForNote(query);
this.queryService.generateMutedUserQueryForNotes(query, me);
this.queryService.generateBlockedUserQueryForNotes(query, me);
this.queryService.generateMutedUserRenotesQueryForNotes(query, me);
const notes = await query.getMany();
notes.sort((a, b) => a.id > b.id ? -1 : 1);

7
packages/backend/src/server/api/endpoints/users/reactions.ts
View file

@ -105,10 +105,15 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
const query = this.queryService.makePaginationQuery(this.noteReactionsRepository.createQueryBuilder('reaction'),
ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate)
.andWhere('reaction.userId = :userId', { userId: ps.userId })
.leftJoinAndSelect('reaction.note', 'note');
.innerJoinAndSelect('reaction.note', 'note');
this.queryService.generateVisibilityQuery(query, me);
this.queryService.generateBlockedHostQueryForNote(query);
if (me) {
this.queryService.generateMutedUserQueryForNotes(query, me);
this.queryService.generateBlockedUserQueryForNotes(query, me);
this.queryService.generateMutedUserRenotesQueryForNotes(query, me);
}
const reactions = (await query
.limit(ps.limit)

6
packages/backend/src/server/api/mastodon/MastodonConverters.ts
View file

@ -252,10 +252,10 @@ export class MastodonConverters {
return await this.convertStatus(status, me);
}
public async convertStatus(status: Entity.Status, me: MiLocalUser | null): Promise<MastodonEntity.Status> {
public async convertStatus(status: Entity.Status, me: MiLocalUser | null, hints?: { note?: MiNote, user?: MiUser }): Promise<MastodonEntity.Status> {
const convertedAccount = this.convertAccount(status.account);
const note = await this.mastodonDataService.requireNote(status.id, me);
const noteUser = await this.getUser(status.account.id);
const note = hints?.note ?? await this.mastodonDataService.requireNote(status.id, me);
const noteUser = hints?.user ?? note.user ?? await this.getUser(status.account.id);
const mentionedRemoteUsers = JSON.parse(note.mentionedRemoteUsers);
const emojis = await this.customEmojiService.populateEmojis(note.emojis, noteUser.host ? noteUser.host : this.config.host);

81
packages/backend/src/server/api/mastodon/MastodonDataService.ts
View file

@ -7,8 +7,8 @@ import { Inject, Injectable } from '@nestjs/common';
import { IsNull } from 'typeorm';
import { DI } from '@/di-symbols.js';
import { QueryService } from '@/core/QueryService.js';
import type { MiNote, NotesRepository } from '@/models/_.js';
import type { MiLocalUser } from '@/models/User.js';
import type { MiChannel, MiNote, NotesRepository } from '@/models/_.js';
import type { MiLocalUser, MiUser } from '@/models/User.js';
import { ApiError } from '../error.js';
/**
@ -27,8 +27,8 @@ export class MastodonDataService {
/**
* Fetches a note in the context of the current user, and throws an exception if not found.
*/
public async requireNote(noteId: string, me?: MiLocalUser | null): Promise<MiNote> {
const note = await this.getNote(noteId, me);
public async requireNote<Rel extends NoteRelations = NoteRelations>(noteId: string, me: MiLocalUser | null | undefined, relations?: Rel): Promise<NoteWithRelations<Rel>> {
const note = await this.getNote(noteId, me, relations);
if (!note) {
throw new ApiError({
@ -46,12 +46,39 @@ export class MastodonDataService {
/**
* Fetches a note in the context of the current user.
*/
public async getNote(noteId: string, me?: MiLocalUser | null): Promise<MiNote | null> {
public async getNote<Rel extends NoteRelations = NoteRelations>(noteId: string, me: MiLocalUser | null | undefined, relations?: Rel): Promise<NoteWithRelations<Rel> | null> {
// Root query: note + required dependencies
const query = this.notesRepository
.createQueryBuilder('note')
.where('note.id = :noteId', { noteId })
.innerJoinAndSelect('note.user', 'user');
.where('note.id = :noteId', { noteId });
// Load relations
if (relations) {
if (relations.reply) {
query.leftJoinAndSelect('note.reply', 'reply');
if (typeof(relations.reply) === 'object') {
if (relations.reply.reply) query.leftJoinAndSelect('note.reply.reply', 'replyReply');
if (relations.reply.renote) query.leftJoinAndSelect('note.reply.renote', 'replyRenote');
if (relations.reply.user) query.innerJoinAndSelect('note.reply.user', 'replyUser');
if (relations.reply.channel) query.leftJoinAndSelect('note.reply.channel', 'replyChannel');
}
}
if (relations.renote) {
query.leftJoinAndSelect('note.renote', 'renote');
if (typeof(relations.renote) === 'object') {
if (relations.renote.reply) query.leftJoinAndSelect('note.renote.reply', 'renoteReply');
if (relations.renote.renote) query.leftJoinAndSelect('note.renote.renote', 'renoteRenote');
if (relations.renote.user) query.innerJoinAndSelect('note.renote.user', 'renoteUser');
if (relations.renote.channel) query.leftJoinAndSelect('note.renote.channel', 'renoteChannel');
}
}
if (relations.user) {
query.innerJoinAndSelect('note.user', 'user');
}
if (relations.channel) {
query.leftJoinAndSelect('note.channel', 'channel');
}
}
// Restrict visibility
this.queryService.generateVisibilityQuery(query, me);
@ -59,7 +86,7 @@ export class MastodonDataService {
this.queryService.generateBlockedUserQueryForNotes(query, me);
}
return await query.getOne();
return await query.getOne() as NoteWithRelations<Rel> | null;
}
/**
@ -82,3 +109,41 @@ export class MastodonDataService {
});
}
}
interface NoteRelations {
reply?: boolean | {
reply?: boolean;
renote?: boolean;
user?: boolean;
channel?: boolean;
};
renote?: boolean | {
reply?: boolean;
renote?: boolean;
user?: boolean;
channel?: boolean;
};
user?: boolean;
channel?: boolean;
}
type NoteWithRelations<Rel extends NoteRelations> = MiNote & {
reply: Rel extends { reply: false }
? null
: null | (MiNote & {
reply: Rel['reply'] extends { reply: true } ? MiNote | null : null;
renote: Rel['reply'] extends { renote: true } ? MiNote | null : null;
user: Rel['reply'] extends { user: true } ? MiUser : null;
channel: Rel['reply'] extends { channel: true } ? MiChannel | null : null;
});
renote: Rel extends { renote: false }
? null
: null | (MiNote & {
reply: Rel['renote'] extends { reply: true } ? MiNote | null : null;
renote: Rel['renote'] extends { renote: true } ? MiNote | null : null;
user: Rel['renote'] extends { user: true } ? MiUser : null;
channel: Rel['renote'] extends { channel: true } ? MiChannel | null : null;
});
user: Rel extends { user: true } ? MiUser : null;
channel: Rel extends { channel: true } ? MiChannel | null : null;
};

24
packages/backend/src/server/api/mastodon/endpoints/status.ts
View file

@ -8,6 +8,10 @@ import { Injectable } from '@nestjs/common';
import { emojiRegexAtStartToEnd } from '@/misc/emoji-regex.js';
import { parseTimelineArgs, TimelineArgs, toBoolean, toInt } from '@/server/api/mastodon/argsUtils.js';
import { MastodonClientService } from '@/server/api/mastodon/MastodonClientService.js';
import { MastodonDataService } from '@/server/api/mastodon/MastodonDataService.js';
import { getNoteSummary } from '@/misc/get-note-summary.js';
import type { Packed } from '@/misc/json-schema.js';
import { isPureRenote } from '@/misc/is-renote.js';
import { convertAttachment, convertPoll, MastodonConverters } from '../MastodonConverters.js';
import type { Entity } from 'megalodon';
import type { FastifyInstance } from 'fastify';
@ -22,6 +26,7 @@ export class ApiStatusMastodon {
constructor(
private readonly mastoConverters: MastodonConverters,
private readonly clientService: MastodonClientService,
private readonly mastodonDataService: MastodonDataService,
) {}
public register(fastify: FastifyInstance): void {
@ -29,13 +34,24 @@ export class ApiStatusMastodon {
if (!_request.params.id) return reply.code(400).send({ error: 'BAD_REQUEST', error_description: 'Missing required parameter "id"' });
const { client, me } = await this.clientService.getAuthClient(_request);
const data = await client.getStatus(_request.params.id);
const response = await this.mastoConverters.convertStatus(data.data, me);
const note = await this.mastodonDataService.requireNote(_request.params.id, me, { user: true, renote: { user: true } });
// Unpack renote for Discord, otherwise the preview breaks
const appearNote = (isPureRenote(note) && _request.headers['user-agent']?.match(/\bDiscordbot\//))
? note.renote as NonNullable<typeof note.renote>
: note;
const data = await client.getStatus(appearNote.id);
const response = await this.mastoConverters.convertStatus(data.data, me, { note: appearNote, user: appearNote.user });
// Fixup - Discord ignores CWs and renders the entire post.
if (response.sensitive && _request.headers['user-agent']?.match(/\bDiscordbot\//)) {
response.content = '(preview disabled for sensitive content)';
response.content = getNoteSummary(data.data satisfies Packed<'Note'>);
response.media_attachments = [];
response.in_reply_to_id = null;
response.in_reply_to_account_id = null;
response.reblog = null;
response.quote = null;
}
return reply.send(response);
@ -170,7 +186,7 @@ export class ApiStatusMastodon {
const data = await client.deleteEmojiReaction(id, react);
return reply.send(data.data);
}
if (!body.media_ids) body.media_ids = undefined;
body.media_ids ??= undefined;
if (body.media_ids && !body.media_ids.length) body.media_ids = undefined;
if (body.poll && !body.poll.options) {

3
packages/backend/src/server/api/stream/channel.ts
View file

@ -65,6 +65,9 @@ export default abstract class Channel {
* ミュートとブロックされてるを処理する
*/
protected isNoteMutedOrBlocked(note: Packed<'Note'>): boolean {
// Ignore notes that require sign-in
if (note.user.requireSigninToViewContents && !this.user) return true;
// 流れてきたNoteがインスタンスミュートしたインスタンスが関わる
if (isInstanceMuted(note, new Set<string>(this.userProfile?.mutedInstances ?? [])) && !this.following[note.userId]) return true;

11
packages/backend/src/server/api/stream/channels/bubble-timeline.ts
View file

@ -12,6 +12,7 @@ import { RoleService } from '@/core/RoleService.js';
import type { MiMeta } from '@/models/Meta.js';
import { isRenotePacked, isQuotePacked } from '@/misc/is-renote.js';
import type { JsonObject } from '@/misc/json-value.js';
import { UtilityService } from '@/core/UtilityService.js';
import Channel, { MiChannelService } from '../channel.js';
class BubbleTimelineChannel extends Channel {
@ -26,6 +27,7 @@ class BubbleTimelineChannel extends Channel {
constructor(
private metaService: MetaService,
private roleService: RoleService,
private readonly utilityService: UtilityService,
noteEntityService: NoteEntityService,
id: string,
@ -56,12 +58,15 @@ class BubbleTimelineChannel extends Channel {
if (note.visibility !== 'public') return;
if (note.channelId != null) return;
if (note.user.host == null) return;
if (!this.instance.bubbleInstances.includes(note.user.host)) return;
if (!this.utilityService.isBubbledHost(note.user.host)) return;
if (note.user.requireSigninToViewContents && this.user == null) return;
if (isRenotePacked(note) && !isQuotePacked(note) && !this.withRenotes) return;
if (note.user.isSilenced && !this.following[note.userId] && note.userId !== this.user!.id) return;
if (note.user.isSilenced) {
if (!this.user) return;
if (note.userId !== this.user.id && !this.following[note.userId]) return;
}
if (this.isNoteMutedOrBlocked(note)) return;
@ -88,6 +93,7 @@ export class BubbleTimelineChannelService implements MiChannelService<false> {
private metaService: MetaService,
private roleService: RoleService,
private noteEntityService: NoteEntityService,
private readonly utilityService: UtilityService,
) {
}
@ -96,6 +102,7 @@ export class BubbleTimelineChannelService implements MiChannelService<false> {
return new BubbleTimelineChannel(
this.metaService,
this.roleService,
this.utilityService,
this.noteEntityService,
id,
connection,

2
packages/backend/test-federation/tsconfig.json
View file

@ -3,7 +3,7 @@
/* Visit https://aka.ms/tsconfig to read more about this file */
/* Projects */
// "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
"incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
// "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */

1
packages/backend/test-server/tsconfig.json
View file

@ -23,6 +23,7 @@
"emitDecoratorMetadata": true,
"resolveJsonModule": true,
"isolatedModules": true,
"incremental": true,
"rootDir": "../src",
"baseUrl": "./",
"paths": {

1
packages/backend/test/tsconfig.json
View file

@ -23,6 +23,7 @@
"emitDecoratorMetadata": true,
"resolveJsonModule": true,
"isolatedModules": true,
"incremental": true,
"baseUrl": "./",
"paths": {
"@/*": ["../src/*"]

11
packages/backend/test/unit/MetaService.ts
View file

@ -11,6 +11,7 @@ import { GlobalModule } from '@/GlobalModule.js';
import { DI } from '@/di-symbols.js';
import { MetaService } from '@/core/MetaService.js';
import { CoreModule } from '@/core/CoreModule.js';
import { MetasRepository } from '@/models/_.js';
import type { TestingModule } from '@nestjs/testing';
import type { DataSource } from 'typeorm';
@ -39,8 +40,8 @@ describe('MetaService', () => {
});
test('fetch (cache)', async () => {
const db = app.get<DataSource>(DI.db);
const spy = jest.spyOn(db, 'transaction');
const metasRepository = app.get<MetasRepository>(DI.metasRepository);
const spy = jest.spyOn(metasRepository, 'createQueryBuilder');
const result = await metaService.fetch();
@ -49,12 +50,12 @@ describe('MetaService', () => {
});
test('fetch (force)', async () => {
const db = app.get<DataSource>(DI.db);
const spy = jest.spyOn(db, 'transaction');
const metasRepository = app.get<MetasRepository>(DI.metasRepository);
const spy = jest.spyOn(metasRepository, 'createQueryBuilder');
const result = await metaService.fetch(true);
expect(result.id).toBe('x');
expect(spy).toHaveBeenCalledTimes(1);
expect(spy).toHaveBeenCalled();
});
});

3
packages/backend/test/unit/NoteCreateService.ts
View file

@ -57,10 +57,13 @@ describe('NoteCreateService', () => {
channelId: null,
channel: null,
userHost: null,
userInstance: null,
replyUserId: null,
replyUserHost: null,
replyUserInstance: null,
renoteUserId: null,
renoteUserHost: null,
renoteUserInstance: null,
processErrors: [],
};

16
packages/backend/test/unit/RoleService.ts
View file

@ -15,6 +15,7 @@ import type { MockFunctionMetadata } from 'jest-mock';
import { GlobalModule } from '@/GlobalModule.js';
import { RoleService } from '@/core/RoleService.js';
import {
InstancesRepository,
MiMeta,
MiRole,
MiRoleAssignment,
@ -39,6 +40,7 @@ const moduleMocker = new ModuleMocker(global);
describe('RoleService', () => {
let app: TestingModule;
let roleService: RoleService;
let instancesRepository: InstancesRepository;
let usersRepository: UsersRepository;
let rolesRepository: RolesRepository;
let roleAssignmentsRepository: RoleAssignmentsRepository;
@ -47,6 +49,19 @@ describe('RoleService', () => {
let clock: lolex.InstalledClock;
async function createUser(data: Partial<MiUser> = {}) {
if (data.host != null) {
await instancesRepository
.createQueryBuilder('instance')
.insert()
.values({
id: genAidx(Date.now()),
firstRetrievedAt: new Date(),
host: data.host,
})
.orIgnore()
.execute();
}
const un = secureRndstr(16);
const x = await usersRepository.insert({
id: genAidx(Date.now()),
@ -145,6 +160,7 @@ describe('RoleService', () => {
app.enableShutdownHooks();
roleService = app.get<RoleService>(RoleService);
instancesRepository = app.get<InstancesRepository>(DI.instancesRepository);
usersRepository = app.get<UsersRepository>(DI.usersRepository);
rolesRepository = app.get<RolesRepository>(DI.rolesRepository);
roleAssignmentsRepository = app.get<RoleAssignmentsRepository>(DI.roleAssignmentsRepository);

18
packages/backend/test/unit/UserSearchService.ts
View file

@ -7,16 +7,18 @@ import { Test, TestingModule } from '@nestjs/testing';
import { describe, jest, test } from '@jest/globals';
import { In } from 'typeorm';
import { UserSearchService } from '@/core/UserSearchService.js';
import { FollowingsRepository, MiUser, UserProfilesRepository, UsersRepository } from '@/models/_.js';
import { FollowingsRepository, InstancesRepository, MiUser, UserProfilesRepository, UsersRepository } from '@/models/_.js';
import { IdService } from '@/core/IdService.js';
import { GlobalModule } from '@/GlobalModule.js';
import { DI } from '@/di-symbols.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { genAidx } from '@/misc/id/aidx.js';
describe('UserSearchService', () => {
let app: TestingModule;
let service: UserSearchService;
let instancesRepository: InstancesRepository;
let usersRepository: UsersRepository;
let followingsRepository: FollowingsRepository;
let idService: IdService;
@ -35,6 +37,19 @@ describe('UserSearchService', () => {
let bobby: MiUser;
async function createUser(data: Partial<MiUser> = {}) {
if (data.host != null) {
await instancesRepository
.createQueryBuilder('instance')
.insert()
.values({
id: genAidx(Date.now()),
firstRetrievedAt: new Date(),
host: data.host,
})
.orIgnore()
.execute();
}
const user = await usersRepository
.insert({
id: idService.gen(),
@ -104,6 +119,7 @@ describe('UserSearchService', () => {
await app.init();
instancesRepository = app.get<InstancesRepository>(DI.instancesRepository);
usersRepository = app.get(DI.usersRepository);
userProfilesRepository = app.get(DI.userProfilesRepository);
followingsRepository = app.get(DI.followingsRepository);

19
packages/backend/test/unit/activitypub.ts
View file

@ -103,6 +103,25 @@ describe('ActivityPub', () => {
let config: Config;
const metaInitial = {
id: 'x',
name: 'Test Instance',
shortName: 'Test Instance',
description: 'Test Instance',
langs: [] as string[],
pinnedUsers: [] as string[],
hiddenTags: [] as string[],
prohibitedWordsForNameOfUser: [] as string[],
silencedHosts: [] as string[],
mediaSilencedHosts: [] as string[],
policies: {},
serverRules: [] as string[],
bannedEmailDomains: [] as string[],
preservedUsernames: [] as string[],
bubbleInstances: [] as string[],
trustedLinkUrlPatterns: [] as string[],
federation: 'all',
federationHosts: [] as string[],
allowUnsignedFetch: 'always',
cacheRemoteFiles: true,
cacheRemoteSensitiveFiles: true,
enableFanoutTimeline: true,

91
packages/backend/test/unit/misc/diff-arrays.ts Normal file
View file

@ -0,0 +1,91 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { diffArrays, diffArraysSimple } from '@/misc/diff-arrays.js';
describe(diffArrays, () => {
it('should return empty result when both inputs are null', () => {
const result = diffArrays(null, null);
expect(result.added).toHaveLength(0);
expect(result.removed).toHaveLength(0);
});
it('should return empty result when both inputs are empty', () => {
const result = diffArrays([], []);
expect(result.added).toHaveLength(0);
expect(result.removed).toHaveLength(0);
});
it('should remove before when after is empty', () => {
const result = diffArrays([1, 2, 3], []);
expect(result.added).toHaveLength(0);
expect(result.removed).toEqual([1, 2, 3]);
});
it('should deduplicate before when after is empty', () => {
const result = diffArrays([1, 1, 2, 2, 3], []);
expect(result.added).toHaveLength(0);
expect(result.removed).toEqual([1, 2, 3]);
});
it('should add after when before is empty', () => {
const result = diffArrays([], [1, 2, 3]);
expect(result.added).toEqual([1, 2, 3]);
expect(result.removed).toHaveLength(0);
});
it('should deduplicate after when before is empty', () => {
const result = diffArrays([], [1, 1, 2, 2, 3]);
expect(result.added).toEqual([1, 2, 3]);
expect(result.removed).toHaveLength(0);
});
it('should return diff when both have values', () => {
const result = diffArrays(
['a', 'b', 'c', 'd'],
['a', 'c', 'e', 'f'],
);
expect(result.added).toEqual(['e', 'f']);
expect(result.removed).toEqual(['b', 'd']);
});
});
describe(diffArraysSimple, () => {
it('should return false when both inputs are null', () => {
const result = diffArraysSimple(null, null);
expect(result).toBe(false);
});
it('should return false when both inputs are empty', () => {
const result = diffArraysSimple([], []);
expect(result).toBe(false);
});
it('should return true when before is populated and after is empty', () => {
const result = diffArraysSimple([1, 2, 3], []);
expect(result).toBe(true);
});
it('should return true when before is empty and after is populated', () => {
const result = diffArraysSimple([], [1, 2, 3]);
expect(result).toBe(true);
});
it('should return true when values have changed', () => {
const result = diffArraysSimple(
['a', 'a', 'b', 'c'],
['a', 'b', 'c', 'd'],
);
expect(result).toBe(true);
});
it('should return false when values have not changed', () => {
const result = diffArraysSimple(
['a', 'a', 'b', 'c'],
['a', 'b', 'c', 'c'],
);
expect(result).toBe(false);
});
});

3
packages/backend/test/unit/misc/is-renote.ts
View file

@ -40,10 +40,13 @@ const base: MiNote = {
channelId: null,
channel: null,
userHost: null,
userInstance: null,
replyUserId: null,
replyUserHost: null,
replyUserInstance: null,
renoteUserId: null,
renoteUserHost: null,
renoteUserInstance: null,
processErrors: [],
};

1
packages/backend/tsconfig.json
View file

@ -23,6 +23,7 @@
"emitDecoratorMetadata": true,
"resolveJsonModule": true,
"isolatedModules": true,
"incremental": true,
"rootDir": "./src",
"baseUrl": "./",
"paths": {

6
packages/frontend-embed/package.json
View file

@ -18,7 +18,7 @@
"@transfem-org/sfm-js": "0.24.5",
"@twemoji/parser": "15.1.1",
"@vitejs/plugin-vue": "5.2.3",
"@vue/compiler-sfc": "3.5.12",
"@vue/compiler-sfc": "3.5.14",
"astring": "1.9.0",
"buraha": "0.0.1",
"estree-walker": "3.0.3",
@ -35,7 +35,7 @@
"typescript": "5.8.3",
"uuid": "11.1.0",
"vite": "6.3.3",
"vue": "3.5.12"
"vue": "3.5.14"
},
"devDependencies": {
"@misskey-dev/summaly": "5.2.1",
@ -49,7 +49,7 @@
"@typescript-eslint/eslint-plugin": "8.31.0",
"@typescript-eslint/parser": "8.31.0",
"@vitest/coverage-v8": "3.1.2",
"@vue/runtime-core": "3.5.12",
"@vue/runtime-core": "3.5.14",
"acorn": "8.14.1",
"cross-env": "7.0.3",
"eslint-plugin-import": "2.31.0",

2
packages/frontend-embed/src/style.scss
View file

@ -101,7 +101,7 @@ rt {
}
}
.ti {
.ti, ph-lg {
width: 1.28em;
vertical-align: -12%;
line-height: 1em;

1
packages/frontend-embed/src/workers/tsconfig.json
View file

@ -1,5 +1,6 @@
{
"compilerOptions": {
"lib": ["esnext", "webworker"],
"incremental": true
}
}

1
packages/frontend-embed/tsconfig.json
View file

@ -23,6 +23,7 @@
"useDefineForClassFields": true,
"verbatimModuleSyntax": true,
"skipLibCheck": true,
"incremental": true,
"baseUrl": ".",
"paths": {
"@/*": ["./src/*"],

1
packages/frontend-shared/tsconfig.json
View file

@ -18,6 +18,7 @@
"esModuleInterop": true,
"verbatimModuleSyntax": true,
"skipLibCheck": true,
"incremental": true,
"baseUrl": ".",
"paths": {
"@/*": ["./*"],

1
packages/frontend/.storybook/tsconfig.json
View file

@ -18,6 +18,7 @@
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"incremental": true,
"jsx": "react",
"jsxFactory": "h"
},

Some files were not shown because too many files have changed in this diff Show more