mirrors/barkey
1
0
Fork 0
mirror of https://codeberg.org/yeentown/barkey.git synced 2025-07-06 20:16:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

merge: Persisted instance blocks (!1068)

Browse source 
View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/1068

Approved-by: dakkar <dakkar@thenautilus.net>
Approved-by: Marie <github@yuugi.dev>
This commit is contained in:
Hazelnoot 2025-06-01 17:25:52 +00:00
commit e1504cfb88
56 changed files with 854 additions and 157 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.gitignore vendored
View file

@ -87,3 +87,7 @@ vite.config.local-dev.ts.timestamp-*
# Sharkey
/packages/megalodon/lib
# TypeScript
.tsbuildinfo
*.tsbuildinfo

3
cypress/tsconfig.json
View file

@ -2,7 +2,8 @@
"compilerOptions": {
"lib": ["dom", "es5"],
"target": "es5",
"types": ["cypress", "node"]
"types": ["cypress", "node"],
"incremental": true
},
"include": ["./**/*.ts"]
}

16
packages/backend/migration/1748104955717-index_IDX_instance_host_key.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class IndexIDXInstanceHostKey1748104955717 {
name = 'IndexIDXInstanceHostKey1748104955717'
async up(queryRunner) {
await queryRunner.query(`CREATE UNIQUE INDEX "IDX_instance_host_key" ON "instance" (((lower(reverse("host")) || '.')::text) text_pattern_ops)`);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "IDX_instance_host_key"`);
}
}

85
packages/backend/migration/1748105111513-add_instance_block_columns.js Normal file
View file

@ -0,0 +1,85 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
* @typedef {{ blockedHosts: string[], silencedHosts: string[], mediaSilencedHosts: string[], federationHosts: string[], bubbleInstances: string[] }} Meta
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AddInstanceBlockColumns1748105111513 {
name = 'AddInstanceBlockColumns1748105111513'
async up(queryRunner) {
// Schema migration
await queryRunner.query(`ALTER TABLE "instance" ADD "isBlocked" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isBlocked" IS 'True if this instance is blocked from federation.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isAllowListed" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isAllowListed" IS 'True if this instance is allow-listed.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isBubbled" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isBubbled" IS 'True if this instance is part of the local bubble.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isSilenced" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isSilenced" IS 'True if this instance is silenced.'`);
await queryRunner.query(`ALTER TABLE "instance" ADD "isMediaSilenced" boolean NOT NULL DEFAULT false`);
await queryRunner.query(`COMMENT ON COLUMN "instance"."isMediaSilenced" IS 'True if this instance is media-silenced.'`);
// Data migration
/** @type {Meta[]} */
const metas = await queryRunner.query(`SELECT "blockedHosts", "silencedHosts", "mediaSilencedHosts", "federationHosts", "bubbleInstances" FROM "meta"`);
if (metas.length > 0) {
/** @type {Meta} */
const meta = metas[0];
// Blocked hosts
if (meta.blockedHosts.length > 0) {
const patterns = buildPatterns(meta.blockedHosts);
await queryRunner.query(`UPDATE "instance" SET "isBlocked" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Silenced hosts
if (meta.silencedHosts.length > 0) {
const patterns = buildPatterns(meta.silencedHosts);
await queryRunner.query(`UPDATE "instance" SET "isSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Media silenced hosts
if (meta.mediaSilencedHosts.length > 0) {
const patterns = buildPatterns(meta.mediaSilencedHosts);
await queryRunner.query(`UPDATE "instance" SET "isMediaSilenced" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Allow-listed hosts
if (meta.federationHosts.length > 0) {
const patterns = buildPatterns(meta.federationHosts);
await queryRunner.query(`UPDATE "instance" SET "isAllowListed" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
// Bubbled hosts
if (meta.bubbleInstances.length > 0) {
const patterns = buildPatterns(meta.bubbleInstances);
await queryRunner.query(`UPDATE "instance" SET "isBubbled" = true WHERE ((lower(reverse("host")) || '.')::text) LIKE ANY ($1)`, [ patterns ]);
}
}
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isMediaSilenced"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isSilenced"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isBubbled"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isAllowListed"`);
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "isBlocked"`);
}
}
/**
* @param {string[]} input
* @returns {string[]}
*/
function buildPatterns(input) {
return input.map(i => i.toLowerCase().split('').reverse().join('') + '.%');
}

44
packages/backend/migration/1748128176881-add_instance_foreign_keys.js Normal file
View file

@ -0,0 +1,44 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AddInstanceForeignKeys1748128176881 {
name = 'AddInstanceForeignKeys1748128176881'
async up(queryRunner) {
// Fix-up: Some older instances have users without a matching instance entry
await queryRunner.query(`
INSERT INTO "instance" ("id", "host", "firstRetrievedAt")
SELECT
MIN("id"),
"host",
COALESCE(MIN("lastFetchedAt"), CURRENT_TIMESTAMP)
FROM "user"
WHERE
"host" IS NOT NULL AND
NOT EXISTS (select 1 from "instance" where "instance"."host" = "user"."host")
GROUP BY "host"
`);
await queryRunner.query(`ALTER TABLE "user" ADD CONSTRAINT "FK_user_host" FOREIGN KEY ("host") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_userHost" FOREIGN KEY ("userHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_replyUserHost" FOREIGN KEY ("replyUserHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "note" ADD CONSTRAINT "FK_note_renoteUserHost" FOREIGN KEY ("renoteUserHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_renoteUserHost"`);
await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_replyUserHost"`);
await queryRunner.query(`ALTER TABLE "note" DROP CONSTRAINT "FK_note_userHost"`);
await queryRunner.query(`ALTER TABLE "user" DROP CONSTRAINT "FK_user_host"`);
}
}

26
packages/backend/migration/1748137683887-add_instance_foreign_keys_to_following.js Normal file
View file

@ -0,0 +1,26 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AddInstanceForeignKeysToFollowing1748137683887 {
name = 'AddInstanceForeignKeysToFollowing1748137683887'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "following" ADD CONSTRAINT "FK_following_followerHost" FOREIGN KEY ("followerHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
await queryRunner.query(`ALTER TABLE "following" ADD CONSTRAINT "FK_following_followeeHost" FOREIGN KEY ("followeeHost") REFERENCES "instance"("host") ON DELETE CASCADE ON UPDATE NO ACTION`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "following" DROP CONSTRAINT "FK_following_followeeHost"`);
await queryRunner.query(`ALTER TABLE "following" DROP CONSTRAINT "FK_following_followerHost"`);
}
}

25
packages/backend/migration/1748191631151-analyze_instance-user-note-following.js Normal file
View file

@ -0,0 +1,25 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
/**
* @typedef {import('typeorm').MigrationInterface} MigrationInterface
*/
/**
* @class
* @implements {MigrationInterface}
*/
export class AnalyzeInstanceUserNoteFollowing1748191631151 {
name = 'AnalyzeInstanceUserNoteFollowing1748191631151'
async up(queryRunner) {
// Refresh statistics for tables impacted by new indexes.
// This helps the query planner to efficiently use them without waiting for the next full vacuum.
await queryRunner.query(`ANALYZE "instance", "user", "following", "note"`);
}
async down(queryRunner) {
}
}

3
packages/backend/package.json
View file

@ -10,6 +10,9 @@
"start": "node ./built/boot/entry.js",
"start:test": "cross-env NODE_ENV=test node ./built/boot/entry.js",
"migrate": "pnpm typeorm migration:run -d ormconfig.js",
"migrate:revert": "pnpm typeorm migration:revert -d ormconfig.js",
"migrate:generate": "pnpm typeorm migration:generate -d ormconfig.js",
"migrate:create": "pnpm typeorm migration:create",
"revert": "pnpm typeorm migration:revert -d ormconfig.js",
"check:connect": "node ./scripts/check_connect.js",
"build": "swc src -d built -D --strip-leading-paths",

11
packages/backend/src/core/FanoutTimelineEndpointService.ts
View file

@ -136,10 +136,10 @@ export class FanoutTimelineEndpointService {
const parentFilter = filter;
filter = (note) => {
if (!ps.ignoreAuthorFromInstanceBlock) {
if (this.utilityService.isBlockedHost(this.meta.blockedHosts, note.userHost)) return false;
if (note.userInstance?.isBlocked) return false;
}
if (note.userId !== note.renoteUserId && this.utilityService.isBlockedHost(this.meta.blockedHosts, note.renoteUserHost)) return false;
if (note.userId !== note.replyUserId && this.utilityService.isBlockedHost(this.meta.blockedHosts, note.replyUserHost)) return false;
if (note.userId !== note.renoteUserId && note.renoteUserInstance?.isBlocked) return false;
if (note.userId !== note.replyUserId && note.replyUserInstance?.isBlocked) return false;
return parentFilter(note);
};
@ -194,7 +194,10 @@ export class FanoutTimelineEndpointService {
.leftJoinAndSelect('note.renote', 'renote')
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser')
.leftJoinAndSelect('note.channel', 'channel');
.leftJoinAndSelect('note.channel', 'channel')
.leftJoinAndSelect('note.userInstance', 'userInstance')
.leftJoinAndSelect('note.replyUserInstance', 'replyUserInstance')
.leftJoinAndSelect('note.renoteUserInstance', 'renoteUserInstance');
const notes = (await query.getMany()).filter(noteFilter);

103
packages/backend/src/core/FederatedInstanceService.ts
View file

@ -5,23 +5,24 @@
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import * as Redis from 'ioredis';
import { QueryFailedError } from 'typeorm';
import type { InstancesRepository } from '@/models/_.js';
import type { InstancesRepository, MiMeta } from '@/models/_.js';
import type { MiInstance } from '@/models/Instance.js';
import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js';
import { MemoryKVCache } from '@/misc/cache.js';
import { IdService } from '@/core/IdService.js';
import { DI } from '@/di-symbols.js';
import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { Serialized } from '@/types.js';
import { diffArrays, diffArraysSimple } from '@/misc/diff-arrays.js';
@Injectable()
export class FederatedInstanceService implements OnApplicationShutdown {
public federatedInstanceCache: RedisKVCache<MiInstance | null>;
private readonly federatedInstanceCache: MemoryKVCache<MiInstance | null>;
constructor(
@Inject(DI.redis)
private redisClient: Redis.Redis,
@Inject(DI.redisForSub)
private redisForSub: Redis.Redis,
@Inject(DI.instancesRepository)
private instancesRepository: InstancesRepository,
@ -29,67 +30,46 @@ export class FederatedInstanceService implements OnApplicationShutdown {
private utilityService: UtilityService,
private idService: IdService,
) {
this.federatedInstanceCache = new RedisKVCache<MiInstance | null>(this.redisClient, 'federatedInstance', {
lifetime: 1000 * 60 * 30, // 30m
memoryCacheLifetime: 1000 * 60 * 3, // 3m
fetcher: (key) => this.instancesRepository.findOneBy({ host: key }),
toRedisConverter: (value) => JSON.stringify(value),
fromRedisConverter: (value) => {
const parsed = JSON.parse(value);
if (parsed == null) return null;
return {
...parsed,
firstRetrievedAt: new Date(parsed.firstRetrievedAt),
latestRequestReceivedAt: parsed.latestRequestReceivedAt ? new Date(parsed.latestRequestReceivedAt) : null,
infoUpdatedAt: parsed.infoUpdatedAt ? new Date(parsed.infoUpdatedAt) : null,
notRespondingSince: parsed.notRespondingSince ? new Date(parsed.notRespondingSince) : null,
};
},
});
this.federatedInstanceCache = new MemoryKVCache(1000 * 60 * 3); // 3m
this.redisForSub.on('message', this.onMessage);
}
@bindThis
public async fetchOrRegister(host: string): Promise<MiInstance> {
host = this.utilityService.toPuny(host);
const cached = await this.federatedInstanceCache.get(host);
const cached = this.federatedInstanceCache.get(host);
if (cached) return cached;
const index = await this.instancesRepository.findOneBy({ host });
let index = await this.instancesRepository.findOneBy({ host });
if (index == null) {
let i;
try {
i = await this.instancesRepository.insertOne({
await this.instancesRepository.createQueryBuilder('instance')
.insert()
.values({
id: this.idService.gen(),
host,
firstRetrievedAt: new Date(),
});
} catch (e: unknown) {
if (e instanceof QueryFailedError) {
if (isDuplicateKeyValueError(e)) {
i = await this.instancesRepository.findOneBy({ host });
}
}
isBlocked: this.utilityService.isBlockedHost(host),
isSilenced: this.utilityService.isSilencedHost(host),
isMediaSilenced: this.utilityService.isMediaSilencedHost(host),
isAllowListed: this.utilityService.isAllowListedHost(host),
isBubbled: this.utilityService.isBubbledHost(host),
})
.orIgnore()
.execute();
if (i == null) {
throw e;
}
}
this.federatedInstanceCache.set(host, i);
return i;
} else {
this.federatedInstanceCache.set(host, index);
return index;
index = await this.instancesRepository.findOneByOrFail({ host });
}
this.federatedInstanceCache.set(host, index);
return index;
}
@bindThis
public async fetch(host: string): Promise<MiInstance | null> {
host = this.utilityService.toPuny(host);
const cached = await this.federatedInstanceCache.get(host);
const cached = this.federatedInstanceCache.get(host);
if (cached !== undefined) return cached;
const index = await this.instancesRepository.findOneBy({ host });
@ -117,8 +97,35 @@ export class FederatedInstanceService implements OnApplicationShutdown {
this.federatedInstanceCache.set(result.host, result);
}
private syncCache(before: Serialized<MiMeta | undefined>, after: Serialized<MiMeta>): void {
const changed =
diffArraysSimple(before?.blockedHosts, after.blockedHosts) ||
diffArraysSimple(before?.silencedHosts, after.silencedHosts) ||
diffArraysSimple(before?.mediaSilencedHosts, after.mediaSilencedHosts) ||
diffArraysSimple(before?.federationHosts, after.federationHosts) ||
diffArraysSimple(before?.bubbleInstances, after.bubbleInstances);
if (changed) {
// We have to clear the whole thing, otherwise subdomains won't be synced.
this.federatedInstanceCache.clear();
}
}
@bindThis
private async onMessage(_: string, data: string): Promise<void> {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
const { type, body } = obj.message as GlobalEvents['internal']['payload'];
if (type === 'metaUpdated') {
this.syncCache(body.before, body.after);
}
}
}
@bindThis
public dispose(): void {
this.redisForSub.off('message', this.onMessage);
this.federatedInstanceCache.dispose();
}

113
packages/backend/src/core/MetaService.ts
View file

@ -4,7 +4,7 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { DataSource } from 'typeorm';
import { DataSource, EntityManager } from 'typeorm';
import * as Redis from 'ioredis';
import { DI } from '@/di-symbols.js';
import { MiMeta } from '@/models/Meta.js';
@ -12,6 +12,9 @@ import { GlobalEventService } from '@/core/GlobalEventService.js';
import { bindThis } from '@/decorators.js';
import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { FeaturedService } from '@/core/FeaturedService.js';
import { MiInstance } from '@/models/Instance.js';
import { diffArrays } from '@/misc/diff-arrays.js';
import type { MetasRepository } from '@/models/_.js';
import type { OnApplicationShutdown } from '@nestjs/common';
@Injectable()
@ -26,6 +29,9 @@ export class MetaService implements OnApplicationShutdown {
@Inject(DI.db)
private db: DataSource,
@Inject(DI.metasRepository)
private readonly metasRepository: MetasRepository,
private featuredService: FeaturedService,
private globalEventService: GlobalEventService,
) {
@ -67,35 +73,35 @@ export class MetaService implements OnApplicationShutdown {
public async fetch(noCache = false): Promise<MiMeta> {
if (!noCache && this.cache) return this.cache;
return await this.db.transaction(async transactionalEntityManager => {
// 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する
const metas = await transactionalEntityManager.find(MiMeta, {
order: {
// 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する
let meta = await this.metasRepository.createQueryBuilder('meta')
.select()
.orderBy({
id: 'DESC',
})
.limit(1)
.getOne();
if (!meta) {
await this.metasRepository.createQueryBuilder('meta')
.insert()
.values({
id: 'x',
})
.orIgnore()
.execute();
meta = await this.metasRepository.createQueryBuilder('meta')
.select()
.orderBy({
id: 'DESC',
},
});
})
.limit(1)
.getOneOrFail();
}
const meta = metas[0];
if (meta) {
this.cache = meta;
return meta;
} else {
// metaが空のときfetchMetaが同時に呼ばれるとここが同時に呼ばれてしまうことがあるのでフェイルセーフなupsertを使う
const saved = await transactionalEntityManager
.upsert(
MiMeta,
{
id: 'x',
},
['id'],
)
.then((x) => transactionalEntityManager.findOneByOrFail(MiMeta, x.identifiers[0]));
this.cache = saved;
return saved;
}
});
this.cache = meta;
return meta;
}
@bindThis
@ -103,7 +109,7 @@ export class MetaService implements OnApplicationShutdown {
let before: MiMeta | undefined;
const updated = await this.db.transaction(async transactionalEntityManager => {
const metas = await transactionalEntityManager.find(MiMeta, {
const metas: (MiMeta | undefined)[] = await transactionalEntityManager.find(MiMeta, {
order: {
id: 'DESC',
},
@ -126,6 +132,10 @@ export class MetaService implements OnApplicationShutdown {
},
});
// Propagate changes to blockedHosts, silencedHosts, mediaSilencedHosts, federationInstances, and bubbleInstances to the relevant instance rows
// Do this inside the transaction to avoid potential race condition (when an instance gets registered while we're updating).
await this.persistBlocks(transactionalEntityManager, before ?? {}, afters[0]);
return afters[0];
});
@ -159,4 +169,49 @@ export class MetaService implements OnApplicationShutdown {
public onApplicationShutdown(signal?: string | undefined): void {
this.dispose();
}
private async persistBlocks(tem: EntityManager, before: Partial<MiMeta>, after: Partial<MiMeta>): Promise<void> {
await this.persistBlock(tem, before.blockedHosts, after.blockedHosts, 'isBlocked');
await this.persistBlock(tem, before.silencedHosts, after.silencedHosts, 'isSilenced');
await this.persistBlock(tem, before.mediaSilencedHosts, after.mediaSilencedHosts, 'isMediaSilenced');
await this.persistBlock(tem, before.federationHosts, after.federationHosts, 'isAllowListed');
await this.persistBlock(tem, before.bubbleInstances, after.bubbleInstances, 'isBubbled');
}
private async persistBlock(tem: EntityManager, before: string[] | undefined, after: string[] | undefined, field: keyof MiInstance): Promise<void> {
const { added, removed } = diffArrays(before, after);
if (removed.length > 0) {
await this.updateInstancesByHost(tem, field, false, removed);
}
if (added.length > 0) {
await this.updateInstancesByHost(tem, field, true, added);
}
}
private async updateInstancesByHost(tem: EntityManager, field: keyof MiInstance, value: boolean, hosts: string[]): Promise<void> {
// Use non-array queries when possible, as they are indexed and can be much faster.
if (hosts.length === 1) {
const pattern = genHostPattern(hosts[0]);
await tem
.createQueryBuilder(MiInstance, 'instance')
.update()
.set({ [field]: value })
.where('(lower(reverse("host")) || \'.\') LIKE :pattern', { pattern })
.execute();
} else if (hosts.length > 1) {
const patterns = hosts.map(host => genHostPattern(host));
await tem
.createQueryBuilder(MiInstance, 'instance')
.update()
.set({ [field]: value })
.where('(lower(reverse("host")) || \'.\') LIKE ANY (:patterns)', { patterns })
.execute();
}
}
}
function genHostPattern(host: string): string {
return host.toLowerCase().split('').reverse().join('') + '.%';
}

57
packages/backend/src/core/QueryService.ts
View file

@ -243,47 +243,40 @@ export class QueryService {
q.andWhere(new Brackets(qb => {
qb
.where(new Brackets(qb => {
qb.where('note.renoteId IS NOT NULL');
qb.andWhere('note.text IS NULL');
qb.andWhere(`note.userId NOT IN (${ mutingQuery.getQuery() })`);
}))
.orWhere('note.renoteId IS NULL')
.orWhere('note.text IS NOT NULL');
.orWhere('note.text IS NOT NULL')
.orWhere('note.cw IS NOT NULL')
.orWhere('note.replyId IS NOT NULL')
.orWhere('note.fileIds != \'{}\'')
.orWhere(`note.userId NOT IN (${ mutingQuery.getQuery() })`);
}));
q.setParameters(mutingQuery.getParameters());
}
@bindThis
public generateBlockedHostQueryForNote(q: SelectQueryBuilder<any>, excludeAuthor?: boolean): void {
let nonBlockedHostQuery: (part: string) => string;
if (this.meta.blockedHosts.length === 0) {
nonBlockedHostQuery = () => '1=1';
} else {
nonBlockedHostQuery = (match: string) => `('.' || ${match}) NOT ILIKE ALL(select '%.' || x from (select unnest("blockedHosts") as x from "meta") t)`;
public generateBlockedHostQueryForNote(q: SelectQueryBuilder<any>, excludeAuthor?: boolean, allowSilenced = true): void {
function checkFor(key: 'user' | 'replyUser' | 'renoteUser') {
q.leftJoin(`note.${key}Instance`, `${key}Instance`);
q.andWhere(new Brackets(qb => {
qb.orWhere(`note.${key}Id IS NULL`) // no corresponding user
.orWhere(`note.${key}Host IS NULL`) // local
.orWhere(`${key}Instance.isBlocked = false`); // not blocked
if (!allowSilenced) {
qb.orWhere(`${key}Instance.isSilenced = false`); // not silenced
}
if (excludeAuthor) {
qb.orWhere(`note.userId = note.${key}Id`); // author
}
}));
}
if (excludeAuthor) {
const instanceSuspension = (user: string) => new Brackets(qb => qb
.where(`note.${user}Id IS NULL`) // no corresponding user
.orWhere(`note.userId = note.${user}Id`)
.orWhere(`note.${user}Host IS NULL`) // local
.orWhere(nonBlockedHostQuery(`note.${user}Host`)));
q
.andWhere(instanceSuspension('replyUser'))
.andWhere(instanceSuspension('renoteUser'));
} else {
const instanceSuspension = (user: string) => new Brackets(qb => qb
.where(`note.${user}Id IS NULL`) // no corresponding user
.orWhere(`note.${user}Host IS NULL`) // local
.orWhere(nonBlockedHostQuery(`note.${user}Host`)));
q
.andWhere(instanceSuspension('user'))
.andWhere(instanceSuspension('replyUser'))
.andWhere(instanceSuspension('renoteUser'));
if (!excludeAuthor) {
checkFor('user');
}
checkFor('replyUser');
checkFor('renoteUser');
}
}

2
packages/backend/src/core/ReversiService.ts
View file

@ -587,6 +587,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
lastActiveDate: parsed.user1.lastActiveDate != null ? new Date(parsed.user1.lastActiveDate) : null,
lastFetchedAt: parsed.user1.lastFetchedAt != null ? new Date(parsed.user1.lastFetchedAt) : null,
movedAt: parsed.user1.movedAt != null ? new Date(parsed.user1.movedAt) : null,
instance: null,
} : null,
user2: parsed.user2 != null ? {
...parsed.user2,
@ -597,6 +598,7 @@ export class ReversiService implements OnApplicationShutdown, OnModuleInit {
lastActiveDate: parsed.user2.lastActiveDate != null ? new Date(parsed.user2.lastActiveDate) : null,
lastFetchedAt: parsed.user2.lastFetchedAt != null ? new Date(parsed.user2.lastFetchedAt) : null,
movedAt: parsed.user2.movedAt != null ? new Date(parsed.user2.movedAt) : null,
instance: null,
} : null,
};
} else {

39
packages/backend/src/core/UtilityService.ts
View file

@ -49,22 +49,49 @@ export class UtilityService {
return regexp.test(email);
}
public isBlockedHost(host: string | null): boolean;
public isBlockedHost(blockedHosts: string[], host: string | null): boolean;
@bindThis
public isBlockedHost(blockedHosts: string[], host: string | null): boolean {
public isBlockedHost(blockedHostsOrHost: string[] | string | null, host?: string | null): boolean {
const blockedHosts = Array.isArray(blockedHostsOrHost) ? blockedHostsOrHost : this.meta.blockedHosts;
host = Array.isArray(blockedHostsOrHost) ? host : blockedHostsOrHost;
if (host == null) return false;
return blockedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
public isSilencedHost(host: string | null): boolean;
public isSilencedHost(silencedHosts: string[], host: string | null): boolean;
@bindThis
public isSilencedHost(silencedHosts: string[] | undefined, host: string | null): boolean {
if (!silencedHosts || host == null) return false;
public isSilencedHost(silencedHostsOrHost: string[] | string | null, host?: string | null): boolean {
const silencedHosts = Array.isArray(silencedHostsOrHost) ? silencedHostsOrHost : this.meta.silencedHosts;
host = Array.isArray(silencedHostsOrHost) ? host : silencedHostsOrHost;
if (host == null) return false;
return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
public isMediaSilencedHost(host: string | null): boolean;
public isMediaSilencedHost(silencedHosts: string[], host: string | null): boolean;
@bindThis
public isMediaSilencedHost(silencedHosts: string[] | undefined, host: string | null): boolean {
if (!silencedHosts || host == null) return false;
return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
public isMediaSilencedHost(mediaSilencedHostsOrHost: string[] | string | null, host?: string | null): boolean {
const mediaSilencedHosts = Array.isArray(mediaSilencedHostsOrHost) ? mediaSilencedHostsOrHost : this.meta.mediaSilencedHosts;
host = Array.isArray(mediaSilencedHostsOrHost) ? host : mediaSilencedHostsOrHost;
if (host == null) return false;
return mediaSilencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
@bindThis
public isAllowListedHost(host: string | null): boolean {
if (host == null) return false;
return this.meta.federationHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
@bindThis
public isBubbledHost(host: string | null): boolean {
if (host == null) return false;
return this.meta.bubbleInstances.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`));
}
@bindThis

5
packages/backend/src/core/WebhookTestService.ts
View file

@ -63,6 +63,7 @@ function generateDummyUser(override?: Partial<MiUser>): MiUser {
emojis: [],
score: 0,
host: null,
instance: null,
inbox: null,
sharedInbox: null,
featured: null,
@ -114,10 +115,13 @@ function generateDummyNote(override?: Partial<MiNote>): MiNote {
channelId: null,
channel: null,
userHost: null,
userInstance: null,
replyUserId: null,
replyUserHost: null,
replyUserInstance: null,
renoteUserId: null,
renoteUserHost: null,
renoteUserInstance: null,
updatedAt: null,
processErrors: [],
...override,
@ -449,6 +453,7 @@ export class WebhookTestService {
isAdmin: false,
isModerator: false,
isSystem: false,
instance: undefined,
...override,
};
}

3
packages/backend/src/core/activitypub/models/ApPersonService.ts
View file

@ -398,6 +398,9 @@ export class ApPersonService implements OnModuleInit, OnApplicationShutdown {
const followerscollection = await _resolver.resolveCollection(person.followers!).catch(() => { return null; });
const followingcollection = await _resolver.resolveCollection(person.following!).catch(() => { return null; });
// Register the instance first, to avoid FK errors
await this.federatedInstanceService.fetchOrRegister(host);
try {
// Start transaction
await this.db.transaction(async transactionalEntityManager => {

23
packages/backend/src/core/chart/charts/federation.ts
View file

@ -44,10 +44,6 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
const suspendedInstancesQuery = this.instancesRepository.createQueryBuilder('instance')
.select('instance.host')
.where('instance.suspensionState != \'none\'');
const pubsubSubQuery = this.followingsRepository.createQueryBuilder('f')
.select('f.followerHost')
.where('f.followerHost IS NOT NULL');
@ -64,22 +60,25 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followeeHost)')
.where('following.followeeHost IS NOT NULL')
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followeeHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.innerJoin('following.followeeInstance', 'followeeInstance')
.andWhere('followeeInstance.suspensionState = \'none\'')
.andWhere('followeeInstance.isBlocked = false')
.getRawOne()
.then(x => parseInt(x.count, 10)),
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followerHost)')
.where('following.followerHost IS NOT NULL')
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followerHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere(`following.followerHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.innerJoin('following.followerInstance', 'followerInstance')
.andWhere('followerInstance.isBlocked = false')
.andWhere('followerInstance.suspensionState = \'none\'')
.getRawOne()
.then(x => parseInt(x.count, 10)),
this.followingsRepository.createQueryBuilder('following')
.select('COUNT(DISTINCT following.followeeHost)')
.where('following.followeeHost IS NOT NULL')
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || following.followeeHost) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere(`following.followeeHost NOT IN (${ suspendedInstancesQuery.getQuery() })`)
.innerJoin('following.followeeInstance', 'followeeInstance')
.andWhere('followeeInstance.isBlocked = false')
.andWhere('followeeInstance.suspensionState = \'none\'')
.andWhere(`following.followeeHost IN (${ pubsubSubQuery.getQuery() })`)
.setParameters(pubsubSubQuery.getParameters())
.getRawOne()
@ -87,7 +86,7 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
this.instancesRepository.createQueryBuilder('instance')
.select('COUNT(instance.id)')
.where(`instance.host IN (${ subInstancesQuery.getQuery() })`)
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || instance.host) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere('instance.isBlocked = false')
.andWhere('instance.suspensionState = \'none\'')
.andWhere('instance.isNotResponding = false')
.getRawOne()
@ -95,7 +94,7 @@ export default class FederationChart extends Chart<typeof schema> { // eslint-di
this.instancesRepository.createQueryBuilder('instance')
.select('COUNT(instance.id)')
.where(`instance.host IN (${ pubInstancesQuery.getQuery() })`)
.andWhere(this.meta.blockedHosts.length === 0 ? '1=1' : '(\'.\' || instance.host) NOT ILIKE ALL(select \'%.\' || x from (select unnest("blockedHosts") as x from "meta") t)')
.andWhere('instance.isBlocked = false')
.andWhere('instance.suspensionState = \'none\'')
.andWhere('instance.isNotResponding = false')
.getRawOne()

6
packages/backend/src/core/entities/InstanceEntityService.ts
View file

@ -43,7 +43,7 @@ export class InstanceEntityService {
isNotResponding: instance.isNotResponding,
isSuspended: instance.suspensionState !== 'none',
suspensionState: instance.suspensionState,
isBlocked: this.utilityService.isBlockedHost(this.meta.blockedHosts, instance.host),
isBlocked: instance.isBlocked,
softwareName: instance.softwareName,
softwareVersion: instance.softwareVersion,
openRegistrations: instance.openRegistrations,
@ -51,8 +51,8 @@ export class InstanceEntityService {
description: instance.description,
maintainerName: instance.maintainerName,
maintainerEmail: instance.maintainerEmail,
isSilenced: this.utilityService.isSilencedHost(this.meta.silencedHosts, instance.host),
isMediaSilenced: this.utilityService.isMediaSilencedHost(this.meta.mediaSilencedHosts, instance.host),
isSilenced: instance.isSilenced,
isMediaSilenced: instance.isMediaSilenced,
iconUrl: instance.iconUrl,
faviconUrl: instance.faviconUrl,
themeColor: instance.themeColor,

2
packages/backend/src/core/entities/UserEntityService.ts
View file

@ -609,7 +609,7 @@ export class UserEntityService implements OnModuleInit {
requireSigninToViewContents: user.requireSigninToViewContents === false ? undefined : true,
makeNotesFollowersOnlyBefore: user.makeNotesFollowersOnlyBefore ?? undefined,
makeNotesHiddenBefore: user.makeNotesHiddenBefore ?? undefined,
instance: user.host ? this.federatedInstanceService.federatedInstanceCache.fetch(user.host).then(instance => instance ? {
instance: user.host ? this.federatedInstanceService.fetch(user.host).then(instance => instance ? {
name: instance.name,
softwareName: instance.softwareName,
softwareVersion: instance.softwareVersion,

9
packages/backend/src/misc/cache.ts
View file

@ -308,8 +308,17 @@ export class MemoryKVCache<T> {
}
}
/**
* Removes all entries from the cache, but does not dispose it.
*/
@bindThis
public clear(): void {
this.cache.clear();
}
@bindThis
public dispose(): void {
this.clear();
clearInterval(this.gcIntervalHandle);
}

102
packages/backend/src/misc/diff-arrays.ts Normal file
View file

@ -0,0 +1,102 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export interface DiffResult<T> {
added: T[];
removed: T[];
}
/**
* Calculates the difference between two snapshots of data.
* Null, undefined, and empty arrays are supported, and duplicate values are ignored.
* Result sets are de-duplicated, and will be empty if no data was added or removed (respectively).
* The inputs are treated as un-ordered, so a re-ordering of the same data will NOT be considered a change.
* @param dataBefore Array containing data before the change
* @param dataAfter Array containing data after the change
*/
export function diffArrays<T>(dataBefore: T[] | null | undefined, dataAfter: T[] | null | undefined): DiffResult<T> {
const before = dataBefore ? new Set(dataBefore) : null;
const after = dataAfter ? new Set(dataAfter) : null;
// data before AND after => changed
if (before?.size && after?.size) {
const added: T[] = [];
const removed: T[] = [];
for (const host of before) {
// before and NOT after => removed
// delete operation removes duplicates to speed up the "after" loop
if (!after.delete(host)) {
removed.push(host);
}
}
for (const host of after) {
// after and NOT before => added
if (!before.has(host)) {
added.push(host);
}
}
return { added, removed };
}
// data ONLY before => all removed
if (before?.size) {
return { added: [], removed: Array.from(before) };
}
// data ONLY after => all added
if (after?.size) {
return { added: Array.from(after), removed: [] };
}
// data NEITHER before nor after => no change
return { added: [], removed: [] };
}
/**
* Checks for any difference between two snapshots of data.
* Null, undefined, and empty arrays are supported, and duplicate values are ignored.
* The inputs are treated as un-ordered, so a re-ordering of the same data will NOT be considered a change.
* @param dataBefore Array containing data before the change
* @param dataAfter Array containing data after the change
*/
export function diffArraysSimple<T>(dataBefore: T[] | null | undefined, dataAfter: T[] | null | undefined): boolean {
const before = dataBefore ? new Set(dataBefore) : null;
const after = dataAfter ? new Set(dataAfter) : null;
if (before?.size && after?.size) {
// different size => changed
if (before.size !== after.size) return true;
// removed => changed
for (const host of before) {
// delete operation removes duplicates to speed up the "after" loop
if (!after.delete(host)) {
return true;
}
}
// added => changed
for (const host of after) {
if (!before.has(host)) {
return true;
}
}
// identical values => no change
return false;
}
// before and NOT after => change
if (before?.size) return true;
// after and NOT before => change
if (after?.size) return true;
// NEITHER before nor after => no change
return false;
}

21
packages/backend/src/models/Following.ts
View file

@ -4,6 +4,7 @@
*/
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm';
import { MiInstance } from '@/models/Instance.js';
import { id } from './util/id.js';
import { MiUser } from './User.js';
@ -66,6 +67,16 @@ export class MiFollowing {
})
public followerHost: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'followerHost',
foreignKeyConstraintName: 'FK_following_followerHost',
referencedColumnName: 'host',
})
public followerInstance: MiInstance | null;
@Column('varchar', {
length: 512, nullable: true,
comment: '[Denormalized]',
@ -85,6 +96,16 @@ export class MiFollowing {
})
public followeeHost: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'followeeHost',
foreignKeyConstraintName: 'FK_following_followeeHost',
referencedColumnName: 'host',
})
public followeeInstance: MiInstance | null;
@Column('varchar', {
length: 512, nullable: true,
comment: '[Denormalized]',

51
packages/backend/src/models/Instance.ts
View file

@ -6,6 +6,7 @@
import { Entity, PrimaryColumn, Index, Column } from 'typeorm';
import { id } from './util/id.js';
@Index('IDX_instance_host_key', { synchronize: false })
@Entity('instance')
export class MiInstance {
@PrimaryColumn(id())
@ -98,6 +99,56 @@ export class MiInstance {
})
public suspensionState: 'none' | 'manuallySuspended' | 'goneSuspended' | 'autoSuspendedForNotResponding';
/**
* True if this instance is blocked from federation.
*/
@Column('boolean', {
nullable: false,
default: false,
comment: 'True if this instance is blocked from federation.',
})
public isBlocked: boolean;
/**
* True if this instance is allow-listed.
*/
@Column('boolean', {
nullable: false,
default: false,
comment: 'True if this instance is allow-listed.',
})
public isAllowListed: boolean;
/**
* True if this instance is part of the local bubble.
*/
@Column('boolean', {
nullable: false,
default: false,
comment: 'True if this instance is part of the local bubble.',
})
public isBubbled: boolean;
/**
* True if this instance is silenced.
*/
@Column('boolean', {
nullable: false,
default: false,
comment: 'True if this instance is silenced.',
})
public isSilenced: boolean;
/**
* True if this instance is media-silenced.
*/
@Column('boolean', {
nullable: false,
default: false,
comment: 'True if this instance is media-silenced.',
})
public isMediaSilenced: boolean;
@Column('varchar', {
length: 64, nullable: true,
comment: 'The software of the Instance.',

31
packages/backend/src/models/Note.ts
View file

@ -5,6 +5,7 @@
import { Entity, Index, JoinColumn, Column, PrimaryColumn, ManyToOne } from 'typeorm';
import { noteVisibilities } from '@/types.js';
import { MiInstance } from '@/models/Instance.js';
import { id } from './util/id.js';
import { MiUser } from './User.js';
import { MiChannel } from './Channel.js';
@ -222,6 +223,16 @@ export class MiNote {
})
public userHost: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'userHost',
foreignKeyConstraintName: 'FK_note_userHost',
referencedColumnName: 'host',
})
public userInstance: MiInstance | null;
@Column({
...id(),
nullable: true,
@ -235,6 +246,16 @@ export class MiNote {
})
public replyUserHost: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'replyUserHost',
foreignKeyConstraintName: 'FK_note_replyUserHost',
referencedColumnName: 'host',
})
public replyUserInstance: MiInstance | null;
@Column({
...id(),
nullable: true,
@ -247,6 +268,16 @@ export class MiNote {
comment: '[Denormalized]',
})
public renoteUserHost: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'renoteUserHost',
foreignKeyConstraintName: 'FK_note_renoteUserHost',
referencedColumnName: 'host',
})
public renoteUserInstance: MiInstance | null;
//#endregion
constructor(data: Partial<MiNote>) {

13
packages/backend/src/models/User.ts
View file

@ -3,8 +3,9 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Entity, Column, Index, OneToOne, JoinColumn, PrimaryColumn } from 'typeorm';
import { Entity, Column, Index, OneToOne, JoinColumn, PrimaryColumn, ManyToOne } from 'typeorm';
import { type UserUnsignedFetchOption, userUnsignedFetchOptions } from '@/const.js';
import { MiInstance } from '@/models/Instance.js';
import { id } from './util/id.js';
import { MiDriveFile } from './DriveFile.js';
@ -292,6 +293,16 @@ export class MiUser {
})
public host: string | null;
@ManyToOne(() => MiInstance, {
onDelete: 'CASCADE',
})
@JoinColumn({
name: 'host',
foreignKeyConstraintName: 'FK_user_host',
referencedColumnName: 'host',
})
public instance: MiInstance | null;
@Column('varchar', {
length: 512, nullable: true,
comment: 'The inbox URL of the User. It will be null if the origin of the user is local.',

1
packages/backend/src/server/api/endpoints/antennas/notes.ts
View file

@ -121,6 +121,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
this.queryService.generateVisibilityQuery(query, me);
this.queryService.generateMutedUserQueryForNotes(query, me);
this.queryService.generateBlockedUserQueryForNotes(query, me);
this.queryService.generateMutedUserRenotesQueryForNotes(query, me);
const notes = await query.getMany();
if (sinceId != null && untilId == null) {

2
packages/backend/src/server/api/endpoints/channels/timeline.ts
View file

@ -138,9 +138,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
.leftJoinAndSelect('note.channel', 'channel');
this.queryService.generateBlockedHostQueryForNote(query);
this.queryService.generateVisibilityQuery(query, me);
if (me) {
this.queryService.generateMutedUserQueryForNotes(query, me);
this.queryService.generateBlockedUserQueryForNotes(query, me);
this.queryService.generateMutedUserRenotesQueryForNotes(query, me);
}
if (ps.withRenotes === false) {

3
packages/backend/src/server/api/endpoints/clips/notes.ts
View file

@ -92,10 +92,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
.andWhere('clipNote.clipId = :clipId', { clipId: clip.id });
this.queryService.generateBlockedHostQueryForNote(query);
this.queryService.generateVisibilityQuery(query, me);
if (me) {
this.queryService.generateVisibilityQuery(query, me);
this.queryService.generateMutedUserQueryForNotes(query, me);
this.queryService.generateBlockedUserQueryForNotes(query, me);
this.queryService.generateMutedUserRenotesQueryForNotes(query, me);
}
const notes = await query

3
packages/backend/src/server/api/endpoints/notes/bubble-timeline.ts
View file

@ -85,7 +85,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate)
.andWhere('note.visibility = \'public\'')
.andWhere('note.channelId IS NULL')
.andWhere('note.userHost IN (:...hosts)', { hosts: this.serverSettings.bubbleInstances })
.andWhere('note.userHost IS NULL')
.andWhere('userInstance.isBubbled = true') // This comes from generateVisibilityQuery below
.innerJoinAndSelect('note.user', 'user')
.leftJoinAndSelect('note.reply', 'reply')
.leftJoinAndSelect('note.renote', 'renote')

13
packages/backend/src/server/api/endpoints/notes/following.ts
View file

@ -4,7 +4,7 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { ObjectLiteral, SelectQueryBuilder } from 'typeorm';
import { IsNull, ObjectLiteral, SelectQueryBuilder } from 'typeorm';
import { SkLatestNote, MiFollowing } from '@/models/_.js';
import type { NotesRepository } from '@/models/_.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
@ -130,7 +130,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
.leftJoinAndSelect('note.renote', 'renote')
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser')
.leftJoinAndSelect('note.channel', 'channel')
// Exclude channel notes
.andWhere({ channelId: IsNull() })
;
// Limit to files, if requested
@ -145,11 +147,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
// Hide blocked users / instances
query.andWhere('"user"."isSuspended" = false');
query.andWhere('("replyUser" IS NULL OR "replyUser"."isSuspended" = false)');
query.andWhere('("renoteUser" IS NULL OR "renoteUser"."isSuspended" = false)');
this.queryService.generateBlockedHostQueryForNote(query);
// Respect blocks and mutes
// Respect blocks, mutes, and privacy
this.queryService.generateVisibilityQuery(query, me);
this.queryService.generateBlockedUserQueryForNotes(query, me);
this.queryService.generateMutedUserQueryForNotes(query, me);
@ -161,7 +162,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
// Query and return the next page
const notes = await query.getMany();
return await this.noteEntityService.packMany(notes, me);
return await this.noteEntityService.packMany(notes, me, { skipHide: true });
});
}
}

5
packages/backend/src/server/api/endpoints/notes/search-by-tag.ts
View file

@ -96,10 +96,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
if (!this.serverSettings.enableBotTrending) query.andWhere('user.isBot = FALSE');
this.queryService.generateVisibilityQuery(query, me);
this.queryService.generateBlockedHostQueryForNote(query);
this.queryService.generateBlockedHostQueryForNote(query, undefined, false);
if (me) this.queryService.generateMutedUserQueryForNotes(query, me);
if (me) this.queryService.generateBlockedUserQueryForNotes(query, me);
if (me) this.queryService.generateMutedUserRenotesQueryForNotes(query, me);
const followings = me ? await this.cacheService.userFollowingsCache.fetch(me.id) : {};
@ -160,7 +160,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
if (note.user?.isSuspended) return false;
if (note.userHost) {
if (!this.utilityService.isFederationAllowedHost(note.userHost)) return false;
if (this.utilityService.isSilencedHost(this.serverSettings.silencedHosts, note.userHost)) return false;
}
return true;
});

2
packages/backend/src/server/api/endpoints/roles/notes.ts
View file

@ -107,10 +107,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
.leftJoinAndSelect('reply.user', 'replyUser')
.leftJoinAndSelect('renote.user', 'renoteUser');
this.queryService.generateVisibilityQuery(query, me);
this.queryService.generateBlockedHostQueryForNote(query);
this.queryService.generateMutedUserQueryForNotes(query, me);
this.queryService.generateBlockedUserQueryForNotes(query, me);
this.queryService.generateMutedUserRenotesQueryForNotes(query, me);
const notes = await query.getMany();
notes.sort((a, b) => a.id > b.id ? -1 : 1);

7
packages/backend/src/server/api/endpoints/users/reactions.ts
View file

@ -105,10 +105,15 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
const query = this.queryService.makePaginationQuery(this.noteReactionsRepository.createQueryBuilder('reaction'),
ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate)
.andWhere('reaction.userId = :userId', { userId: ps.userId })
.leftJoinAndSelect('reaction.note', 'note');
.innerJoinAndSelect('reaction.note', 'note');
this.queryService.generateVisibilityQuery(query, me);
this.queryService.generateBlockedHostQueryForNote(query);
if (me) {
this.queryService.generateMutedUserQueryForNotes(query, me);
this.queryService.generateBlockedUserQueryForNotes(query, me);
this.queryService.generateMutedUserRenotesQueryForNotes(query, me);
}
const reactions = (await query
.limit(ps.limit)

2
packages/backend/test-federation/tsconfig.json
View file

@ -3,7 +3,7 @@
/* Visit https://aka.ms/tsconfig to read more about this file */
/* Projects */
// "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
"incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
// "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */

1
packages/backend/test-server/tsconfig.json
View file

@ -23,6 +23,7 @@
"emitDecoratorMetadata": true,
"resolveJsonModule": true,
"isolatedModules": true,
"incremental": true,
"rootDir": "../src",
"baseUrl": "./",
"paths": {

1
packages/backend/test/tsconfig.json
View file

@ -23,6 +23,7 @@
"emitDecoratorMetadata": true,
"resolveJsonModule": true,
"isolatedModules": true,
"incremental": true,
"baseUrl": "./",
"paths": {
"@/*": ["../src/*"]

11
packages/backend/test/unit/MetaService.ts
View file

@ -11,6 +11,7 @@ import { GlobalModule } from '@/GlobalModule.js';
import { DI } from '@/di-symbols.js';
import { MetaService } from '@/core/MetaService.js';
import { CoreModule } from '@/core/CoreModule.js';
import { MetasRepository } from '@/models/_.js';
import type { TestingModule } from '@nestjs/testing';
import type { DataSource } from 'typeorm';
@ -39,8 +40,8 @@ describe('MetaService', () => {
});
test('fetch (cache)', async () => {
const db = app.get<DataSource>(DI.db);
const spy = jest.spyOn(db, 'transaction');
const metasRepository = app.get<MetasRepository>(DI.metasRepository);
const spy = jest.spyOn(metasRepository, 'createQueryBuilder');
const result = await metaService.fetch();
@ -49,12 +50,12 @@ describe('MetaService', () => {
});
test('fetch (force)', async () => {
const db = app.get<DataSource>(DI.db);
const spy = jest.spyOn(db, 'transaction');
const metasRepository = app.get<MetasRepository>(DI.metasRepository);
const spy = jest.spyOn(metasRepository, 'createQueryBuilder');
const result = await metaService.fetch(true);
expect(result.id).toBe('x');
expect(spy).toHaveBeenCalledTimes(1);
expect(spy).toHaveBeenCalled();
});
});

3
packages/backend/test/unit/NoteCreateService.ts
View file

@ -57,10 +57,13 @@ describe('NoteCreateService', () => {
channelId: null,
channel: null,
userHost: null,
userInstance: null,
replyUserId: null,
replyUserHost: null,
replyUserInstance: null,
renoteUserId: null,
renoteUserHost: null,
renoteUserInstance: null,
processErrors: [],
};

16
packages/backend/test/unit/RoleService.ts
View file

@ -15,6 +15,7 @@ import type { MockFunctionMetadata } from 'jest-mock';
import { GlobalModule } from '@/GlobalModule.js';
import { RoleService } from '@/core/RoleService.js';
import {
InstancesRepository,
MiMeta,
MiRole,
MiRoleAssignment,
@ -39,6 +40,7 @@ const moduleMocker = new ModuleMocker(global);
describe('RoleService', () => {
let app: TestingModule;
let roleService: RoleService;
let instancesRepository: InstancesRepository;
let usersRepository: UsersRepository;
let rolesRepository: RolesRepository;
let roleAssignmentsRepository: RoleAssignmentsRepository;
@ -47,6 +49,19 @@ describe('RoleService', () => {
let clock: lolex.InstalledClock;
async function createUser(data: Partial<MiUser> = {}) {
if (data.host != null) {
await instancesRepository
.createQueryBuilder('instance')
.insert()
.values({
id: genAidx(Date.now()),
firstRetrievedAt: new Date(),
host: data.host,
})
.orIgnore()
.execute();
}
const un = secureRndstr(16);
const x = await usersRepository.insert({
id: genAidx(Date.now()),
@ -145,6 +160,7 @@ describe('RoleService', () => {
app.enableShutdownHooks();
roleService = app.get<RoleService>(RoleService);
instancesRepository = app.get<InstancesRepository>(DI.instancesRepository);
usersRepository = app.get<UsersRepository>(DI.usersRepository);
rolesRepository = app.get<RolesRepository>(DI.rolesRepository);
roleAssignmentsRepository = app.get<RoleAssignmentsRepository>(DI.roleAssignmentsRepository);

18
packages/backend/test/unit/UserSearchService.ts
View file

@ -7,16 +7,18 @@ import { Test, TestingModule } from '@nestjs/testing';
import { describe, jest, test } from '@jest/globals';
import { In } from 'typeorm';
import { UserSearchService } from '@/core/UserSearchService.js';
import { FollowingsRepository, MiUser, UserProfilesRepository, UsersRepository } from '@/models/_.js';
import { FollowingsRepository, InstancesRepository, MiUser, UserProfilesRepository, UsersRepository } from '@/models/_.js';
import { IdService } from '@/core/IdService.js';
import { GlobalModule } from '@/GlobalModule.js';
import { DI } from '@/di-symbols.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { genAidx } from '@/misc/id/aidx.js';
describe('UserSearchService', () => {
let app: TestingModule;
let service: UserSearchService;
let instancesRepository: InstancesRepository;
let usersRepository: UsersRepository;
let followingsRepository: FollowingsRepository;
let idService: IdService;
@ -35,6 +37,19 @@ describe('UserSearchService', () => {
let bobby: MiUser;
async function createUser(data: Partial<MiUser> = {}) {
if (data.host != null) {
await instancesRepository
.createQueryBuilder('instance')
.insert()
.values({
id: genAidx(Date.now()),
firstRetrievedAt: new Date(),
host: data.host,
})
.orIgnore()
.execute();
}
const user = await usersRepository
.insert({
id: idService.gen(),
@ -104,6 +119,7 @@ describe('UserSearchService', () => {
await app.init();
instancesRepository = app.get<InstancesRepository>(DI.instancesRepository);
usersRepository = app.get(DI.usersRepository);
userProfilesRepository = app.get(DI.userProfilesRepository);
followingsRepository = app.get(DI.followingsRepository);

19
packages/backend/test/unit/activitypub.ts
View file

@ -103,6 +103,25 @@ describe('ActivityPub', () => {
let config: Config;
const metaInitial = {
id: 'x',
name: 'Test Instance',
shortName: 'Test Instance',
description: 'Test Instance',
langs: [] as string[],
pinnedUsers: [] as string[],
hiddenTags: [] as string[],
prohibitedWordsForNameOfUser: [] as string[],
silencedHosts: [] as string[],
mediaSilencedHosts: [] as string[],
policies: {},
serverRules: [] as string[],
bannedEmailDomains: [] as string[],
preservedUsernames: [] as string[],
bubbleInstances: [] as string[],
trustedLinkUrlPatterns: [] as string[],
federation: 'all',
federationHosts: [] as string[],
allowUnsignedFetch: 'always',
cacheRemoteFiles: true,
cacheRemoteSensitiveFiles: true,
enableFanoutTimeline: true,

91
packages/backend/test/unit/misc/diff-arrays.ts Normal file
View file

@ -0,0 +1,91 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { diffArrays, diffArraysSimple } from '@/misc/diff-arrays.js';
describe(diffArrays, () => {
it('should return empty result when both inputs are null', () => {
const result = diffArrays(null, null);
expect(result.added).toHaveLength(0);
expect(result.removed).toHaveLength(0);
});
it('should return empty result when both inputs are empty', () => {
const result = diffArrays([], []);
expect(result.added).toHaveLength(0);
expect(result.removed).toHaveLength(0);
});
it('should remove before when after is empty', () => {
const result = diffArrays([1, 2, 3], []);
expect(result.added).toHaveLength(0);
expect(result.removed).toEqual([1, 2, 3]);
});
it('should deduplicate before when after is empty', () => {
const result = diffArrays([1, 1, 2, 2, 3], []);
expect(result.added).toHaveLength(0);
expect(result.removed).toEqual([1, 2, 3]);
});
it('should add after when before is empty', () => {
const result = diffArrays([], [1, 2, 3]);
expect(result.added).toEqual([1, 2, 3]);
expect(result.removed).toHaveLength(0);
});
it('should deduplicate after when before is empty', () => {
const result = diffArrays([], [1, 1, 2, 2, 3]);
expect(result.added).toEqual([1, 2, 3]);
expect(result.removed).toHaveLength(0);
});
it('should return diff when both have values', () => {
const result = diffArrays(
['a', 'b', 'c', 'd'],
['a', 'c', 'e', 'f'],
);
expect(result.added).toEqual(['e', 'f']);
expect(result.removed).toEqual(['b', 'd']);
});
});
describe(diffArraysSimple, () => {
it('should return false when both inputs are null', () => {
const result = diffArraysSimple(null, null);
expect(result).toBe(false);
});
it('should return false when both inputs are empty', () => {
const result = diffArraysSimple([], []);
expect(result).toBe(false);
});
it('should return true when before is populated and after is empty', () => {
const result = diffArraysSimple([1, 2, 3], []);
expect(result).toBe(true);
});
it('should return true when before is empty and after is populated', () => {
const result = diffArraysSimple([], [1, 2, 3]);
expect(result).toBe(true);
});
it('should return true when values have changed', () => {
const result = diffArraysSimple(
['a', 'a', 'b', 'c'],
['a', 'b', 'c', 'd'],
);
expect(result).toBe(true);
});
it('should return false when values have not changed', () => {
const result = diffArraysSimple(
['a', 'a', 'b', 'c'],
['a', 'b', 'c', 'c'],
);
expect(result).toBe(false);
});
});

3
packages/backend/test/unit/misc/is-renote.ts
View file

@ -40,10 +40,13 @@ const base: MiNote = {
channelId: null,
channel: null,
userHost: null,
userInstance: null,
replyUserId: null,
replyUserHost: null,
replyUserInstance: null,
renoteUserId: null,
renoteUserHost: null,
renoteUserInstance: null,
processErrors: [],
};

1
packages/backend/tsconfig.json
View file

@ -23,6 +23,7 @@
"emitDecoratorMetadata": true,
"resolveJsonModule": true,
"isolatedModules": true,
"incremental": true,
"rootDir": "./src",
"baseUrl": "./",
"paths": {

1
packages/frontend-embed/src/workers/tsconfig.json
View file

@ -1,5 +1,6 @@
{
"compilerOptions": {
"lib": ["esnext", "webworker"],
"incremental": true
}
}

1
packages/frontend-embed/tsconfig.json
View file

@ -23,6 +23,7 @@
"useDefineForClassFields": true,
"verbatimModuleSyntax": true,
"skipLibCheck": true,
"incremental": true,
"baseUrl": ".",
"paths": {
"@/*": ["./src/*"],

1
packages/frontend-shared/tsconfig.json
View file

@ -18,6 +18,7 @@
"esModuleInterop": true,
"verbatimModuleSyntax": true,
"skipLibCheck": true,
"incremental": true,
"baseUrl": ".",
"paths": {
"@/*": ["./*"],

1
packages/frontend/.storybook/tsconfig.json
View file

@ -18,6 +18,7 @@
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"incremental": true,
"jsx": "react",
"jsxFactory": "h"
},

1
packages/frontend/src/workers/tsconfig.json
View file

@ -1,5 +1,6 @@
{
"compilerOptions": {
"lib": ["esnext", "webworker"],
"incremental": true
}
}

1
packages/frontend/test/tsconfig.json
View file

@ -22,6 +22,7 @@
"emitDecoratorMetadata": true,
"resolveJsonModule": true,
"isolatedModules": true,
"incremental": true,
"baseUrl": "./",
"paths": {
"@/*": ["../src/*"]

1
packages/frontend/tsconfig.json
View file

@ -23,6 +23,7 @@
"useDefineForClassFields": true,
"verbatimModuleSyntax": true,
"skipLibCheck": true,
"incremental": true,
"baseUrl": ".",
"paths": {
"@/*": ["./src/*"],

1
packages/misskey-bubble-game/tsconfig.json
View file

@ -16,6 +16,7 @@
"noImplicitReturns": true,
"esModuleInterop": true,
"skipLibCheck": true,
"incremental": true,
"typeRoots": [
"./node_modules/@types"
],

1
packages/misskey-js/generator/tsconfig.json
View file

@ -8,6 +8,7 @@
"strictFunctionTypes": true,
"strictNullChecks": true,
"esModuleInterop": true,
"incremental": true,
"lib": [
"esnext",
]

1
packages/misskey-js/tsconfig.json
View file

@ -17,6 +17,7 @@
"esModuleInterop": true,
"exactOptionalPropertyTypes": true,
"skipLibCheck": true,
"incremental": true,
"typeRoots": [
"./node_modules/@types"
],

1
packages/misskey-reversi/tsconfig.json
View file

@ -16,6 +16,7 @@
"noImplicitReturns": true,
"esModuleInterop": true,
"skipLibCheck": true,
"incremental": true,
"typeRoots": [
"./node_modules/@types"
],

1
packages/sw/tsconfig.json
View file

@ -20,6 +20,7 @@
"resolveJsonModule": true,
"isolatedModules": true,
"skipLibCheck": true,
"incremental": true,
"baseUrl": ".",
"paths": {
"@/*": ["./src/*"],