diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts index 5c271b81e3..a0f2607ddc 100644 --- a/packages/backend/src/core/HttpRequestService.ts +++ b/packages/backend/src/core/HttpRequestService.ts @@ -235,7 +235,7 @@ export class HttpRequestService { } @bindThis - public async getActivityJson(url: string, isLocalAddressAllowed = false): Promise { + public async getActivityJson(url: string, isLocalAddressAllowed = false, allowAnonymous = false): Promise { this.apUtilityService.assertApUrl(url); const res = await this.send(url, { @@ -255,7 +255,11 @@ export class HttpRequestService { // Make sure the object ID matches the final URL (which is where it actually exists). // The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match. - this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); + if (allowAnonymous && activity.id == null) { + activity.id = res.url; + } else { + this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); + } return activity as IObjectWithId; } diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index 92c91f47de..c06939eae2 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -36,7 +36,7 @@ import InstanceChart from '@/core/chart/charts/instance.js'; import FederationChart from '@/core/chart/charts/federation.js'; import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js'; -import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js'; +import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js'; import { ApNoteService } from './models/ApNoteService.js'; import { ApLoggerService } from './ApLoggerService.js'; import { ApDbResolverService } from './ApDbResolverService.js'; @@ -106,22 +106,25 @@ export class ApInboxService { let result = undefined as string | void; if (isCollectionOrOrderedCollection(activity)) { const results = [] as [string, string | void][]; - // eslint-disable-next-line no-param-reassign resolver ??= this.apResolverService.createResolver(); - const items = toArray(isCollection(activity) ? activity.items : activity.orderedItems); - if (items.length >= resolver.getRecursionLimit()) { - throw new Error(`skipping activity: collection would surpass recursion limit: ${this.utilityService.extractDbHost(actor.uri)}`); - } - - for (const item of items) { - const act = await resolver.resolve(item); - if (act.id == null || this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) { - this.logger.debug('skipping activity: activity id is null or mismatching'); - continue; + const items = await resolver.resolveCollectionItems(activity); + for (let i = 0; i < items.length; i++) { + const act = items[i]; + if (act.id != null) { + if (this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) { + this.logger.warn('skipping activity: activity id mismatch'); + continue; + } + } else { + // Activity ID should only be string or undefined. + act.id = undefined; } + try { - results.push([getApId(item), await this.performOneActivity(actor, act, resolver)]); + const id = getNullableApId(act) ?? `${getNullableApId(activity)}#${i}`; + const result = await this.performOneActivity(actor, act, resolver); + results.push([id, result]); } catch (err) { if (err instanceof Error || typeof err === 'string') { this.logger.error(err); diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index b665b51700..4c7cac2169 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -184,10 +184,11 @@ export class ApRequestService { * Get AP object with http-signature * @param user http-signature user * @param url URL to fetch - * @param followAlternate + * @param allowAnonymous If a fetched object lacks an ID, then it will be auto-generated from the final URL. (default: false) + * @param followAlternate Whether to resolve HTML responses to their referenced canonical AP endpoint. (default: true) */ @bindThis - public async signedGet(url: string, user: { id: MiUser['id'] }, followAlternate?: boolean): Promise { + public async signedGet(url: string, user: { id: MiUser['id'] }, allowAnonymous = false, followAlternate?: boolean): Promise { this.apUtilityService.assertApUrl(url); const _followAlternate = followAlternate ?? true; @@ -258,7 +259,7 @@ export class ApRequestService { if (alternate) { const href = alternate.getAttribute('href'); if (href && this.apUtilityService.haveSameAuthority(url, href)) { - return await this.signedGet(href, user, false); + return await this.signedGet(href, user, allowAnonymous, false); } } } catch { @@ -275,7 +276,11 @@ export class ApRequestService { // Make sure the object ID matches the final URL (which is where it actually exists). // The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match. - this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); + if (allowAnonymous && activity.id == null) { + activity.id = res.url; + } else { + this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); + } return activity as IObjectWithId; } diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index 5e58f848c0..7997eccced 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -5,6 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { IsNull, Not } from 'typeorm'; +import promiseLimit from 'promise-limit'; import type { MiLocalUser, MiRemoteUser } from '@/models/User.js'; import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta, SkApFetchLog } from '@/models/_.js'; import type { Config } from '@/config.js'; @@ -19,11 +20,12 @@ import { ApLogService, calculateDurationSince, extractObjectContext } from '@/co import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; -import { getApId, getNullableApId, IObjectWithId, isCollectionOrOrderedCollection } from './type.js'; +import { toArray } from '@/misc/prelude/array.js'; +import { AnyCollection, getApId, getNullableApId, IObjectWithId, isCollection, isCollectionOrOrderedCollection, isCollectionPage, isOrderedCollection, isOrderedCollectionPage } from './type.js'; import { ApDbResolverService } from './ApDbResolverService.js'; import { ApRendererService } from './ApRendererService.js'; import { ApRequestService } from './ApRequestService.js'; -import type { IObject, ICollection, IOrderedCollection, ApObject } from './type.js'; +import type { IObject, ApObject, IAnonymousObject } from './type.js'; export class Resolver { private history: Set; @@ -63,11 +65,16 @@ export class Resolver { return this.recursionLimit; } + public async resolveCollection(value: string | IObjectWithId, allowAnonymous?: boolean, sentFromUri?: string): Promise; + public async resolveCollection(value: string | IObject, allowAnonymous: boolean | undefined, sentFromUri: string): Promise; + public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise; @bindThis - public async resolveCollection(value: string | IObject): Promise { + public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise { const collection = typeof value === 'string' - ? await this.resolve(value) - : value; + ? sentFromUri + ? await this.secureResolve(value, sentFromUri, allowAnonymous) + : await this.resolve(value, allowAnonymous) + : value; // TODO try and remove this eventually, as it's a major security foot-gun if (isCollectionOrOrderedCollection(collection)) { return collection; @@ -76,20 +83,110 @@ export class Resolver { } } + public async resolveCollectionItems(collection: IAnonymousObject, limit?: number | null, allowAnonymousItems?: true, concurrency?: number): Promise; + public async resolveCollectionItems(collection: string | IObjectWithId, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise; + public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise; + /** + * Recursively resolves items from a collection. + * Stops when reaching the resolution limit or an optional item limit - whichever is lower. + * This method supports Collection, OrderedCollection, and individual pages of either type. + * Malformed collections (mixing Ordered and un-Ordered types) are also supported. + * @param collection Collection to resolve from - can be a URL or object of any supported collection type. + * @param limit Maximum number of items to resolve. If null or undefined (default), then items will be resolved until reaching the recursion limit. + * @param allowAnonymousItems If true, collection items can be anonymous (lack an ID). If false (default), then an error is thrown when reaching an item without ID. + * @param concurrency Maximum number of items to resolve at once. (default: 4) + */ + @bindThis + public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency = 4): Promise { + const resolvedItems: IObject[] = []; + + // This is pulled up to avoid code duplication below + const iterate = async(items: ApObject, current: AnyCollection) => { + const sentFrom = current.id; + const itemArr = toArray(items); + const itemLimit = limit ?? Number.MAX_SAFE_INTEGER; + const allowAnonymous = allowAnonymousItems ?? false; + await this.resolveItemArray(itemArr, sentFrom, itemLimit, concurrency, allowAnonymous, resolvedItems); + }; + + let current: AnyCollection | null = await this.resolveCollection(collection); + do { + // Iterate all items in the current page + if (current.items) { + await iterate(current.items, current); + } + if (current.orderedItems) { + await iterate(current.orderedItems, current); + } + + if (this.history.size >= this.recursionLimit) { + // Stop when we reach the fetch limit + current = null; + } else if (limit != null && resolvedItems.length >= limit) { + // Stop when we reach the item limit + current = null; + } else if (isCollection(current) || isOrderedCollection(current)) { + // Continue to first page + current = current.first ? await this.resolveCollection(current.first, true, current.id) : null; + } else if (isCollectionPage(current) || isOrderedCollectionPage(current)) { + // Continue to next page + current = current.next ? await this.resolveCollection(current.next, true, current.id) : null; + } else { + // Stop in all other conditions + current = null; + } + } while (current != null); + + return resolvedItems; + } + + private async resolveItemArray(source: (string | IObject)[], sentFrom: undefined, itemLimit: number, concurrency: number, allowAnonymousItems: true, destination: IAnonymousObject[]): Promise; + private async resolveItemArray(source: (string | IObject)[], sentFrom: string, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObjectWithId[]): Promise; + private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise; + private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise { + const recursionLimit = this.recursionLimit - this.history.size; + const batchLimit = Math.min(source.length, recursionLimit, itemLimit); + + const limiter = promiseLimit(concurrency); + const batch = await Promise.all(source + .slice(0, batchLimit) + .map(item => limiter(async () => { + if (sentFrom) { + // Use secureResolve to avoid re-fetching items that were included inline. + return await this.secureResolve(item, sentFrom, allowAnonymousItems); + } else if (allowAnonymousItems) { + return await this.resolveAnonymous(item); + } else { + // ID is required if we have neither sentFrom not allowAnonymousItems + const id = getApId(item); + return await this.resolve(id); + } + }))); + + destination.push(...batch); + }; + /** * Securely resolves an AP object or URL that has been sent from another instance. * An input object is trusted if and only if its ID matches the authority of sentFromUri. * In all other cases, the object is re-fetched from remote by input string or object ID. + * @param input The input object or URL to resolve + * @param sentFromUri The URL where this object originated. This MUST be accurate - all security checks depend on this value! + * @param allowAnonymous If true, anonymous objects are allowed and will have their ID set to sentFromUri. If false (default) then anonymous objects will be rejected with an error. */ @bindThis - public async secureResolve(input: ApObject, sentFromUri: string): Promise { + public async secureResolve(input: string | IObject | [string | IObject], sentFromUri: string, allowAnonymous?: boolean): Promise { // Unpack arrays to get the value element. const value = fromTuple(input); - if (value == null) { - throw new IdentifiableError('20058164-9de1-4573-8715-425753a21c1d', 'Cannot resolve null input'); + + // If anonymous input is allowed, then any object is automatically valid if we set the ID. + // We can short-circuit here and avoid un-necessary checks. + if (allowAnonymous && typeof(value) === 'object' && value.id == null) { + value.id = sentFromUri; + return value as IObjectWithId; } - // This will throw if the input has no ID, which is good because we can't verify an anonymous object anyway. + // This ensures the input has a string ID, protecting against type confusion and rejecting anonymous objects. const id = getApId(value); // Check if we can use the provided object as-is. @@ -100,28 +197,52 @@ export class Resolver { } // If the checks didn't pass, then we must fetch the object and use that. - return await this.resolve(id); + return await this.resolve(id, allowAnonymous); } - public async resolve(value: string | [string]): Promise; - public async resolve(value: string | IObject | [string | IObject]): Promise; + /** + * Resolves an anonymous object. + * The returned value will not have any ID present. + * If one is provided in the response, it will be removed automatically. + */ @bindThis - public async resolve(value: string | IObject | [string | IObject]): Promise { + public async resolveAnonymous(value: string | IObject | [string | IObject]): Promise { value = fromTuple(value); + const object = await this.resolve(value); + object.id = undefined; + + return object as IAnonymousObject; + } + + public async resolve(value: string | [string], allowAnonymous?: boolean): Promise; + public async resolve(value: string | IObjectWithId | [string | IObjectWithId], allowAnonymous?: boolean): Promise; + public async resolve(value: string | IObject | [string | IObject], allowAnonymous?: boolean): Promise; + /** + * Resolves a URL or object to an AP object. + * Tuples are expanded to their first element before anything else, and non-string inputs are returned as-is. + * Otherwise, the string URL is fetched and validated to represent a valid ActivityPub object. + * @param value The input value to resolve + * @param allowAnonymous Determines what to do if a response object lacks an ID field. If false (default), then an exception is thrown. If true, then the ID is populated from the final response URL. + */ + @bindThis + public async resolve(value: string | IObject | [string | IObject], allowAnonymous = false): Promise { + value = fromTuple(value); + + // TODO try and remove this eventually, as it's a major security foot-gun if (typeof value !== 'string') { return value; } const host = this.utilityService.extractDbHost(value); if (this.config.activityLogging.enabled && !this.utilityService.isSelfHost(host)) { - return await this._resolveLogged(value, host); + return await this._resolveLogged(value, host, allowAnonymous); } else { - return await this._resolve(value, host); + return await this._resolve(value, host, allowAnonymous); } } - private async _resolveLogged(requestUri: string, host: string): Promise { + private async _resolveLogged(requestUri: string, host: string, allowAnonymous: boolean): Promise { const startTime = process.hrtime.bigint(); const log = await this.apLogService.createFetchLog({ @@ -130,7 +251,7 @@ export class Resolver { }); try { - const result = await this._resolve(requestUri, host, log); + const result = await this._resolve(requestUri, host, allowAnonymous, log); log.accepted = true; log.result = 'ok'; @@ -150,7 +271,7 @@ export class Resolver { } } - private async _resolve(value: string, host: string, log?: SkApFetchLog): Promise { + private async _resolve(value: string, host: string, allowAnonymous: boolean, log?: SkApFetchLog): Promise { if (value.includes('#')) { // URLs with fragment parts cannot be resolved correctly because // the fragment part does not get transmitted over HTTP(S). @@ -181,8 +302,8 @@ export class Resolver { } const object = (this.user - ? await this.apRequestService.signedGet(value, this.user) - : await this.httpRequestService.getActivityJson(value)); + ? await this.apRequestService.signedGet(value, this.user, allowAnonymous) + : await this.httpRequestService.getActivityJson(value, false, allowAnonymous)); if (log) { const { object: objectOnly, context, contextHash } = extractObjectContext(object); diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 281733d484..ae9fe118bc 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -43,6 +43,18 @@ export interface IObjectWithId extends IObject { id: string; } +export function isObjectWithId(object: IObject): object is IObjectWithId { + return typeof(object.id) === 'string'; +} + +export interface IAnonymousObject extends IObject { + id: undefined; +} + +export function isAnonymousObject(object: IObject): object is IAnonymousObject { + return object.id === undefined; +} + /** * Get array of ActivityStreams Objects id */ @@ -125,48 +137,46 @@ export interface IActivity extends IObject { }; } -export interface ICollection extends IObject { +export interface CollectionBase extends IObject { + totalItems?: number; + first?: IObject | string; + last?: IObject | string; + current?: IObject | string; + partOf?: IObject | string; + next?: IObject | string; + prev?: IObject | string; + items?: ApObject; + orderedItems?: ApObject; +} + +export interface ICollection extends CollectionBase { type: 'Collection'; totalItems: number; - first?: IObject | string; - last?: IObject | string; - current?: IObject | string; items?: ApObject; + orderedItems?: undefined; } -export interface IOrderedCollection extends IObject { +export interface IOrderedCollection extends CollectionBase { type: 'OrderedCollection'; totalItems: number; - first?: IObject | string; - last?: IObject | string; - current?: IObject | string; + items?: undefined; orderedItems?: ApObject; } -export interface ICollectionPage extends IObject { +export interface ICollectionPage extends CollectionBase { type: 'CollectionPage'; - totalItems: number; - first?: IObject | string; - last?: IObject | string; - current?: IObject | string; - partOf?: IObject | string; - next?: IObject | string; - prev?: IObject | string; items?: ApObject; + orderedItems?: undefined; } -export interface IOrderedCollectionPage extends IObject { +export interface IOrderedCollectionPage extends CollectionBase { type: 'OrderedCollectionPage'; - totalItems: number; - first?: IObject | string; - last?: IObject | string; - current?: IObject | string; - partOf?: IObject | string; - next?: IObject | string; - prev?: IObject | string; + items?: undefined; orderedItems?: ApObject; } +export type AnyCollection = ICollection | IOrderedCollection | ICollectionPage | IOrderedCollectionPage; + export const validPost = ['Note', 'Question', 'Article', 'Audio', 'Document', 'Image', 'Page', 'Video', 'Event']; export const isPost = (object: IObject): object is IPost => { @@ -269,7 +279,7 @@ export const isCollectionPage = (object: IObject): object is ICollectionPage => export const isOrderedCollectionPage = (object: IObject): object is IOrderedCollectionPage => getApType(object) === 'OrderedCollectionPage'; -export const isCollectionOrOrderedCollection = (object: IObject): object is ICollection | IOrderedCollection => +export const isCollectionOrOrderedCollection = (object: IObject): object is AnyCollection => isCollection(object) || isOrderedCollection(object) || isCollectionPage(object) || isOrderedCollectionPage(object); export interface IApPropertyValue extends IObject { diff --git a/packages/backend/src/server/api/endpoints/ap/get.ts b/packages/backend/src/server/api/endpoints/ap/get.ts index 14286bc23e..06dd37a140 100644 --- a/packages/backend/src/server/api/endpoints/ap/get.ts +++ b/packages/backend/src/server/api/endpoints/ap/get.ts @@ -7,6 +7,7 @@ import { Injectable } from '@nestjs/common'; import ms from 'ms'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { ApResolverService } from '@/core/activitypub/ApResolverService.js'; +import { isCollectionOrOrderedCollection, isOrderedCollection, isOrderedCollectionPage } from '@/core/activitypub/type.js'; export const meta = { tags: ['federation'], @@ -33,6 +34,9 @@ export const paramDef = { type: 'object', properties: { uri: { type: 'string' }, + expandCollectionItems: { type: 'boolean' }, + expandCollectionLimit: { type: 'integer', nullable: true }, + allowAnonymous: { type: 'boolean' }, }, required: ['uri'], } as const; @@ -44,7 +48,18 @@ export default class extends Endpoint { // eslint- ) { super(meta, paramDef, async (ps, me) => { const resolver = this.apResolverService.createResolver(); - const object = await resolver.resolve(ps.uri); + const object = await resolver.resolve(ps.uri, ps.allowAnonymous ?? false); + + if (ps.expandCollectionItems && isCollectionOrOrderedCollection(object)) { + const items = await resolver.resolveCollectionItems(object, ps.expandCollectionLimit, ps.allowAnonymous ?? false); + + if (isOrderedCollection(object) || isOrderedCollectionPage(object)) { + object.orderedItems = items; + } else { + object.items = items; + } + } + return object; }); } diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 55302960dc..5e5f4f5db5 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -12918,6 +12918,9 @@ export type operations = { content: { 'application/json': { uri: string; + expandCollectionItems?: boolean; + expandCollectionLimit?: number | null; + allowAnonymous?: boolean; }; }; };