merge: Resolve AP collection items (prerequisite for future work) (!1067)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/1067

Approved-by: dakkar <dakkar@thenautilus.net>
Approved-by: Marie <github@yuugi.dev>
This commit is contained in:
Hazelnoot 2025-06-01 17:34:24 +00:00
commit 4c99406aa2
7 changed files with 226 additions and 65 deletions

View file

@ -235,7 +235,7 @@ export class HttpRequestService {
}
@bindThis
public async getActivityJson(url: string, isLocalAddressAllowed = false): Promise<IObjectWithId> {
public async getActivityJson(url: string, isLocalAddressAllowed = false, allowAnonymous = false): Promise<IObjectWithId> {
this.apUtilityService.assertApUrl(url);
const res = await this.send(url, {
@ -255,7 +255,11 @@ export class HttpRequestService {
// Make sure the object ID matches the final URL (which is where it actually exists).
// The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match.
if (allowAnonymous && activity.id == null) {
activity.id = res.url;
} else {
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
}
return activity as IObjectWithId;
}

View file

@ -36,7 +36,7 @@ import InstanceChart from '@/core/chart/charts/instance.js';
import FederationChart from '@/core/chart/charts/federation.js';
import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js';
import { UpdateInstanceQueue } from '@/core/UpdateInstanceQueue.js';
import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js';
import { getApHrefNullable, getApId, getApIds, getApType, getNullableApId, isAccept, isActor, isAdd, isAnnounce, isApObject, isBlock, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isDislike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost, isActivity, IObjectWithId } from './type.js';
import { ApNoteService } from './models/ApNoteService.js';
import { ApLoggerService } from './ApLoggerService.js';
import { ApDbResolverService } from './ApDbResolverService.js';
@ -106,22 +106,25 @@ export class ApInboxService {
let result = undefined as string | void;
if (isCollectionOrOrderedCollection(activity)) {
const results = [] as [string, string | void][];
// eslint-disable-next-line no-param-reassign
resolver ??= this.apResolverService.createResolver();
const items = toArray(isCollection(activity) ? activity.items : activity.orderedItems);
if (items.length >= resolver.getRecursionLimit()) {
throw new Error(`skipping activity: collection would surpass recursion limit: ${this.utilityService.extractDbHost(actor.uri)}`);
}
for (const item of items) {
const act = await resolver.resolve(item);
if (act.id == null || this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) {
this.logger.debug('skipping activity: activity id is null or mismatching');
const items = await resolver.resolveCollectionItems(activity);
for (let i = 0; i < items.length; i++) {
const act = items[i];
if (act.id != null) {
if (this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) {
this.logger.warn('skipping activity: activity id mismatch');
continue;
}
} else {
// Activity ID should only be string or undefined.
act.id = undefined;
}
try {
results.push([getApId(item), await this.performOneActivity(actor, act, resolver)]);
const id = getNullableApId(act) ?? `${getNullableApId(activity)}#${i}`;
const result = await this.performOneActivity(actor, act, resolver);
results.push([id, result]);
} catch (err) {
if (err instanceof Error || typeof err === 'string') {
this.logger.error(err);

View file

@ -184,10 +184,11 @@ export class ApRequestService {
* Get AP object with http-signature
* @param user http-signature user
* @param url URL to fetch
* @param followAlternate
* @param allowAnonymous If a fetched object lacks an ID, then it will be auto-generated from the final URL. (default: false)
* @param followAlternate Whether to resolve HTML responses to their referenced canonical AP endpoint. (default: true)
*/
@bindThis
public async signedGet(url: string, user: { id: MiUser['id'] }, followAlternate?: boolean): Promise<IObjectWithId> {
public async signedGet(url: string, user: { id: MiUser['id'] }, allowAnonymous = false, followAlternate?: boolean): Promise<IObjectWithId> {
this.apUtilityService.assertApUrl(url);
const _followAlternate = followAlternate ?? true;
@ -258,7 +259,7 @@ export class ApRequestService {
if (alternate) {
const href = alternate.getAttribute('href');
if (href && this.apUtilityService.haveSameAuthority(url, href)) {
return await this.signedGet(href, user, false);
return await this.signedGet(href, user, allowAnonymous, false);
}
}
} catch {
@ -275,7 +276,11 @@ export class ApRequestService {
// Make sure the object ID matches the final URL (which is where it actually exists).
// The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match.
if (allowAnonymous && activity.id == null) {
activity.id = res.url;
} else {
this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url);
}
return activity as IObjectWithId;
}

View file

@ -5,6 +5,7 @@
import { Inject, Injectable } from '@nestjs/common';
import { IsNull, Not } from 'typeorm';
import promiseLimit from 'promise-limit';
import type { MiLocalUser, MiRemoteUser } from '@/models/User.js';
import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta, SkApFetchLog } from '@/models/_.js';
import type { Config } from '@/config.js';
@ -19,11 +20,12 @@ import { ApLogService, calculateDurationSince, extractObjectContext } from '@/co
import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { getApId, getNullableApId, IObjectWithId, isCollectionOrOrderedCollection } from './type.js';
import { toArray } from '@/misc/prelude/array.js';
import { AnyCollection, getApId, getNullableApId, IObjectWithId, isCollection, isCollectionOrOrderedCollection, isCollectionPage, isOrderedCollection, isOrderedCollectionPage } from './type.js';
import { ApDbResolverService } from './ApDbResolverService.js';
import { ApRendererService } from './ApRendererService.js';
import { ApRequestService } from './ApRequestService.js';
import type { IObject, ICollection, IOrderedCollection, ApObject } from './type.js';
import type { IObject, ApObject, IAnonymousObject } from './type.js';
export class Resolver {
private history: Set<string>;
@ -63,11 +65,16 @@ export class Resolver {
return this.recursionLimit;
}
public async resolveCollection(value: string | IObjectWithId, allowAnonymous?: boolean, sentFromUri?: string): Promise<AnyCollection & IObjectWithId>;
public async resolveCollection(value: string | IObject, allowAnonymous: boolean | undefined, sentFromUri: string): Promise<AnyCollection & IObjectWithId>;
public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise<AnyCollection>;
@bindThis
public async resolveCollection(value: string | IObject): Promise<ICollection | IOrderedCollection> {
public async resolveCollection(value: string | IObject, allowAnonymous?: boolean, sentFromUri?: string): Promise<AnyCollection> {
const collection = typeof value === 'string'
? await this.resolve(value)
: value;
? sentFromUri
? await this.secureResolve(value, sentFromUri, allowAnonymous)
: await this.resolve(value, allowAnonymous)
: value; // TODO try and remove this eventually, as it's a major security foot-gun
if (isCollectionOrOrderedCollection(collection)) {
return collection;
@ -76,20 +83,110 @@ export class Resolver {
}
}
public async resolveCollectionItems(collection: IAnonymousObject, limit?: number | null, allowAnonymousItems?: true, concurrency?: number): Promise<IAnonymousObject[]>;
public async resolveCollectionItems(collection: string | IObjectWithId, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise<IObjectWithId[]>;
public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency?: number): Promise<IObject[]>;
/**
* Recursively resolves items from a collection.
* Stops when reaching the resolution limit or an optional item limit - whichever is lower.
* This method supports Collection, OrderedCollection, and individual pages of either type.
* Malformed collections (mixing Ordered and un-Ordered types) are also supported.
* @param collection Collection to resolve from - can be a URL or object of any supported collection type.
* @param limit Maximum number of items to resolve. If null or undefined (default), then items will be resolved until reaching the recursion limit.
* @param allowAnonymousItems If true, collection items can be anonymous (lack an ID). If false (default), then an error is thrown when reaching an item without ID.
* @param concurrency Maximum number of items to resolve at once. (default: 4)
*/
@bindThis
public async resolveCollectionItems(collection: string | IObject, limit?: number | null, allowAnonymousItems?: boolean, concurrency = 4): Promise<IObject[]> {
const resolvedItems: IObject[] = [];
// This is pulled up to avoid code duplication below
const iterate = async(items: ApObject, current: AnyCollection) => {
const sentFrom = current.id;
const itemArr = toArray(items);
const itemLimit = limit ?? Number.MAX_SAFE_INTEGER;
const allowAnonymous = allowAnonymousItems ?? false;
await this.resolveItemArray(itemArr, sentFrom, itemLimit, concurrency, allowAnonymous, resolvedItems);
};
let current: AnyCollection | null = await this.resolveCollection(collection);
do {
// Iterate all items in the current page
if (current.items) {
await iterate(current.items, current);
}
if (current.orderedItems) {
await iterate(current.orderedItems, current);
}
if (this.history.size >= this.recursionLimit) {
// Stop when we reach the fetch limit
current = null;
} else if (limit != null && resolvedItems.length >= limit) {
// Stop when we reach the item limit
current = null;
} else if (isCollection(current) || isOrderedCollection(current)) {
// Continue to first page
current = current.first ? await this.resolveCollection(current.first, true, current.id) : null;
} else if (isCollectionPage(current) || isOrderedCollectionPage(current)) {
// Continue to next page
current = current.next ? await this.resolveCollection(current.next, true, current.id) : null;
} else {
// Stop in all other conditions
current = null;
}
} while (current != null);
return resolvedItems;
}
private async resolveItemArray(source: (string | IObject)[], sentFrom: undefined, itemLimit: number, concurrency: number, allowAnonymousItems: true, destination: IAnonymousObject[]): Promise<void>;
private async resolveItemArray(source: (string | IObject)[], sentFrom: string, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObjectWithId[]): Promise<void>;
private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise<void>;
private async resolveItemArray(source: (string | IObject)[], sentFrom: string | undefined, itemLimit: number, concurrency: number, allowAnonymousItems: boolean, destination: IObject[]): Promise<void> {
const recursionLimit = this.recursionLimit - this.history.size;
const batchLimit = Math.min(source.length, recursionLimit, itemLimit);
const limiter = promiseLimit<IObject>(concurrency);
const batch = await Promise.all(source
.slice(0, batchLimit)
.map(item => limiter(async () => {
if (sentFrom) {
// Use secureResolve to avoid re-fetching items that were included inline.
return await this.secureResolve(item, sentFrom, allowAnonymousItems);
} else if (allowAnonymousItems) {
return await this.resolveAnonymous(item);
} else {
// ID is required if we have neither sentFrom not allowAnonymousItems
const id = getApId(item);
return await this.resolve(id);
}
})));
destination.push(...batch);
};
/**
* Securely resolves an AP object or URL that has been sent from another instance.
* An input object is trusted if and only if its ID matches the authority of sentFromUri.
* In all other cases, the object is re-fetched from remote by input string or object ID.
* @param input The input object or URL to resolve
* @param sentFromUri The URL where this object originated. This MUST be accurate - all security checks depend on this value!
* @param allowAnonymous If true, anonymous objects are allowed and will have their ID set to sentFromUri. If false (default) then anonymous objects will be rejected with an error.
*/
@bindThis
public async secureResolve(input: ApObject, sentFromUri: string): Promise<IObjectWithId> {
public async secureResolve(input: string | IObject | [string | IObject], sentFromUri: string, allowAnonymous?: boolean): Promise<IObjectWithId> {
// Unpack arrays to get the value element.
const value = fromTuple(input);
if (value == null) {
throw new IdentifiableError('20058164-9de1-4573-8715-425753a21c1d', 'Cannot resolve null input');
// If anonymous input is allowed, then any object is automatically valid if we set the ID.
// We can short-circuit here and avoid un-necessary checks.
if (allowAnonymous && typeof(value) === 'object' && value.id == null) {
value.id = sentFromUri;
return value as IObjectWithId;
}
// This will throw if the input has no ID, which is good because we can't verify an anonymous object anyway.
// This ensures the input has a string ID, protecting against type confusion and rejecting anonymous objects.
const id = getApId(value);
// Check if we can use the provided object as-is.
@ -100,28 +197,52 @@ export class Resolver {
}
// If the checks didn't pass, then we must fetch the object and use that.
return await this.resolve(id);
return await this.resolve(id, allowAnonymous);
}
public async resolve(value: string | [string]): Promise<IObjectWithId>;
public async resolve(value: string | IObject | [string | IObject]): Promise<IObject>;
/**
* Resolves an anonymous object.
* The returned value will not have any ID present.
* If one is provided in the response, it will be removed automatically.
*/
@bindThis
public async resolve(value: string | IObject | [string | IObject]): Promise<IObject> {
public async resolveAnonymous(value: string | IObject | [string | IObject]): Promise<IAnonymousObject> {
value = fromTuple(value);
const object = await this.resolve(value);
object.id = undefined;
return object as IAnonymousObject;
}
public async resolve(value: string | [string], allowAnonymous?: boolean): Promise<IObjectWithId>;
public async resolve(value: string | IObjectWithId | [string | IObjectWithId], allowAnonymous?: boolean): Promise<IObjectWithId>;
public async resolve(value: string | IObject | [string | IObject], allowAnonymous?: boolean): Promise<IObject>;
/**
* Resolves a URL or object to an AP object.
* Tuples are expanded to their first element before anything else, and non-string inputs are returned as-is.
* Otherwise, the string URL is fetched and validated to represent a valid ActivityPub object.
* @param value The input value to resolve
* @param allowAnonymous Determines what to do if a response object lacks an ID field. If false (default), then an exception is thrown. If true, then the ID is populated from the final response URL.
*/
@bindThis
public async resolve(value: string | IObject | [string | IObject], allowAnonymous = false): Promise<IObject> {
value = fromTuple(value);
// TODO try and remove this eventually, as it's a major security foot-gun
if (typeof value !== 'string') {
return value;
}
const host = this.utilityService.extractDbHost(value);
if (this.config.activityLogging.enabled && !this.utilityService.isSelfHost(host)) {
return await this._resolveLogged(value, host);
return await this._resolveLogged(value, host, allowAnonymous);
} else {
return await this._resolve(value, host);
return await this._resolve(value, host, allowAnonymous);
}
}
private async _resolveLogged(requestUri: string, host: string): Promise<IObjectWithId> {
private async _resolveLogged(requestUri: string, host: string, allowAnonymous: boolean): Promise<IObjectWithId> {
const startTime = process.hrtime.bigint();
const log = await this.apLogService.createFetchLog({
@ -130,7 +251,7 @@ export class Resolver {
});
try {
const result = await this._resolve(requestUri, host, log);
const result = await this._resolve(requestUri, host, allowAnonymous, log);
log.accepted = true;
log.result = 'ok';
@ -150,7 +271,7 @@ export class Resolver {
}
}
private async _resolve(value: string, host: string, log?: SkApFetchLog): Promise<IObjectWithId> {
private async _resolve(value: string, host: string, allowAnonymous: boolean, log?: SkApFetchLog): Promise<IObjectWithId> {
if (value.includes('#')) {
// URLs with fragment parts cannot be resolved correctly because
// the fragment part does not get transmitted over HTTP(S).
@ -181,8 +302,8 @@ export class Resolver {
}
const object = (this.user
? await this.apRequestService.signedGet(value, this.user)
: await this.httpRequestService.getActivityJson(value));
? await this.apRequestService.signedGet(value, this.user, allowAnonymous)
: await this.httpRequestService.getActivityJson(value, false, allowAnonymous));
if (log) {
const { object: objectOnly, context, contextHash } = extractObjectContext(object);

View file

@ -43,6 +43,18 @@ export interface IObjectWithId extends IObject {
id: string;
}
export function isObjectWithId(object: IObject): object is IObjectWithId {
return typeof(object.id) === 'string';
}
export interface IAnonymousObject extends IObject {
id: undefined;
}
export function isAnonymousObject(object: IObject): object is IAnonymousObject {
return object.id === undefined;
}
/**
* Get array of ActivityStreams Objects id
*/
@ -125,48 +137,46 @@ export interface IActivity extends IObject {
};
}
export interface ICollection extends IObject {
export interface CollectionBase extends IObject {
totalItems?: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
partOf?: IObject | string;
next?: IObject | string;
prev?: IObject | string;
items?: ApObject;
orderedItems?: ApObject;
}
export interface ICollection extends CollectionBase {
type: 'Collection';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
items?: ApObject;
orderedItems?: undefined;
}
export interface IOrderedCollection extends IObject {
export interface IOrderedCollection extends CollectionBase {
type: 'OrderedCollection';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
items?: undefined;
orderedItems?: ApObject;
}
export interface ICollectionPage extends IObject {
export interface ICollectionPage extends CollectionBase {
type: 'CollectionPage';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
partOf?: IObject | string;
next?: IObject | string;
prev?: IObject | string;
items?: ApObject;
orderedItems?: undefined;
}
export interface IOrderedCollectionPage extends IObject {
export interface IOrderedCollectionPage extends CollectionBase {
type: 'OrderedCollectionPage';
totalItems: number;
first?: IObject | string;
last?: IObject | string;
current?: IObject | string;
partOf?: IObject | string;
next?: IObject | string;
prev?: IObject | string;
items?: undefined;
orderedItems?: ApObject;
}
export type AnyCollection = ICollection | IOrderedCollection | ICollectionPage | IOrderedCollectionPage;
export const validPost = ['Note', 'Question', 'Article', 'Audio', 'Document', 'Image', 'Page', 'Video', 'Event'];
export const isPost = (object: IObject): object is IPost => {
@ -269,7 +279,7 @@ export const isCollectionPage = (object: IObject): object is ICollectionPage =>
export const isOrderedCollectionPage = (object: IObject): object is IOrderedCollectionPage =>
getApType(object) === 'OrderedCollectionPage';
export const isCollectionOrOrderedCollection = (object: IObject): object is ICollection | IOrderedCollection =>
export const isCollectionOrOrderedCollection = (object: IObject): object is AnyCollection =>
isCollection(object) || isOrderedCollection(object) || isCollectionPage(object) || isOrderedCollectionPage(object);
export interface IApPropertyValue extends IObject {

View file

@ -7,6 +7,7 @@ import { Injectable } from '@nestjs/common';
import ms from 'ms';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { ApResolverService } from '@/core/activitypub/ApResolverService.js';
import { isCollectionOrOrderedCollection, isOrderedCollection, isOrderedCollectionPage } from '@/core/activitypub/type.js';
export const meta = {
tags: ['federation'],
@ -33,6 +34,9 @@ export const paramDef = {
type: 'object',
properties: {
uri: { type: 'string' },
expandCollectionItems: { type: 'boolean' },
expandCollectionLimit: { type: 'integer', nullable: true },
allowAnonymous: { type: 'boolean' },
},
required: ['uri'],
} as const;
@ -44,7 +48,18 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
) {
super(meta, paramDef, async (ps, me) => {
const resolver = this.apResolverService.createResolver();
const object = await resolver.resolve(ps.uri);
const object = await resolver.resolve(ps.uri, ps.allowAnonymous ?? false);
if (ps.expandCollectionItems && isCollectionOrOrderedCollection(object)) {
const items = await resolver.resolveCollectionItems(object, ps.expandCollectionLimit, ps.allowAnonymous ?? false);
if (isOrderedCollection(object) || isOrderedCollectionPage(object)) {
object.orderedItems = items;
} else {
object.items = items;
}
}
return object;
});
}

View file

@ -12918,6 +12918,9 @@ export type operations = {
content: {
'application/json': {
uri: string;
expandCollectionItems?: boolean;
expandCollectionLimit?: number | null;
allowAnonymous?: boolean;
};
};
};