merge: maybe laxer match on authority - fixes #815 (!773)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/773

Closes #815

Approved-by: Amber Null <puppygirlhornyposting@gmail.com>
Approved-by: Marie <github@yuugi.dev>
Approved-by: Hazelnoot <acomputerdog@gmail.com>

packages/backend/package.json

@@ -93,6 +93,7 @@
 		"@swc/core": "1.6.6",
 		"@transfem-org/sfm-js": "0.24.5",
 		"@twemoji/parser": "15.1.1",
+		"@types/psl": "^1.1.3",
 		"accepts": "1.3.8",
 		"ajv": "8.17.1",
 		"archiver": "7.0.1",
@@ -135,9 +136,9 @@
 		"json5": "2.2.3",
 		"jsonld": "8.3.2",
 		"jsrsasign": "11.1.0",
+		"juice": "11.0.0",
 		"megalodon": "workspace:*",
 		"meilisearch": "0.42.0",
-		"juice": "11.0.0",
 		"microformats-parser": "2.0.2",
 		"mime-types": "2.1.35",
 		"misskey-js": "workspace:*",
@@ -158,6 +159,7 @@
 		"probe-image-size": "7.2.3",
 		"promise-limit": "2.7.0",
 		"proxy-addr": "^2.0.7",
+		"psl": "^1.13.0",
 		"pug": "3.0.3",
 		"punycode": "2.3.1",
 		"qrcode": "1.5.4",

packages/backend/src/core/UtilityService.ts

@@ -4,9 +4,10 @@
  */
 
 import { URL } from 'node:url';
-import { toASCII } from 'punycode';
+import punycode from 'punycode/punycode.js';
 import { Inject, Injectable } from '@nestjs/common';
 import RE2 from 're2';
+import psl from 'psl';
 import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
 import { bindThis } from '@/decorators.js';
@@ -106,13 +107,13 @@ export class UtilityService {
 
 	@bindThis
 	public toPuny(host: string): string {
-		return toASCII(host.toLowerCase());
+		return punycode.toASCII(host.toLowerCase());
 	}
 
 	@bindThis
 	public toPunyNullable(host: string | null | undefined): string | null {
 		if (host == null) return null;
-		return toASCII(host.toLowerCase());
+		return punycode.toASCII(host.toLowerCase());
 	}
 
 	@bindThis
@@ -122,6 +123,26 @@ export class UtilityService {
 		return host;
 	}
 
+	private specialSuffix(hostname: string): string | null {
+		// masto.host provides domain names for its clients, we have to
+		// treat it as if it were a public suffix
+		const mastoHost = hostname.match(/\.?([a-zA-Z0-9-]+\.masto\.host)$/i);
+		if (mastoHost) {
+			return mastoHost[1];
+		}
+
+		return null;
+	}
+
+	@bindThis
+	public punyHostPSLDomain(url: string): string {
+		const urlObj = new URL(url);
+		const hostname = urlObj.hostname;
+		const domain = this.specialSuffix(hostname) ?? psl.get(hostname) ?? hostname;
+		const host = `${this.toPuny(domain)}${urlObj.port.length > 0 ? ':' + urlObj.port : ''}`;
+		return host;
+	}
+
 	public isFederationAllowedHost(host: string): boolean {
 		if (this.meta.federation === 'none') return false;
 		if (this.meta.federation === 'specified' && !this.meta.federationHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`))) return false;

packages/backend/src/core/activitypub/ApRequestService.ts

@@ -243,7 +243,7 @@ export class ApRequestService {
 				if (alternate) {
 					const href = alternate.getAttribute('href');
 					if (href) {
-						if (this.utilityService.punyHost(url) === this.utilityService.punyHost(href)) {
+						if (this.utilityService.punyHostPSLDomain(url) === this.utilityService.punyHostPSLDomain(href)) {
 							return await this.signedGet(href, user, false);
 						}
 					}

packages/backend/src/core/activitypub/ApResolverService.ts

@@ -131,7 +131,7 @@ export class Resolver {
 			throw new UnrecoverableError(`invalid AP object ${value}: missing id`);
 		}
 
-		if (this.utilityService.punyHost(object.id) !== this.utilityService.punyHost(value)) {
+		if (this.utilityService.punyHostPSLDomain(object.id) !== this.utilityService.punyHostPSLDomain(value)) {
 			throw new UnrecoverableError(`invalid AP object ${value}: id ${object.id} has different host`);
 		}
 

packages/backend/src/core/activitypub/models/ApNoteService.ts

@@ -192,8 +192,8 @@ export class ApNoteService {
 				throw new UnrecoverableError(`unexpected schema of note.url ${url} in ${entryUri}`);
 			}
 
-			if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(note.id)) {
+			if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(note.id)) {
-				throw new Error(`note url <> uri host mismatch: ${url} <> ${note.id} in ${entryUri}`);
+				throw new UnrecoverableError(`note url <> uri host mismatch: ${url} <> ${note.id} in ${entryUri}`);
 			}
 		}
 
@@ -444,7 +444,7 @@ export class ApNoteService {
 				throw new UnrecoverableError(`unexpected schema of note.url ${url} in ${noteUri}`);
 			}
 
-			if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(note.id)) {
+			if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(note.id)) {
 				throw new UnrecoverableError(`note url <> id host mismatch: ${url} <> ${note.id} in ${noteUri}`);
 			}
 		}

packages/backend/src/core/activitypub/models/ApPersonService.ts

@@ -138,7 +138,7 @@ export class ApPersonService implements OnModuleInit {
 	 */
 	@bindThis
 	private validateActor(x: IObject, uri: string): IActor {
-		const expectHost = this.utilityService.punyHost(uri);
+		const expectHost = this.utilityService.punyHostPSLDomain(uri);
 
 		if (!isActor(x)) {
 			throw new UnrecoverableError(`invalid Actor type '${x.type}' in ${uri}`);
@@ -152,7 +152,7 @@ export class ApPersonService implements OnModuleInit {
 			throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox type`);
 		}
 
-		const inboxHost = this.utilityService.punyHost(x.inbox);
+		const inboxHost = this.utilityService.punyHostPSLDomain(x.inbox);
 		if (inboxHost !== expectHost) {
 			throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox ${inboxHost}`);
 		}
@@ -160,7 +160,7 @@ export class ApPersonService implements OnModuleInit {
 		const sharedInboxObject = x.sharedInbox ?? (x.endpoints ? x.endpoints.sharedInbox : undefined);
 		if (sharedInboxObject != null) {
 			const sharedInbox = getApId(sharedInboxObject);
-			if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHost(sharedInbox) === expectHost)) {
+			if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHostPSLDomain(sharedInbox) === expectHost)) {
 				throw new UnrecoverableError(`invalid Actor ${uri} - wrong shared inbox ${sharedInbox}`);
 			}
 		}
@@ -170,7 +170,7 @@ export class ApPersonService implements OnModuleInit {
 			if (xCollection != null) {
 				const collectionUri = getApId(xCollection);
 				if (typeof collectionUri === 'string' && collectionUri.length > 0) {
-					if (this.utilityService.punyHost(collectionUri) !== expectHost) {
+					if (this.utilityService.punyHostPSLDomain(collectionUri) !== expectHost) {
 						throw new UnrecoverableError(`invalid Actor ${uri} - wrong ${collection} ${collectionUri}`);
 					}
 				} else if (collectionUri != null) {
@@ -202,7 +202,7 @@ export class ApPersonService implements OnModuleInit {
 			x.summary = truncate(x.summary, summaryLength);
 		}
 
-		const idHost = this.utilityService.punyHost(x.id);
+		const idHost = this.utilityService.punyHostPSLDomain(x.id);
 		if (idHost !== expectHost) {
 			throw new UnrecoverableError(`invalid Actor ${uri} - wrong id ${x.id}`);
 		}
@@ -212,7 +212,7 @@ export class ApPersonService implements OnModuleInit {
 				throw new UnrecoverableError(`invalid Actor ${uri} - wrong publicKey.id type`);
 			}
 
-			const publicKeyIdHost = this.utilityService.punyHost(x.publicKey.id);
+			const publicKeyIdHost = this.utilityService.punyHostPSLDomain(x.publicKey.id);
 			if (publicKeyIdHost !== expectHost) {
 				throw new UnrecoverableError(`invalid Actor ${uri} - wrong publicKey.id ${x.publicKey.id}`);
 			}
@@ -351,7 +351,7 @@ export class ApPersonService implements OnModuleInit {
 				throw new UnrecoverableError(`unexpected schema of person url ${url} in ${uri}`);
 			}
 
-			if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(person.id)) {
+			if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(person.id)) {
 				throw new UnrecoverableError(`person url <> uri host mismatch: ${url} <> ${person.id} in ${uri}`);
 			}
 		}
@@ -563,7 +563,7 @@ export class ApPersonService implements OnModuleInit {
 				throw new UnrecoverableError(`unexpected schema of person url ${url} in ${uri}`);
 			}
 
-			if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(person.id)) {
+			if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(person.id)) {
 				throw new UnrecoverableError(`person url <> uri host mismatch: ${url} <> ${person.id} in ${uri}`);
 			}
 		}

packages/backend/test/unit/UtilityService.ts

@@ -0,0 +1,43 @@
+import * as assert from 'assert';
+import { Test } from '@nestjs/testing';
+
+import { CoreModule } from '@/core/CoreModule.js';
+import { UtilityService } from '@/core/UtilityService.js';
+import { GlobalModule } from '@/GlobalModule.js';
+
+describe('UtilityService', () => {
+	let utilityService: UtilityService;
+
+	beforeAll(async () => {
+		const app = await Test.createTestingModule({
+			imports: [GlobalModule, CoreModule],
+		}).compile();
+		utilityService = app.get<UtilityService>(UtilityService);
+	});
+
+	describe('punyHost', () => {
+		test('simple', () => {
+			assert.equal(utilityService.punyHost('http://www.foo.com'), 'www.foo.com');
+		});
+		test('japanese', () => {
+			assert.equal(utilityService.punyHost('http://www.新聞.com'), 'www.xn--efvv70d.com');
+		});
+	});
+
+	describe('punyHostPSLDomain', () => {
+		test('simple', () => {
+			assert.equal(utilityService.punyHostPSLDomain('http://www.foo.com'), 'foo.com');
+		});
+		test('japanese', () => {
+			assert.equal(utilityService.punyHostPSLDomain('http://www.新聞.com'), 'xn--efvv70d.com');
+		});
+		test('lower', () => {
+			assert.equal(utilityService.punyHostPSLDomain('http://foo.github.io'), 'foo.github.io');
+			assert.equal(utilityService.punyHostPSLDomain('http://foo.bar.github.io'), 'bar.github.io');
+		});
+		test('special', () => {
+			assert.equal(utilityService.punyHostPSLDomain('http://foo.masto.host'), 'foo.masto.host');
+			assert.equal(utilityService.punyHostPSLDomain('http://foo.bar.masto.host'), 'bar.masto.host');
+		});
+	});
+});

packages/backend/test/utils.ts

@@ -612,8 +612,8 @@ export async function initTestDb(justBorrow = false, initEntities?: any[]) {
 		username: config.db.user,
 		password: config.db.pass,
 		database: config.db.db,
-		synchronize: true && !justBorrow,
+		synchronize: !justBorrow,
-		dropSchema: true && !justBorrow,
+		dropSchema: !justBorrow,
 		entities: initEntities ?? entities,
 	});
 

pnpm-lock.yaml

@@ -170,6 +170,9 @@ importers:
       '@twemoji/parser':
         specifier: 15.1.1
         version: 15.1.1
+      '@types/psl':
+        specifier: ^1.1.3
+        version: 1.1.3
       accepts:
         specifier: 1.3.8
         version: 1.3.8
@@ -365,6 +368,9 @@ importers:
       proxy-addr:
         specifier: ^2.0.7
         version: 2.0.7
+      psl:
+        specifier: ^1.13.0
+        version: 1.13.0
       pug:
         specifier: 3.0.3
         version: 3.0.3
@@ -4835,6 +4841,9 @@ packages:
   '@types/proxy-addr@2.0.3':
     resolution: {integrity: sha512-TgAHHO4tNG3HgLTUhB+hM4iwW6JUNeQHCLnF1DjaDA9c69PN+IasoFu2MYDhubFc+ZIw5c5t9DMtjvrD6R3Egg==}
 
+  '@types/psl@1.1.3':
+    resolution: {integrity: sha512-Iu174JHfLd7i/XkXY6VDrqSlPvTDQOtQI7wNAXKKOAADJ9TduRLkNdMgjGiMxSttUIZnomv81JAbAbC0DhggxA==}
+
   '@types/pug@2.0.10':
     resolution: {integrity: sha512-Sk/uYFOBAB7mb74XcpizmH0KOR2Pv3D2Hmrh1Dmy5BmK3MpdSa5kqZcg6EKBdklU0bFXX9gCfzvpnyUehrPIuA==}
 
@@ -5307,6 +5316,7 @@ packages:
 
   acorn-import-assertions@1.9.0:
     resolution: {integrity: sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==}
+    deprecated: package has been renamed to acorn-import-attributes
     peerDependencies:
       acorn: ^8
 
@@ -9736,8 +9746,8 @@ packages:
   pseudomap@1.0.2:
     resolution: {integrity: sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==}
 
-  psl@1.9.0:
+  psl@1.13.0:
-    resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==}
+    resolution: {integrity: sha512-BFwmFXiJoFqlUpZ5Qssolv15DMyc84gTBds1BjsV1BfXEo1UyyD7GsmN67n7J77uRhoSNW1AXtXKPLcBFQn9Aw==}
 
   pstree.remy@1.1.8:
     resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==}
@@ -13103,7 +13113,7 @@ snapshots:
   '@eslint/config-array@0.17.1':
     dependencies:
       '@eslint/object-schema': 2.1.4
-      debug: 4.3.5(supports-color@8.1.1)
+      debug: 4.3.7
       minimatch: 3.1.2
     transitivePeerDependencies:
       - supports-color
@@ -13125,7 +13135,7 @@ snapshots:
   '@eslint/eslintrc@3.1.0':
     dependencies:
       ajv: 6.12.6
-      debug: 4.3.5(supports-color@8.1.1)
+      debug: 4.3.7
       espree: 10.1.0
       globals: 14.0.0
       ignore: 5.3.1
@@ -15648,6 +15658,8 @@ snapshots:
     dependencies:
       '@types/node': 20.14.12
 
+  '@types/psl@1.1.3': {}
+
   '@types/pug@2.0.10': {}
 
   '@types/punycode@2.1.4': {}
@@ -18435,7 +18447,7 @@ snapshots:
       ajv: 6.12.6
       chalk: 4.1.2
       cross-spawn: 7.0.3
-      debug: 4.3.5(supports-color@8.1.1)
+      debug: 4.3.7
       escape-string-regexp: 4.0.0
       eslint-scope: 8.0.2
       eslint-visitor-keys: 4.0.0
@@ -22014,7 +22026,9 @@ snapshots:
 
   pseudomap@1.0.2: {}
 
-  psl@1.9.0: {}
+  psl@1.13.0:
+    dependencies:
+      punycode: 2.3.1
 
   pstree.remy@1.1.8: {}
 
@@ -23256,7 +23270,7 @@ snapshots:
 
   tough-cookie@4.1.4:
     dependencies:
-      psl: 1.9.0
+      psl: 1.13.0
       punycode: 2.3.1
       universalify: 0.2.0
       url-parse: 1.5.10