From b44abe0eaaeb85111d94046b182e03e67993a101 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 13 Jun 2025 23:18:06 -0400 Subject: [PATCH] set X-Robots-Tag to disable indexing API endpoints --- packages/backend/src/server/ActivityPubServerService.ts | 4 ++++ packages/backend/src/server/FileServerService.ts | 4 ++++ packages/backend/src/server/api/ApiCallService.ts | 4 ++++ .../src/server/api/mastodon/MastodonApiServerService.ts | 7 +++++++ packages/backend/src/server/web/UrlPreviewService.ts | 4 ++++ 5 files changed, 23 insertions(+) diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index 41beadb56d..a362308b17 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -791,6 +791,10 @@ export class ActivityPubServerService { reply.header('Access-Control-Allow-Origin', '*'); reply.header('Access-Control-Expose-Headers', 'Vary'); + // Tell crawlers not to index AP endpoints. + // https://developers.google.com/search/docs/crawling-indexing/block-indexing + reply.header('X-Robots-Tag', 'noindex'); + /* tell any caching proxy that they should not cache these responses: we wouldn't want the proxy to return a 403 to someone presenting a valid signature, or return a cached diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 1a372cb789..0910c0d36b 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -70,6 +70,10 @@ export class FileServerService { fastify.addHook('onRequest', (request, reply, done) => { reply.header('Content-Security-Policy', 'default-src \'none\'; img-src \'self\'; media-src \'self\'; style-src \'unsafe-inline\''); reply.header('Access-Control-Allow-Origin', '*'); + + // Tell crawlers not to index files endpoints. + // https://developers.google.com/search/docs/crawling-indexing/block-indexing + reply.header('X-Robots-Tag', 'noindex'); done(); }); diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index 6d6c86bb82..66d968224a 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -148,6 +148,10 @@ export class ApiCallService implements OnApplicationShutdown { request: FastifyRequest<{ Body: Record | undefined, Querystring: Record }>, reply: FastifyReply, ): void { + // Tell crawlers not to index API endpoints. + // https://developers.google.com/search/docs/crawling-indexing/block-indexing + reply.header('X-Robots-Tag', 'noindex'); + const body = request.method === 'GET' ? request.query : request.body; diff --git a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts index 74fd9d7d59..072dacf708 100644 --- a/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts +++ b/packages/backend/src/server/api/mastodon/MastodonApiServerService.ts @@ -71,6 +71,13 @@ export class MastodonApiServerService { done(); }); + // Tell crawlers not to index API endpoints. + // https://developers.google.com/search/docs/crawling-indexing/block-indexing + fastify.addHook('onRequest', (request, reply, done) => { + reply.header('X-Robots-Tag', 'noindex'); + done(); + }); + // External endpoints this.apiAccountMastodon.register(fastify); this.apiAppsMastodon.register(fastify); diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index ed5d87d15d..71a142fc6f 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -125,6 +125,10 @@ export class UrlPreviewService { reply: FastifyReply, ): Promise { if (!this.meta.urlPreviewEnabled) { + // Tell crawlers not to index URL previews. + // https://developers.google.com/search/docs/crawling-indexing/block-indexing + reply.header('X-Robots-Tag', 'noindex'); + return reply.code(403).send({ error: { message: 'URL preview is disabled',