merge: Adding robots.txt override via admin control panel for #418 (!877)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/877

Approved-by: dakkar <dakkar@thenautilus.net>
Approved-by: Hazelnoot <acomputerdog@gmail.com>

locales/index.d.ts

@@ -11626,6 +11626,14 @@ export interface Locale extends ILocale {
      * Scheduled Notes
      */
     "scheduledNotes": string;
+    /**
+     * Custom robots.txt
+     */
+    "robotsTxt": string;
+    /**
+     * Adding entries here will override the default robots.txt packaged with Sharkey. Maximum 2048 characters.
+     */
+    "robotsTxtDescription": string;
 }
 declare const locales: {
     [lang: string]: Locale;

packages/backend/migration/1738346484187-robotsTxt.js

@@ -0,0 +1,16 @@
+/*
+ * SPDX-FileCopyrightText: marie and other Sharkey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+export class RobotsTxt1738346484187 {
+    name = 'RobotsTxt1738346484187'
+
+    async up(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" ADD "robotsTxt" text`);
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "robotsTxt"`);
+    }
+}

packages/backend/src/core/entities/MetaEntityService.ts

@@ -95,6 +95,7 @@ export class MetaEntityService {
 			mcaptchaInstanceUrl: instance.mcaptchaInstanceUrl,
 			enableRecaptcha: instance.enableRecaptcha,
 			enableAchievements: instance.enableAchievements,
+			robotsTxt: instance.robotsTxt,
 			recaptchaSiteKey: instance.recaptchaSiteKey,
 			enableTurnstile: instance.enableTurnstile,
 			turnstileSiteKey: instance.turnstileSiteKey,

packages/backend/src/models/Meta.ts

@@ -599,6 +599,11 @@ export class MiMeta {
 	})
 	public enableAchievements: boolean;
 
+	@Column('varchar', {
+		length: 2048, nullable: true,
+	})
+	public robotsTxt: string | null;
+
 	@Column('jsonb', {
 		default: { },
 	})

packages/backend/src/models/json-schema/meta.ts

@@ -139,6 +139,10 @@ export const packedMetaLiteSchema = {
 			type: 'boolean',
 			optional: false, nullable: true,
 		},
+		robotsTxt: {
+			type: 'string',
+			optional: false, nullable: true,
+		},
 		enableTestcaptcha: {
 			type: 'boolean',
 			optional: false, nullable: false,

packages/backend/src/server/api/endpoints/admin/meta.ts

@@ -391,6 +391,10 @@ export const meta = {
 				type: 'boolean',
 				optional: false, nullable: false,
 			},
+			robotsTxt: {
+				type: 'string',
+				optional: false, nullable: true,
+			},
 			enableIdenticonGeneration: {
 				type: 'boolean',
 				optional: false, nullable: false,
@@ -708,6 +712,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				enableStatsForFederatedInstances: instance.enableStatsForFederatedInstances,
 				enableServerMachineStats: instance.enableServerMachineStats,
 				enableAchievements: instance.enableAchievements,
+				robotsTxt: instance.robotsTxt,
 				enableIdenticonGeneration: instance.enableIdenticonGeneration,
 				bannedEmailDomains: instance.bannedEmailDomains,
 				policies: { ...DEFAULT_POLICIES, ...instance.policies },

packages/backend/src/server/api/endpoints/admin/update-meta.ts

@@ -149,6 +149,7 @@ export const paramDef = {
 		enableStatsForFederatedInstances: { type: 'boolean' },
 		enableServerMachineStats: { type: 'boolean' },
 		enableAchievements: { type: 'boolean' },
+		robotsTxt: { type: 'string', nullable: true },
 		enableIdenticonGeneration: { type: 'boolean' },
 		serverRules: { type: 'array', items: { type: 'string' } },
 		bannedEmailDomains: { type: 'array', items: { type: 'string' } },
@@ -636,6 +637,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				set.enableAchievements = ps.enableAchievements;
 			}
 
+			if (ps.robotsTxt !== undefined) {
+				set.robotsTxt = ps.robotsTxt;
+			}
+
 			if (ps.enableIdenticonGeneration !== undefined) {
 				set.enableIdenticonGeneration = ps.enableIdenticonGeneration;
 			}

packages/backend/src/server/web/ClientServerService.ts

@@ -488,7 +488,12 @@ export class ClientServerService {
 		});
 
 		fastify.get('/robots.txt', async (request, reply) => {
-			return await reply.sendFile('/robots.txt', staticAssets);
+			if (this.meta.robotsTxt) {
+				reply.header('Content-Type', 'text/plain');
+				return await reply.send(this.meta.robotsTxt);
+			} else {
+				return await reply.sendFile('/robots.txt', staticAssets);
+			}
 		});
 
 		// OpenSearch XML

packages/frontend/src/pages/admin/settings.vue

@@ -159,6 +159,11 @@ SPDX-License-Identifier: AGPL-3.0-only
 							<template #label>{{ i18n.ts.enableBotTrending }}<span v-if="otherForm.modifiedStates.enableBotTrending" class="_modified">{{ i18n.ts.modified }}</span></template>
 							<template #caption>{{ i18n.ts.turnOffBotTrending }}</template>
 						</MkSwitch>
+
+						<MkTextarea v-model="otherForm.state.robotsTxt">
+							<template #label>{{ i18n.ts.robotsTxt }}<span v-if="otherForm.modifiedStates.robotsTxt" class="_modified">{{ i18n.ts.modified }}</span></template>
+							<template #caption>{{ i18n.ts.robotsTxtDescription }}</template>
+						</MkTextarea>
 					</div>
 				</MkFolder>
 
@@ -369,10 +374,12 @@ const serviceWorkerForm = useForm({
 const otherForm = useForm({
 	enableAchievements: meta.enableAchievements,
 	enableBotTrending: meta.enableBotTrending,
+	robotsTxt: meta.robotsTxt,
 }, async (state) => {
 	await os.apiWithDialog('admin/update-meta', {
 		enableAchievements: state.enableAchievements,
 		enableBotTrending: state.enableBotTrending,
+		robotsTxt: state.robotsTxt,
 	});
 	fetchInstance(true);
 });

sharkey-locales/en-US.yml

@@ -434,3 +434,6 @@ scheduledNotes: "Scheduled Notes"
 _permissions:
   "read:notes-schedule": "View your list of scheduled notes"
   "write:notes-schedule": "Compose or delete scheduled notes"
+
+robotsTxt: "Custom robots.txt"
+robotsTxtDescription: "Adding entries here will override the default robots.txt packaged with Sharkey."