merge: Add BunnyCDN Edge Storage support (!952)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/952

Closes #1020

Approved-by: Hazelnoot <acomputerdog@gmail.com>
Approved-by: dakkar <dakkar@thenautilus.net>

packages/backend/src/core/BunnyService.ts

@@ -0,0 +1,102 @@
+/*
+ * SPDX-FileCopyrightText: marie and sharkey-project
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+import * as https from 'node:https';
+import * as fs from 'node:fs';
+import { Readable } from 'node:stream';
+import { finished } from 'node:stream/promises';
+import { Injectable } from '@nestjs/common';
+import type { MiMeta } from '@/models/Meta.js';
+import { HttpRequestService } from '@/core/HttpRequestService.js';
+import { bindThis } from '@/decorators.js';
+import { IdentifiableError } from '@/misc/identifiable-error.js';
+import Logger from '@/logger.js';
+
+@Injectable()
+export class BunnyService {
+	private bunnyCdnLogger: Logger;
+
+	constructor(
+		private httpRequestService: HttpRequestService,
+	) {
+		this.bunnyCdnLogger = new Logger('bunnycdn', 'blue');
+	}
+
+	@bindThis
+	public getBunnyInfo(meta: MiMeta) {
+		if (!meta.objectStorageEndpoint || !meta.objectStorageBucket || !meta.objectStorageSecretKey) {
+			throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140bf90', 'Failed to use BunnyCDN, One of the required fields is missing.');
+		}
+
+		return {
+			endpoint: meta.objectStorageEndpoint,
+			/*
+			   The way S3 works is that the Secret Key is essentially the password for the API but Bunny calls their password AccessKey so we call it accessKey here.
+			   Bunny also doesn't specify a username/s3 access key when doing HTTP API requests so we end up not using our Access Key field from the form.
+			*/
+			accessKey: meta.objectStorageSecretKey,
+			zone: meta.objectStorageBucket,
+			fullUrl: `https://${meta.objectStorageEndpoint}/${meta.objectStorageBucket}`,
+		};
+	}
+
+	@bindThis
+	public usingBunnyCDN(meta: MiMeta) {
+		const client = this.getBunnyInfo(meta);
+		return new URL(client.fullUrl).hostname.endsWith('bunnycdn.com');
+	}
+
+	@bindThis
+	public async upload(meta: MiMeta, path: string, input: fs.ReadStream | Buffer) {
+		const client = this.getBunnyInfo(meta);
+
+		// Required to convert the buffer from webpublic and thumbnail to a ReadableStream for PUT
+		const data = Buffer.isBuffer(input) ? Readable.from(input) : input;
+
+		const agent = this.httpRequestService.getAgentByUrl(new URL(`${client.fullUrl}/${path}`), !meta.objectStorageUseProxy, true);
+
+		// Seperation of path and host/domain is required here
+		const options = {
+			method: 'PUT',
+			host: client.endpoint,
+			path: `/${client.zone}/${path}`,
+			headers: {
+				AccessKey: client.accessKey,
+				'Content-Type': 'application/octet-stream',
+			},
+			agent: agent,
+		};
+
+		const req = https.request(options);
+
+		// Log and return if BunnyCDN detects wrong data (return is used to prevent console spam as this event occurs multiple times)
+		req.on('response', (res) => {
+			if (res.statusCode === 401) {
+				this.bunnyCdnLogger.error('Invalid AccessKey or region hostname');
+				data.destroy();
+				return;
+			}
+		});
+
+		req.on('error', (error) => {
+			this.bunnyCdnLogger.error(error);
+			data.destroy();
+			throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140bf91', 'An error has occured during the connectiong to BunnyCDN');
+		});
+
+		data.pipe(req).on('finish', () => {
+			data.destroy();
+		});
+		
+		// wait till stream gets destroyed upon finish of piping to prevent the UI from showing the upload as success way too early
+		await finished(data);
+	}
+
+	@bindThis
+	public delete(meta: MiMeta, file: string) {
+		const client = this.getBunnyInfo(meta);
+		return this.httpRequestService.send(`${client.fullUrl}/${file}`, { method: 'DELETE', headers: { AccessKey: client.accessKey } });
+	}
+}

packages/backend/src/core/CoreModule.ts

@@ -61,6 +61,7 @@ import { ReactionsBufferingService } from './ReactionsBufferingService.js';
 import { RelayService } from './RelayService.js';
 import { RoleService } from './RoleService.js';
 import { S3Service } from './S3Service.js';
+import { BunnyService } from './BunnyService.js';
 import { SignupService } from './SignupService.js';
 import { WebAuthnService } from './WebAuthnService.js';
 import { UserBlockingService } from './UserBlockingService.js';
@@ -208,6 +209,7 @@ const $ReactionsBufferingService: Provider = { provide: 'ReactionsBufferingServi
 const $RelayService: Provider = { provide: 'RelayService', useExisting: RelayService };
 const $RoleService: Provider = { provide: 'RoleService', useExisting: RoleService };
 const $S3Service: Provider = { provide: 'S3Service', useExisting: S3Service };
+const $BunnyService: Provider = { provide: 'BunnyService', useExisting: BunnyService };
 const $SignupService: Provider = { provide: 'SignupService', useExisting: SignupService };
 const $WebAuthnService: Provider = { provide: 'WebAuthnService', useExisting: WebAuthnService };
 const $UserBlockingService: Provider = { provide: 'UserBlockingService', useExisting: UserBlockingService };
@@ -367,6 +369,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
 		RelayService,
 		RoleService,
 		S3Service,
+		BunnyService,
 		SignupService,
 		WebAuthnService,
 		UserBlockingService,
@@ -522,6 +525,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
 		$RelayService,
 		$RoleService,
 		$S3Service,
+		$BunnyService,
 		$SignupService,
 		$WebAuthnService,
 		$UserBlockingService,
@@ -678,6 +682,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
 		RelayService,
 		RoleService,
 		S3Service,
+		BunnyService,
 		SignupService,
 		WebAuthnService,
 		UserBlockingService,
@@ -832,6 +837,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp
 		$RelayService,
 		$RoleService,
 		$S3Service,
+		$BunnyService,
 		$SignupService,
 		$WebAuthnService,
 		$UserBlockingService,

packages/backend/src/core/DriveService.ts

@@ -44,6 +44,7 @@ import { correctFilename } from '@/misc/correct-filename.js';
 import { isMimeImage } from '@/misc/is-mime-image.js';
 import { ModerationLogService } from '@/core/ModerationLogService.js';
 import { UtilityService } from '@/core/UtilityService.js';
+import { BunnyService } from '@/core/BunnyService.js';
 
 type AddFileArgs = {
 	/** User who wish to add file */
@@ -121,6 +122,7 @@ export class DriveService {
 		private downloadService: DownloadService,
 		private internalStorageService: InternalStorageService,
 		private s3Service: S3Service,
+		private bunnyService: BunnyService,
 		private imageProcessingService: ImageProcessingService,
 		private videoProcessingService: VideoProcessingService,
 		private globalEventService: GlobalEventService,
@@ -405,20 +407,28 @@ export class DriveService {
 		);
 		if (this.meta.objectStorageSetPublicRead) params.ACL = 'public-read';
 
-		await this.s3Service.upload(this.meta, params)
-			.then(
-				result => {
-					if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput
-						this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
-					} else { // AbortMultipartUploadCommandOutput
-						this.registerLogger.error(`Upload Result Aborted: key = ${key}, filename = ${filename}`);
-					}
-				})
-			.catch(
+		if (this.bunnyService.usingBunnyCDN(this.meta)) {
+			await this.bunnyService.upload(this.meta, key, stream).catch(
 				err => {
 					this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err);
 				},
 			);
+		} else {
+			await this.s3Service.upload(this.meta, params)
+				.then(
+					result => {
+						if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput
+							this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
+						} else { // AbortMultipartUploadCommandOutput
+							this.registerLogger.error(`Upload Result Aborted: key = ${key}, filename = ${filename}`);
+						}
+					})
+				.catch(
+					err => {
+						this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err);
+					},
+				);
+		}
 	}
 
 	// Expire oldest file (without avatar or banner) of remote user
@@ -814,8 +824,11 @@ export class DriveService {
 				Bucket: this.meta.objectStorageBucket,
 				Key: key,
 			} as DeleteObjectCommandInput;
-
-			await this.s3Service.delete(this.meta, param);
+			if (this.bunnyService.usingBunnyCDN(this.meta)) {
+				await this.bunnyService.delete(this.meta, key);
+			} else {
+				await this.s3Service.delete(this.meta, param);
+			}
 		} catch (err: any) {
 			if (err.name === 'NoSuchKey') {
 				this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error);