mirrors/hedgedoc
1
0
Fork 0
mirror of https://github.com/hedgedoc/hedgedoc.git synced 2025-05-22 11:15:23 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

refactor(media): store filenames, use pre-signed s3/azure URLs, UUIDs

Browse source 
Signed-off-by: Erik Michelson <github@erik.michelson.eu>
This commit is contained in:
Erik Michelson 2024-06-12 18:45:49 +02:00 • committed by Philip Molares
parent 4132833b5d
commit 157a0fe278
47 changed files with 869 additions and 389 deletions
Download patch file Download diff file Expand all files Collapse all files

33
backend/src/api/private/media/media.controller.ts
View file

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2022 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
@ -100,28 +100,33 @@ export class MediaController {
'uploadMedia',
);
}
const upload = await this.mediaService.saveFile(file.buffer, user, note);
const upload = await this.mediaService.saveFile(
file.originalname,
file.buffer,
user,
note,
);
return await this.mediaService.toMediaUploadDto(upload);
}
@Get(':filename')
@OpenApi(404, 500)
@Get(':uuid')
@OpenApi(200, 404, 500)
async getMedia(
@Param('filename') filename: string,
@Param('uuid') uuid: string,
@Res() response: Response,
): Promise<void> {
const mediaUpload = await this.mediaService.findUploadByFilename(filename);
const targetUrl = mediaUpload.fileUrl;
response.redirect(targetUrl);
const mediaUpload = await this.mediaService.findUploadByUuid(uuid);
const dto = await this.mediaService.toMediaUploadDto(mediaUpload);
response.send(dto);
}
@Delete(':filename')
@Delete(':uuid')
@OpenApi(204, 403, 404, 500)
async deleteMedia(
@RequestUser() user: User,
@Param('filename') filename: string,
@Param('uuid') uuid: string,
): Promise<void> {
const mediaUpload = await this.mediaService.findUploadByFilename(filename);
const mediaUpload = await this.mediaService.findUploadByUuid(uuid);
if (
await this.permissionsService.checkMediaDeletePermission(
user,
@ -129,18 +134,18 @@ export class MediaController {
)
) {
this.logger.debug(
`Deleting '${filename}' for user '${user.username}'`,
`Deleting '${uuid}' for user '${user.username}'`,
'deleteMedia',
);
await this.mediaService.deleteFile(mediaUpload);
} else {
this.logger.warn(
`${user.username} tried to delete '${filename}', but is not the owner of upload or connected note`,
`${user.username} tried to delete '${uuid}', but is not the owner of upload or connected note`,
'deleteMedia',
);
const mediaUploadNote = await mediaUpload.note;
throw new PermissionError(
`Neither file '${filename}' nor note '${
`Neither file '${uuid}' nor note '${
mediaUploadNote?.publicId ?? 'unknown'
}'is owned by '${user.username}'`,
);

33
backend/src/api/public/media/media.controller.ts
View file

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2022 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
@ -100,28 +100,33 @@ export class MediaController {
`Received filename '${file.originalname}' for note '${note.publicId}' from user '${user.username}'`,
'uploadMedia',
);
const upload = await this.mediaService.saveFile(file.buffer, user, note);
const upload = await this.mediaService.saveFile(
file.originalname,
file.buffer,
user,
note,
);
return await this.mediaService.toMediaUploadDto(upload);
}
@Get(':filename')
@OpenApi(404, 500)
@Get(':uuid')
@OpenApi(200, 404, 500)
async getMedia(
@Param('filename') filename: string,
@Param('uuid') uuid: string,
@Res() response: Response,
): Promise<void> {
const mediaUpload = await this.mediaService.findUploadByFilename(filename);
const targetUrl = mediaUpload.fileUrl;
response.redirect(targetUrl);
const mediaUpload = await this.mediaService.findUploadByUuid(uuid);
const dto = await this.mediaService.toMediaUploadDto(mediaUpload);
response.send(dto);
}
@Delete(':filename')
@Delete(':uuid')
@OpenApi(204, 403, 404, 500)
async deleteMedia(
@RequestUser() user: User,
@Param('filename') filename: string,
@Param('uuid') uuid: string,
): Promise<void> {
const mediaUpload = await this.mediaService.findUploadByFilename(filename);
const mediaUpload = await this.mediaService.findUploadByUuid(uuid);
if (
await this.permissionsService.checkMediaDeletePermission(
user,
@ -129,18 +134,18 @@ export class MediaController {
)
) {
this.logger.debug(
`Deleting '${filename}' for user '${user.username}'`,
`Deleting '${uuid}' for user '${user.username}'`,
'deleteMedia',
);
await this.mediaService.deleteFile(mediaUpload);
} else {
this.logger.warn(
`${user.username} tried to delete '${filename}', but is not the owner of upload or connected note`,
`${user.username} tried to delete '${uuid}', but is not the owner of upload or connected note`,
'deleteMedia',
);
const mediaUploadNote = await mediaUpload.note;
throw new PermissionError(
`Neither file '${filename}' nor note '${
`Neither file '${uuid}' nor note '${
mediaUploadNote?.publicId ?? 'unknown'
}'is owned by '${user.username}'`,
);

4
backend/src/api/utils/descriptions.ts
View file

@ -1,10 +1,12 @@
/*
* SPDX-FileCopyrightText: 2022 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
export const okDescription = 'This request was successful';
export const foundDescription =
'The requested resource was found at another URL';
export const createdDescription =
'The requested resource was successfully created';
export const noContentDescription =

13
backend/src/api/utils/openapi.decorator.ts
View file

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2022 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
@ -8,6 +8,7 @@ import {
ApiBadRequestResponse,
ApiConflictResponse,
ApiCreatedResponse,
ApiFoundResponse,
ApiInternalServerErrorResponse,
ApiNoContentResponse,
ApiNotFoundResponse,
@ -21,6 +22,7 @@ import {
badRequestDescription,
conflictDescription,
createdDescription,
foundDescription,
internalServerErrorDescription,
noContentDescription,
notFoundDescription,
@ -33,6 +35,7 @@ export type HttpStatusCodes =
| 200
| 201
| 204
| 302
| 400
| 401
| 403
@ -130,6 +133,14 @@ export const OpenApi = (
HttpCode(204),
);
break;
case 302:
decoratorsToApply.push(
ApiFoundResponse({
description: description ?? foundDescription,
}),
HttpCode(302),
);
break;
case 400:
decoratorsToApply.push(
ApiBadRequestResponse({

8
backend/src/app.module.ts
View file

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2022 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
@ -29,6 +29,7 @@ import { HistoryModule } from './history/history.module';
import { IdentityModule } from './identity/identity.module';
import { LoggerModule } from './logger/logger.module';
import { TypeormLoggerService } from './logger/typeorm-logger.service';
import { MediaRedirectModule } from './media-redirect/media-redirect.module';
import { MediaModule } from './media/media.module';
import { MonitoringModule } from './monitoring/monitoring.module';
import { NotesModule } from './notes/notes.module';
@ -49,6 +50,10 @@ const routes: Routes = [
path: '/api/private',
module: PrivateApiModule,
},
{
path: '/media',
module: MediaRedirectModule,
},
];
@Module({
@ -112,6 +117,7 @@ const routes: Routes = [
WebsocketModule,
IdentityModule,
SessionModule,
MediaRedirectModule,
],
controllers: [],
providers: [FrontendConfigService],

8
backend/src/config/media.config.spec.ts
View file

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2022 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
@ -16,6 +16,8 @@ describe('mediaConfig', () => {
const secretAccessKey = 'secretAccessKey';
const bucket = 'bucket';
const endPoint = 'https://endPoint';
const region = 'us-east-1';
const pathStyle = false;
// Azure
const azureConnectionString = 'connectionString';
const container = 'container';
@ -54,6 +56,8 @@ describe('mediaConfig', () => {
HD_MEDIA_BACKEND_S3_SECRET_KEY: secretAccessKey,
HD_MEDIA_BACKEND_S3_BUCKET: bucket,
HD_MEDIA_BACKEND_S3_ENDPOINT: endPoint,
HD_MEDIA_BACKEND_S3_REGION: region,
HD_MEDIA_BACKEND_S3_PATH_STYLE: pathStyle.toString(),
/* eslint-enable @typescript-eslint/naming-convention */
},
{
@ -66,6 +70,8 @@ describe('mediaConfig', () => {
expect(config.backend.s3.secretAccessKey).toEqual(secretAccessKey);
expect(config.backend.s3.bucket).toEqual(bucket);
expect(config.backend.s3.endPoint).toEqual(endPoint);
expect(config.backend.s3.region).toEqual(region);
expect(config.backend.s3.pathStyle).toEqual(pathStyle);
restore();
});

14
backend/src/config/media.config.ts
View file

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2022 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
@ -7,7 +7,7 @@ import { registerAs } from '@nestjs/config';
import * as Joi from 'joi';
import { BackendType } from '../media/backends/backend-type.enum';
import { buildErrorMessage } from './utils';
import { buildErrorMessage, parseOptionalBoolean } from './utils';
export interface MediaConfig {
backend: MediaBackendConfig;
@ -23,6 +23,8 @@ export interface MediaBackendConfig {
secretAccessKey: string;
bucket: string;
endPoint: string;
region: string;
pathStyle: boolean;
};
azure: {
connectionString: string;
@ -59,6 +61,10 @@ const mediaSchema = Joi.object({
endPoint: Joi.string()
.uri({ scheme: /^https?/ })
.label('HD_MEDIA_BACKEND_S3_ENDPOINT'),
region: Joi.string().optional().label('HD_MEDIA_BACKEND_S3_REGION'),
pathStyle: Joi.boolean()
.default(false)
.label('HD_MEDIA_BACKEND_S3_PATH_STYLE'),
}),
otherwise: Joi.optional(),
}),
@ -110,6 +116,10 @@ export default registerAs('mediaConfig', () => {
secretAccessKey: process.env.HD_MEDIA_BACKEND_S3_SECRET_KEY,
bucket: process.env.HD_MEDIA_BACKEND_S3_BUCKET,
endPoint: process.env.HD_MEDIA_BACKEND_S3_ENDPOINT,
region: process.env.HD_MEDIA_BACKEND_S3_REGION,
pathStyle: parseOptionalBoolean(
process.env.HD_MEDIA_BACKEND_S3_PATH_STYLE,
),
},
azure: {
connectionString:

2
backend/src/config/mock/media.config.mock.ts
View file

@ -22,6 +22,8 @@ export function createDefaultMockMediaConfig(): MediaConfig {
secretAccessKey: '',
bucket: '',
endPoint: '',
pathStyle: false,
region: '',
},
azure: {
connectionString: '',

14
backend/src/config/utils.spec.ts
View file

@ -8,6 +8,7 @@ import {
ensureNoDuplicatesExist,
findDuplicatesInArray,
needToLog,
parseOptionalBoolean,
parseOptionalNumber,
replaceAuthErrorsWithEnvironmentVariables,
toArrayConfig,
@ -141,4 +142,17 @@ describe('config utils', () => {
expect(parseOptionalNumber('3.14')).toEqual(3.14);
});
});
describe('parseOptionalBoolean', () => {
it('returns undefined on undefined parameter', () => {
expect(parseOptionalBoolean(undefined)).toEqual(undefined);
});
it('correctly parses a given string', () => {
expect(parseOptionalBoolean('true')).toEqual(true);
expect(parseOptionalBoolean('1')).toEqual(true);
expect(parseOptionalBoolean('y')).toEqual(true);
expect(parseOptionalBoolean('false')).toEqual(false);
expect(parseOptionalBoolean('0')).toEqual(false);
expect(parseOptionalBoolean('HedgeDoc')).toEqual(false);
});
});
});

14
backend/src/config/utils.ts
View file

@ -118,3 +118,17 @@ export function parseOptionalNumber(value?: string): number | undefined {
}
return Number(value);
}
/**
* Parses a string to a boolean. The following values are considered true:
* true, 1, y
*
* @param value The value to parse
* @returns The parsed boolean or undefined if the value is undefined
*/
export function parseOptionalBoolean(value?: string): boolean | undefined {
if (value === undefined) {
return undefined;
}
return value === 'true' || value === '1' || value === 'y';
}

35
backend/src/media-redirect/media-redirect.controller.ts Normal file
View file

@ -0,0 +1,35 @@
/*
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Controller, Get, Param, Res } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { Response } from 'express';
import { OpenApi } from '../api/utils/openapi.decorator';
import { ConsoleLoggerService } from '../logger/console-logger.service';
import { MediaService } from '../media/media.service';
@OpenApi()
@ApiTags('media-redirect')
@Controller()
export class MediaRedirectController {
constructor(
private readonly logger: ConsoleLoggerService,
private mediaService: MediaService,
) {
this.logger.setContext(MediaRedirectController.name);
}
@Get(':uuid')
@OpenApi(302, 404, 500)
async getMedia(
@Param('uuid') uuid: string,
@Res() response: Response,
): Promise<void> {
const mediaUpload = await this.mediaService.findUploadByUuid(uuid);
const url = await this.mediaService.getFileUrl(mediaUpload);
response.redirect(url);
}
}

16
backend/src/media-redirect/media-redirect.module.ts Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Module } from '@nestjs/common';
import { LoggerModule } from '../logger/logger.module';
import { MediaModule } from '../media/media.module';
import { MediaRedirectController } from './media-redirect.controller';
@Module({
imports: [MediaModule, LoggerModule],
controllers: [MediaRedirectController],
})
export class MediaRedirectModule {}

66
backend/src/media/backends/azure-backend.ts
View file

@ -1,26 +1,30 @@
/*
* SPDX-FileCopyrightText: 2021 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
import {
BlobSASPermissions,
BlobServiceClient,
BlockBlobClient,
ContainerClient,
generateBlobSASQueryParameters,
StorageSharedKeyCredential,
} from '@azure/storage-blob';
import { Inject, Injectable } from '@nestjs/common';
import { FileTypeResult } from 'file-type';
import mediaConfiguration, { MediaConfig } from '../../config/media.config';
import { MediaBackendError } from '../../errors/errors';
import { ConsoleLoggerService } from '../../logger/console-logger.service';
import { MediaBackend } from '../media-backend.interface';
import { BackendData } from '../media-upload.entity';
import { BackendType } from './backend-type.enum';
@Injectable()
export class AzureBackend implements MediaBackend {
private config: MediaConfig['backend']['azure'];
private client: ContainerClient;
private readonly credential: StorageSharedKeyCredential;
constructor(
private readonly logger: ConsoleLoggerService,
@ -28,56 +32,76 @@ export class AzureBackend implements MediaBackend {
private mediaConfig: MediaConfig,
) {
this.logger.setContext(AzureBackend.name);
this.config = mediaConfig.backend.azure;
if (mediaConfig.backend.use === BackendType.AZURE) {
this.config = this.mediaConfig.backend.azure;
if (this.mediaConfig.backend.use === BackendType.AZURE) {
// only create the client if the backend is configured to azure
const blobServiceClient = BlobServiceClient.fromConnectionString(
this.config.connectionString,
);
this.credential =
blobServiceClient.credential as StorageSharedKeyCredential;
this.client = blobServiceClient.getContainerClient(this.config.container);
}
}
async saveFile(
uuid: string,
buffer: Buffer,
fileName: string,
): Promise<[string, BackendData]> {
fileType: FileTypeResult,
): Promise<null> {
const blockBlobClient: BlockBlobClient =
this.client.getBlockBlobClient(fileName);
this.client.getBlockBlobClient(uuid);
try {
await blockBlobClient.upload(buffer, buffer.length);
const url = this.getUrl(fileName);
this.logger.log(`Uploaded ${url}`, 'saveFile');
return [url, null];
await blockBlobClient.upload(buffer, buffer.length, {
blobHTTPHeaders: {
blobContentType: fileType.mime,
},
});
this.logger.log(`Uploaded file ${uuid}`, 'saveFile');
return null;
} catch (e) {
this.logger.error(
`error: ${(e as Error).message}`,
(e as Error).stack,
'saveFile',
);
throw new MediaBackendError(`Could not save '${fileName}' on Azure`);
throw new MediaBackendError(`Could not save file '${uuid}'`);
}
}
async deleteFile(fileName: string, _: BackendData): Promise<void> {
async deleteFile(uuid: string, _: unknown): Promise<void> {
const blockBlobClient: BlockBlobClient =
this.client.getBlockBlobClient(fileName);
this.client.getBlockBlobClient(uuid);
try {
await blockBlobClient.delete();
const url = this.getUrl(fileName);
this.logger.log(`Deleted ${url}`, 'deleteFile');
return;
const response = await blockBlobClient.delete();
if (response.errorCode !== undefined) {
throw new MediaBackendError(
`Could not delete '${uuid}': ${response.errorCode}`,
);
}
this.logger.log(`Deleted file ${uuid}`, 'deleteFile');
} catch (e) {
this.logger.error(
`error: ${(e as Error).message}`,
(e as Error).stack,
'deleteFile',
);
throw new MediaBackendError(`Could not delete '${fileName}' on Azure`);
throw new MediaBackendError(`Could not delete file ${uuid}`);
}
}
private getUrl(fileName: string): string {
return `${this.client.url}/${fileName}`;
getFileUrl(uuid: string, _: unknown): Promise<string> {
const blockBlobClient: BlockBlobClient =
this.client.getBlockBlobClient(uuid);
const blobSAS = generateBlobSASQueryParameters(
{
containerName: this.config.container,
blobName: uuid,
permissions: BlobSASPermissions.parse('r'),
expiresOn: new Date(new Date().valueOf() + 3600 * 1000),
},
this.credential,
);
return Promise.resolve(`${blockBlobClient.url}?${blobSAS.toString()}`);
}
}

49
backend/src/media/backends/filesystem-backend.ts
View file

@ -1,9 +1,10 @@
/*
* SPDX-FileCopyrightText: 2021 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable } from '@nestjs/common';
import { FileTypeResult } from 'file-type';
import { promises as fs } from 'fs';
import { join } from 'path';
@ -11,11 +12,10 @@ import mediaConfiguration, { MediaConfig } from '../../config/media.config';
import { MediaBackendError } from '../../errors/errors';
import { ConsoleLoggerService } from '../../logger/console-logger.service';
import { MediaBackend } from '../media-backend.interface';
import { BackendData } from '../media-upload.entity';
@Injectable()
export class FilesystemBackend implements MediaBackend {
uploadDirectory = './uploads';
private readonly uploadDirectory;
constructor(
private readonly logger: ConsoleLoggerService,
@ -23,37 +23,56 @@ export class FilesystemBackend implements MediaBackend {
private mediaConfig: MediaConfig,
) {
this.logger.setContext(FilesystemBackend.name);
this.uploadDirectory = mediaConfig.backend.filesystem.uploadPath;
this.uploadDirectory = this.mediaConfig.backend.filesystem.uploadPath;
}
async saveFile(
uuid: string,
buffer: Buffer,
fileName: string,
): Promise<[string, BackendData]> {
const filePath = this.getFilePath(fileName);
this.logger.debug(`Writing file to: ${filePath}`, 'saveFile');
fileType: FileTypeResult,
): Promise<string> {
const filePath = this.getFilePath(uuid, fileType.ext);
this.logger.debug(`Writing uploaded file to '${filePath}'`, 'saveFile');
await this.ensureDirectory();
try {
await fs.writeFile(filePath, buffer, null);
return ['/uploads/' + fileName, null];
return JSON.stringify({ ext: fileType.ext });
} catch (e) {
this.logger.error((e as Error).message, (e as Error).stack, 'saveFile');
throw new MediaBackendError(`Could not save '${filePath}'`);
throw new MediaBackendError(`Could not save file '${filePath}'`);
}
}
async deleteFile(fileName: string, _: BackendData): Promise<void> {
const filePath = this.getFilePath(fileName);
async deleteFile(uuid: string, backendData: string): Promise<void> {
if (!backendData) {
throw new MediaBackendError('No backend data provided');
}
const { ext } = JSON.parse(backendData) as { ext: string };
if (!ext) {
throw new MediaBackendError('No file extension in backend data');
}
const filePath = this.getFilePath(uuid, ext);
try {
return await fs.unlink(filePath);
} catch (e) {
this.logger.error((e as Error).message, (e as Error).stack, 'deleteFile');
throw new MediaBackendError(`Could not delete '${filePath}'`);
throw new MediaBackendError(`Could not delete file '${filePath}'`);
}
}
private getFilePath(fileName: string): string {
return join(this.uploadDirectory, fileName);
getFileUrl(uuid: string, backendData: string): Promise<string> {
if (!backendData) {
throw new MediaBackendError('No backend data provided');
}
const { ext } = JSON.parse(backendData) as { ext: string };
if (!ext) {
throw new MediaBackendError('No file extension in backend data');
}
return Promise.resolve(`/uploads/${uuid}.${ext}`);
}
private getFilePath(fileName: string, extension: string): string {
return join(this.uploadDirectory, `${fileName}.${extension}`);
}
private async ensureDirectory(): Promise<void> {

56
backend/src/media/backends/imgur-backend.ts
View file

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2021 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
@ -11,15 +11,19 @@ import mediaConfiguration, { MediaConfig } from '../../config/media.config';
import { MediaBackendError } from '../../errors/errors';
import { ConsoleLoggerService } from '../../logger/console-logger.service';
import { MediaBackend } from '../media-backend.interface';
import { BackendData } from '../media-upload.entity';
type UploadResult = {
data: {
link: string;
deletehash: string;
deletehash: string | null;
};
};
interface ImgurBackendData {
url: string;
deleteHash: string | null;
}
@Injectable()
export class ImgurBackend implements MediaBackend {
private config: MediaConfig['backend']['imgur'];
@ -30,13 +34,10 @@ export class ImgurBackend implements MediaBackend {
private mediaConfig: MediaConfig,
) {
this.logger.setContext(ImgurBackend.name);
this.config = mediaConfig.backend.imgur;
this.config = this.mediaConfig.backend.imgur;
}
async saveFile(
buffer: Buffer,
fileName: string,
): Promise<[string, BackendData]> {
async saveFile(uuid: string, buffer: Buffer): Promise<string> {
const params = new URLSearchParams();
params.append('image', buffer.toString('base64'));
params.append('type', 'base64');
@ -50,36 +51,41 @@ export class ImgurBackend implements MediaBackend {
.then((res) => ImgurBackend.checkStatus(res))
.then((res) => res.json())) as UploadResult;
this.logger.debug(`Response: ${JSON.stringify(result)}`, 'saveFile');
this.logger.log(`Uploaded ${fileName}`, 'saveFile');
return [result.data.link, result.data.deletehash];
this.logger.log(`Uploaded file ${uuid}`, 'saveFile');
const backendData: ImgurBackendData = {
url: result.data.link,
deleteHash: result.data.deletehash,
};
return JSON.stringify(backendData);
} catch (e) {
this.logger.error(
`error: ${(e as Error).message}`,
(e as Error).stack,
'saveFile',
);
throw new MediaBackendError(`Could not save '${fileName}' on imgur`);
throw new MediaBackendError(`Could not save file ${uuid}`);
}
}
async deleteFile(fileName: string, backendData: BackendData): Promise<void> {
if (backendData === null) {
async deleteFile(uuid: string, jsonBackendData: string): Promise<void> {
const backendData = JSON.parse(jsonBackendData) as ImgurBackendData;
if (backendData.deleteHash === null) {
throw new MediaBackendError(
`We don't have any delete tokens for '${fileName}' and therefore can't delete this image on imgur`,
`We don't have any delete tokens for file ${uuid} and therefore can't delete this image`,
);
}
try {
const result = await fetch(
`https://api.imgur.com/3/image/${backendData}`,
`https://api.imgur.com/3/image/${backendData.deleteHash}`,
{
method: 'POST',
method: 'DELETE',
// eslint-disable-next-line @typescript-eslint/naming-convention
headers: { Authorization: `Client-ID ${this.config.clientID}` },
},
).then((res) => ImgurBackend.checkStatus(res));
);
ImgurBackend.checkStatus(result);
// eslint-disable-next-line @typescript-eslint/no-base-to-string
this.logger.debug(`Response: ${result.toString()}`, 'deleteFile');
this.logger.log(`Deleted ${fileName}`, 'deleteFile');
this.logger.log(`Deleted file ${uuid}`, 'deleteFile');
return;
} catch (e) {
this.logger.error(
@ -87,10 +93,20 @@ export class ImgurBackend implements MediaBackend {
(e as Error).stack,
'deleteFile',
);
throw new MediaBackendError(`Could not delete '${fileName}' on imgur`);
throw new MediaBackendError(`Could not delete file '${uuid}'`);
}
}
getFileUrl(uuid: string, backendData: string | null): Promise<string> {
if (backendData === null) {
throw new MediaBackendError(
`We don't have any data for file ${uuid} and therefore can't get the url of this image`,
);
}
const data = JSON.parse(backendData) as ImgurBackendData;
return Promise.resolve(data.url);
}
private static checkStatus(res: Response): Response {
if (res.ok) {
// res.status >= 200 && res.status < 300

87
backend/src/media/backends/s3-backend.spec.ts
View file

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2023 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
@ -17,6 +17,7 @@ describe('s3 backend', () => {
const mockedS3AccessKeyId = 'mockedS3AccessKeyId';
const mockedS3SecretAccessKey = 'mockedS3SecretAccessKey';
const mockedS3Bucket = 'mockedS3Bucket';
const mockedUuid = 'cbe87987-8e70-4092-a879-878e70b09245';
const mockedLoggerService = Mock.of<ConsoleLoggerService>({
setContext: jest.fn(),
@ -31,6 +32,7 @@ describe('s3 backend', () => {
mockedClient = Mock.of<Client>({
putObject: jest.fn(),
removeObject: jest.fn(),
presignedGetObject: jest.fn(),
});
clientConstructorSpy = jest
@ -143,19 +145,21 @@ describe('s3 backend', () => {
const sut = new S3Backend(mockedLoggerService, mediaConfig);
const mockedBuffer = Mock.of<Buffer>({});
const mockedFileName = 'mockedFileName';
const [url, backendData] = await sut.saveFile(
mockedBuffer,
mockedFileName,
);
await sut.saveFile(mockedUuid, mockedBuffer, {
mime: 'image/png',
ext: 'png',
});
expect(saveSpy).toHaveBeenCalledWith(
mockedS3Bucket,
mockedFileName,
mockedUuid,
mockedBuffer,
mockedBuffer.length,
{
// eslint-disable-next-line @typescript-eslint/naming-convention
'Content-Type': 'image/png',
},
);
expect(url).toBe('https://s3.example.org/mockedS3Bucket/mockedFileName');
expect(backendData).toBeNull();
});
it("will throw a MediaBackendError if the s3 client couldn't save the file", async () => {
@ -167,15 +171,24 @@ describe('s3 backend', () => {
const sut = new S3Backend(mockedLoggerService, mediaConfig);
const mockedBuffer = Mock.of<Buffer>({});
const mockedFileName = 'mockedFileName';
await expect(() =>
sut.saveFile(mockedBuffer, mockedFileName),
).rejects.toThrow("Could not save 'mockedFileName' on S3");
sut.saveFile(mockedUuid, mockedBuffer, {
mime: 'image/png',
ext: 'png',
}),
).rejects.toThrow(
'Could not save file cbe87987-8e70-4092-a879-878e70b09245',
);
expect(saveSpy).toHaveBeenCalledWith(
mockedS3Bucket,
mockedFileName,
mockedUuid,
mockedBuffer,
mockedBuffer.length,
{
// eslint-disable-next-line @typescript-eslint/naming-convention
'Content-Type': 'image/png',
},
);
});
});
@ -185,12 +198,11 @@ describe('s3 backend', () => {
const deleteSpy = jest
.spyOn(mockedClient, 'removeObject')
.mockImplementation(() => Promise.resolve());
const mockedFileName = 'mockedFileName';
const sut = new S3Backend(mockedLoggerService, mediaConfig);
await sut.deleteFile(mockedFileName);
await sut.deleteFile(mockedUuid, null);
expect(deleteSpy).toHaveBeenCalledWith(mockedS3Bucket, mockedFileName);
expect(deleteSpy).toHaveBeenCalledWith(mockedS3Bucket, mockedUuid);
});
it("will throw a MediaBackendError if the client couldn't delete the file", async () => {
@ -198,15 +210,50 @@ describe('s3 backend', () => {
const deleteSpy = jest
.spyOn(mockedClient, 'removeObject')
.mockImplementation(() => Promise.reject(new Error('mocked error')));
const mockedFileName = 'mockedFileName';
const sut = new S3Backend(mockedLoggerService, mediaConfig);
await expect(() => sut.deleteFile(mockedFileName)).rejects.toThrow(
"Could not delete 'mockedFileName' on S3",
await expect(() => sut.deleteFile(mockedUuid, null)).rejects.toThrow(
'Could not delete file cbe87987-8e70-4092-a879-878e70b09245',
);
expect(deleteSpy).toHaveBeenCalledWith(mockedS3Bucket, mockedFileName);
expect(deleteSpy).toHaveBeenCalledWith(mockedS3Bucket, mockedUuid);
});
});
describe('getFileUrl', () => {
it('returns a signed url', async () => {
const mediaConfig = mockMediaConfig('https://s3.example.org');
const fileUrlSpy = jest
.spyOn(mockedClient, 'presignedGetObject')
.mockImplementation(() =>
Promise.resolve(
'https://s3.example.org/mockedS3Bucket/cbe87987-8e70-4092-a879-878e70b09245?mockedSignature',
),
);
const sut = new S3Backend(mockedLoggerService, mediaConfig);
const url = await sut.getFileUrl(mockedUuid, null);
expect(fileUrlSpy).toHaveBeenCalledWith(mockedS3Bucket, mockedUuid);
expect(url).toBe(
'https://s3.example.org/mockedS3Bucket/cbe87987-8e70-4092-a879-878e70b09245?mockedSignature',
);
});
it('throws a MediaBackendError if the client could not generate a signed url', async () => {
const mediaConfig = mockMediaConfig('https://s3.example.org');
const fileUrlSpy = jest
.spyOn(mockedClient, 'presignedGetObject')
.mockImplementation(() => {
throw new Error('mocked error');
});
const sut = new S3Backend(mockedLoggerService, mediaConfig);
await expect(() => sut.getFileUrl(mockedUuid, null)).rejects.toThrow(
'Could not get URL for file cbe87987-8e70-4092-a879-878e70b09245',
);
expect(fileUrlSpy).toHaveBeenCalledWith(mockedS3Bucket, mockedUuid);
});
});
});

68
backend/src/media/backends/s3-backend.ts
View file

@ -1,9 +1,10 @@
/*
* SPDX-FileCopyrightText: 2021 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable } from '@nestjs/common';
import { FileTypeResult } from 'file-type';
import { Client } from 'minio';
import { URL } from 'url';
@ -11,7 +12,6 @@ import mediaConfiguration, { MediaConfig } from '../../config/media.config';
import { MediaBackendError } from '../../errors/errors';
import { ConsoleLoggerService } from '../../logger/console-logger.service';
import { MediaBackend } from '../media-backend.interface';
import { BackendData } from '../media-upload.entity';
import { BackendType } from './backend-type.enum';
@Injectable()
@ -19,64 +19,74 @@ export class S3Backend implements MediaBackend {
private config: MediaConfig['backend']['s3'];
private client: Client;
private static determinePort(url: URL): number | undefined {
const port = parseInt(url.port);
return isNaN(port) ? undefined : port;
}
constructor(
private readonly logger: ConsoleLoggerService,
@Inject(mediaConfiguration.KEY)
private mediaConfig: MediaConfig,
) {
this.logger.setContext(S3Backend.name);
if (mediaConfig.backend.use !== BackendType.S3) {
if (this.mediaConfig.backend.use !== BackendType.S3) {
return;
}
this.config = mediaConfig.backend.s3;
this.config = this.mediaConfig.backend.s3;
const url = new URL(this.config.endPoint);
const isSecure = url.protocol === 'https:';
this.client = new Client({
endPoint: url.hostname,
port: this.determinePort(url),
port: S3Backend.determinePort(url),
useSSL: isSecure,
accessKey: this.config.accessKeyId,
secretKey: this.config.secretAccessKey,
pathStyle: this.config.pathStyle,
region: this.config.region,
});
}
private determinePort(url: URL): number | undefined {
const port = parseInt(url.port);
return isNaN(port) ? undefined : port;
}
async saveFile(
uuid: string,
buffer: Buffer,
fileName: string,
): Promise<[string, BackendData]> {
fileType: FileTypeResult,
): Promise<null> {
try {
await this.client.putObject(this.config.bucket, fileName, buffer);
this.logger.log(`Uploaded file ${fileName}`, 'saveFile');
return [this.getUrl(fileName), null];
await this.client.putObject(
this.config.bucket,
uuid,
buffer,
buffer.length,
{
// eslint-disable-next-line @typescript-eslint/naming-convention
'Content-Type': fileType.mime,
},
);
this.logger.log(`Uploaded file ${uuid}`, 'saveFile');
return null;
} catch (e) {
this.logger.error((e as Error).message, (e as Error).stack, 'saveFile');
throw new MediaBackendError(`Could not save '${fileName}' on S3`);
throw new MediaBackendError(`Could not save file ${uuid}`);
}
}
async deleteFile(fileName: string): Promise<void> {
async deleteFile(uuid: string, _: unknown): Promise<void> {
try {
await this.client.removeObject(this.config.bucket, fileName);
const url = this.getUrl(fileName);
this.logger.log(`Deleted ${url}`, 'deleteFile');
return;
await this.client.removeObject(this.config.bucket, uuid);
this.logger.log(`Deleted uploaded file ${uuid}`, 'deleteFile');
} catch (e) {
this.logger.error((e as Error).message, (e as Error).stack, 'saveFile');
throw new MediaBackendError(`Could not delete '${fileName}' on S3`);
this.logger.error((e as Error).message, (e as Error).stack, 'deleteFile');
throw new MediaBackendError(`Could not delete file ${uuid}`);
}
}
private getUrl(fileName: string): string {
const url = new URL(this.config.endPoint);
if (!url.pathname.endsWith('/')) {
url.pathname += '/';
async getFileUrl(uuid: string, _: unknown): Promise<string> {
try {
return await this.client.presignedGetObject(this.config.bucket, uuid);
} catch (e) {
this.logger.error((e as Error).message, (e as Error).stack, 'getFileUrl');
throw new MediaBackendError(`Could not get URL for file ${uuid}`);
}
url.pathname += `${this.config.bucket}/${fileName}`;
return url.toString();
}
}

58
backend/src/media/backends/webdav-backend.ts
View file

@ -1,9 +1,10 @@
/*
* SPDX-FileCopyrightText: 2021 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable } from '@nestjs/common';
import { FileTypeResult } from 'file-type';
import fetch, { Response } from 'node-fetch';
import { URL } from 'url';
@ -11,14 +12,13 @@ import mediaConfiguration, { MediaConfig } from '../../config/media.config';
import { MediaBackendError } from '../../errors/errors';
import { ConsoleLoggerService } from '../../logger/console-logger.service';
import { MediaBackend } from '../media-backend.interface';
import { BackendData } from '../media-upload.entity';
import { BackendType } from './backend-type.enum';
@Injectable()
export class WebdavBackend implements MediaBackend {
private config: MediaConfig['backend']['webdav'];
private authHeader: string;
private baseUrl: string;
private readonly baseUrl: string;
constructor(
private readonly logger: ConsoleLoggerService,
@ -26,11 +26,10 @@ export class WebdavBackend implements MediaBackend {
private mediaConfig: MediaConfig,
) {
this.logger.setContext(WebdavBackend.name);
if (mediaConfig.backend.use === BackendType.WEBDAV) {
this.config = mediaConfig.backend.webdav;
if (this.mediaConfig.backend.use === BackendType.WEBDAV) {
this.config = this.mediaConfig.backend.webdav;
const url = new URL(this.config.connectionString);
const port = url.port !== '' ? `:${url.port}` : '';
this.baseUrl = `${url.protocol}//${url.hostname}${port}${url.pathname}`;
this.baseUrl = url.toString();
if (this.config.uploadDir && this.config.uploadDir !== '') {
this.baseUrl = WebdavBackend.joinURL(
this.baseUrl,
@ -61,12 +60,14 @@ export class WebdavBackend implements MediaBackend {
}
async saveFile(
uuid: string,
buffer: Buffer,
fileName: string,
): Promise<[string, BackendData]> {
fileType: FileTypeResult,
): Promise<string> {
try {
const contentLength = buffer.length;
await fetch(WebdavBackend.joinURL(this.baseUrl, '/', fileName), {
const remoteFileName = `${uuid}.${fileType.ext}`;
await fetch(WebdavBackend.joinURL(this.baseUrl, '/', remoteFileName), {
method: 'PUT',
body: buffer,
headers: {
@ -77,34 +78,49 @@ export class WebdavBackend implements MediaBackend {
'If-None-Match': '*', // Don't overwrite already existing files
},
}).then((res) => WebdavBackend.checkStatus(res));
this.logger.log(`Uploaded file ${fileName}`, 'saveFile');
return [this.getUrl(fileName), null];
this.logger.log(`Uploaded file ${uuid}`, 'saveFile');
return JSON.stringify({ file: remoteFileName });
} catch (e) {
this.logger.error((e as Error).message, (e as Error).stack, 'saveFile');
throw new MediaBackendError(`Could not save '${fileName}' on WebDav`);
throw new MediaBackendError(`Could not save upload '${uuid}'`);
}
}
async deleteFile(fileName: string, _: BackendData): Promise<void> {
async deleteFile(uuid: string, backendData: string): Promise<void> {
if (!backendData) {
throw new MediaBackendError('No backend data provided');
}
try {
await fetch(WebdavBackend.joinURL(this.baseUrl, '/', fileName), {
const { file } = JSON.parse(backendData) as { file: string };
if (!file) {
throw new MediaBackendError('No file name in backend data');
}
await fetch(WebdavBackend.joinURL(this.baseUrl, '/', file), {
method: 'DELETE',
headers: {
// eslint-disable-next-line @typescript-eslint/naming-convention
Authorization: this.authHeader,
},
}).then((res) => WebdavBackend.checkStatus(res));
const url = this.getUrl(fileName);
this.logger.log(`Deleted ${url}`, 'deleteFile');
this.logger.log(`Deleted upload ${uuid}`, 'deleteFile');
return;
} catch (e) {
this.logger.error((e as Error).message, (e as Error).stack, 'saveFile');
throw new MediaBackendError(`Could not delete '${fileName}' on WebDav`);
this.logger.error((e as Error).message, (e as Error).stack, 'deleteFile');
throw new MediaBackendError(`Could not delete upload '${uuid}'`);
}
}
private getUrl(fileName: string): string {
return WebdavBackend.joinURL(this.config.publicUrl, '/', fileName);
getFileUrl(_: string, backendData: string): Promise<string> {
if (!backendData) {
throw new MediaBackendError('No backend data provided');
}
const { file } = JSON.parse(backendData) as { file: string };
if (!file) {
throw new MediaBackendError('No file name in backend data');
}
return Promise.resolve(
WebdavBackend.joinURL(this.config.publicUrl, '/', file),
);
}
private static generateBasicAuthHeader(

28
backend/src/media/media-backend.interface.ts
View file

@ -1,25 +1,39 @@
/*
* SPDX-FileCopyrightText: 2021 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { BackendData } from './media-upload.entity';
import { FileTypeResult } from 'file-type';
export interface MediaBackend {
/**
* Saves a file according to backend internals.
* @param uuid Unique identifier of the uploaded file
* @param buffer File data
* @param fileName Name of the file to save. Can include a file extension.
* @param fileType File type result
* @throws {MediaBackendError} - there was an error saving the file
* @return Tuple of file URL and internal backend data, which should be saved.
* @return The internal backend data, which should be saved
*/
saveFile(buffer: Buffer, fileName: string): Promise<[string, BackendData]>;
saveFile(
uuid: string,
buffer: Buffer,
fileType?: FileTypeResult,
): Promise<string | null>;
/**
* Delete a file from the backend
* @param fileName String to identify the file
* @param uuid Unique identifier of the uploaded file
* @param backendData Internal backend data
* @throws {MediaBackendError} - there was an error deleting the file
*/
deleteFile(fileName: string, backendData: BackendData): Promise<void>;
deleteFile(uuid: string, backendData: string | null): Promise<void>;
/**
* Get a publicly accessible URL of a file from the backend
* @param uuid Unique identifier of the uploaded file
* @param backendData Internal backend data
* @throws {MediaBackendError} - there was an error getting the file
* @return Public accessible URL of the file
*/
getFileUrl(uuid: string, backendData: string | null): Promise<string>;
}

18
backend/src/media/media-upload.dto.ts
View file

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2023 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
@ -12,12 +12,20 @@ import { Username } from '../utils/username';
export class MediaUploadDto extends BaseDto {
/**
* The id of the media file.
* @example "testfile123.jpg"
* The uuid of the media file.
* @example "7697582e-0020-4188-9758-2e00207188ca"
*/
@IsString()
@ApiProperty()
id: string;
uuid: string;
/**
* The original filename of the media upload.
* @example "example.png"
*/
@IsString()
@ApiProperty()
fileName: string;
/**
* The publicId of the note to which the uploaded file is linked to.
@ -26,7 +34,7 @@ export class MediaUploadDto extends BaseDto {
@IsString()
@IsOptional()
@ApiProperty()
notePublicId: string | null;
noteId: string | null;
/**
* The date when the upload objects was created.

48
backend/src/media/media-upload.entity.ts
View file

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2021 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
@ -15,37 +15,49 @@ import { Note } from '../notes/note.entity';
import { User } from '../users/user.entity';
import { BackendType } from './backends/backend-type.enum';
export type BackendData = string | null;
@Entity()
export class MediaUpload {
/** The unique identifier of a media upload */
@PrimaryColumn()
id: string;
uuid: string;
/**
* The note where a media file was uploaded, required for the media browser in the note editor.
* Can be set to null after creation when the note was deleted without the associated uploads
*/
@ManyToOne((_) => Note, (note) => note.mediaUploads, {
nullable: true,
})
note: Promise<Note | null>;
/** The user who uploaded the media file or {@code null} if uploaded by a guest user */
@ManyToOne((_) => User, (user) => user.mediaUploads, {
nullable: true,
})
user: Promise<User | null>;
/** The original filename of the media upload */
@Column()
fileName: string;
/** The backend type where this upload is stored */
@Column({
nullable: false,
})
backendType: string;
@Column()
fileUrl: string;
/**
* Additional data, depending on the backend type, serialized as JSON.
* This can include for example required additional identifiers for retrieving the file from the backend or to
* delete the file afterward again.
*/
@Column({
nullable: true,
type: 'text',
})
backendData: BackendData | null;
backendData: string | null;
/** The date when the upload was created */
@CreateDateColumn()
createdAt: Date;
@ -53,30 +65,30 @@ export class MediaUpload {
private constructor() {}
/**
* Create a new media upload enity
* @param id the id of the upload
* Create a new media upload entity
*
* @param uuid the unique identifier of the upload
* @param fileName the original filename of the uploaded file
* @param note the note the upload should be associated with. This is required despite the fact the note field is optional, because it's possible to delete a note without also deleting the associated media uploads, but a note is required for the initial creation.
* @param user the user that owns the upload
* @param extension which file extension the upload has
* @param backendType on which type of media backend the upload is saved
* @param backendData the backend data returned by the media backend
* @param fileUrl the url where the upload can be accessed
*/
public static create(
id: string,
uuid: string,
fileName: string,
note: Note,
user: User | null,
extension: string,
backendType: BackendType,
fileUrl: string,
backendData: string | null,
): Omit<MediaUpload, 'createdAt'> {
const upload = new MediaUpload();
upload.id = id;
upload.uuid = uuid;
upload.fileName = fileName;
upload.note = Promise.resolve(note);
upload.user = Promise.resolve(user);
upload.backendType = backendType;
upload.backendData = null;
upload.fileUrl = fileUrl;
upload.backendData = backendData;
return upload;
}
}

135
backend/src/media/media.service.spec.ts
View file

@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2022 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
@ -35,7 +35,7 @@ import { User } from '../users/user.entity';
import { UsersModule } from '../users/users.module';
import { BackendType } from './backends/backend-type.enum';
import { FilesystemBackend } from './backends/filesystem-backend';
import { BackendData, MediaUpload } from './media-upload.entity';
import { MediaUpload } from './media-upload.entity';
import { MediaService } from './media.service';
describe('MediaService', () => {
@ -120,14 +120,16 @@ describe('MediaService', () => {
);
const user = User.create('test123', 'Test 123') as User;
const uuid = 'f7d334bb-6bb6-451b-9334-bb6bb6d51b5a';
const filename = 'test.jpg';
const note = Note.create(user) as Note;
const mediaUpload = MediaUpload.create(
'test',
uuid,
filename,
note,
user,
'.jpg',
BackendType.FILESYSTEM,
'test/test',
null,
) as MediaUpload;
const createQueryBuilder = {
@ -174,40 +176,40 @@ describe('MediaService', () => {
it('works', async () => {
const testImage = await fs.readFile('test/public-api/fixtures/test.png');
let fileId = '';
jest
.spyOn(mediaRepo, 'save')
.mockImplementationOnce(async (entry: MediaUpload) => {
fileId = entry.id;
return entry;
});
let givenUuid = '';
jest.spyOn(mediaRepo, 'save').mockImplementation();
jest
.spyOn(service.mediaBackend, 'saveFile')
.mockImplementationOnce(
async (
buffer: Buffer,
fileName: string,
): Promise<[string, BackendData]> => {
async (uuid: string, buffer: Buffer): Promise<string | null> => {
expect(buffer).toEqual(testImage);
return [fileName, null];
givenUuid = uuid;
return null;
},
);
const upload = await service.saveFile(testImage, user, note);
expect(upload.fileUrl).toEqual(fileId);
jest.spyOn(mediaRepo, 'save').mockImplementationOnce(async (entry) => {
expect(entry.uuid).toEqual(givenUuid);
return entry as MediaUpload;
});
const upload = await service.saveFile('test.jpg', testImage, user, note);
expect(upload.fileName).toEqual('test.jpg');
expect(upload.uuid).toEqual(givenUuid);
await expect(upload.note).resolves.toEqual(note);
await expect(upload.user).resolves.toEqual(user);
});
describe('fails:', () => {
it('MIME type not identifiable', async () => {
await expect(
service.saveFile(Buffer.alloc(1), user, note),
service.saveFile('fail.png', Buffer.alloc(1), user, note),
).rejects.toThrow(ClientError);
});
it('MIME type not supported', async () => {
const testText = await fs.readFile('test/public-api/fixtures/test.zip');
await expect(service.saveFile(testText, user, note)).rejects.toThrow(
ClientError,
);
await expect(
service.saveFile('fail.zip', testText, user, note),
).rejects.toThrow(ClientError);
});
});
});
@ -215,7 +217,12 @@ describe('MediaService', () => {
describe('deleteFile', () => {
it('works', async () => {
const mockMediaUploadEntry = {
id: 'testMediaUpload',
uuid: '64f260cc-e0d0-47e7-b260-cce0d097e767',
fileName: 'testFileName',
note: Promise.resolve({
id: 123,
} as Note),
backendType: BackendType.FILESYSTEM,
backendData: 'testBackendData',
user: Promise.resolve({
username: 'hardcoded',
@ -224,8 +231,8 @@ describe('MediaService', () => {
jest
.spyOn(service.mediaBackend, 'deleteFile')
.mockImplementationOnce(
async (fileName: string, backendData: BackendData): Promise<void> => {
expect(fileName).toEqual(mockMediaUploadEntry.id);
async (uuid: string, backendData: string | null): Promise<void> => {
expect(uuid).toEqual(mockMediaUploadEntry.uuid);
expect(backendData).toEqual(mockMediaUploadEntry.backendData);
},
);
@ -238,23 +245,49 @@ describe('MediaService', () => {
await service.deleteFile(mockMediaUploadEntry);
});
});
describe('getFileUrl', () => {
it('works', async () => {
const mockMediaUploadEntry = {
uuid: '64f260cc-e0d0-47e7-b260-cce0d097e767',
fileName: 'testFileName',
note: Promise.resolve({
id: 123,
} as Note),
backendType: BackendType.FILESYSTEM,
backendData: '{"ext": "png"}',
user: Promise.resolve({
username: 'hardcoded',
} as User),
} as MediaUpload;
await expect(service.getFileUrl(mockMediaUploadEntry)).resolves.toEqual(
'/uploads/64f260cc-e0d0-47e7-b260-cce0d097e767.png',
);
});
});
describe('findUploadByFilename', () => {
it('works', async () => {
const testFileName = 'testFilename';
const username = 'hardcoded';
const backendData = 'testBackendData';
const mockMediaUploadEntry = {
id: 'testMediaUpload',
backendData: backendData,
uuid: '64f260cc-e0d0-47e7-b260-cce0d097e767',
fileName: testFileName,
note: Promise.resolve({
id: 123,
} as Note),
backendType: BackendType.FILESYSTEM,
backendData,
user: Promise.resolve({
username: username,
username,
} as User),
} as MediaUpload;
jest
.spyOn(mediaRepo, 'findOne')
.mockResolvedValueOnce(mockMediaUploadEntry);
const mediaUpload = await service.findUploadByFilename(testFileName);
expect((await mediaUpload.user).username).toEqual(username);
expect((await mediaUpload.user)?.username).toEqual(username);
expect(mediaUpload.backendData).toEqual(backendData);
});
it("fails: can't find mediaUpload", async () => {
@ -271,10 +304,15 @@ describe('MediaService', () => {
const username = 'hardcoded';
it('with one upload from user', async () => {
const mockMediaUploadEntry = {
id: 'testMediaUpload',
backendData: 'testBackendData',
uuid: '64f260cc-e0d0-47e7-b260-cce0d097e767',
fileName: 'testFileName',
note: Promise.resolve({
id: 123,
} as Note),
backendType: BackendType.FILESYSTEM,
backendData: null,
user: Promise.resolve({
username: username,
username,
} as User),
} as MediaUpload;
createQueryBuilderFunc.getMany = () => [mockMediaUploadEntry];
@ -304,11 +342,16 @@ describe('MediaService', () => {
describe('works', () => {
it('with one upload to note', async () => {
const mockMediaUploadEntry = {
id: 'testMediaUpload',
backendData: 'testBackendData',
uuid: '64f260cc-e0d0-47e7-b260-cce0d097e767',
fileName: 'testFileName',
note: Promise.resolve({
id: 123,
} as Note),
backendType: BackendType.FILESYSTEM,
backendData: null,
user: Promise.resolve({
username: 'mockUser',
} as User),
} as MediaUpload;
const createQueryBuilder = {
where: () => createQueryBuilder,
@ -371,19 +414,19 @@ describe('MediaService', () => {
Alias.create('test', mockNote, true) as Alias,
]);
const mockMediaUploadEntry = {
id: 'testMediaUpload',
backendData: 'testBackendData',
note: Promise.resolve(mockNote),
uuid: '64f260cc-e0d0-47e7-b260-cce0d097e767',
fileName: 'testFileName',
note: mockNote,
backendType: BackendType.FILESYSTEM,
backendData: null,
user: Promise.resolve({
username: 'hardcoded',
username: 'mockUser',
} as User),
} as MediaUpload;
jest
.spyOn(mediaRepo, 'save')
.mockImplementationOnce(async (entry: MediaUpload) => {
expect(await entry.note).toBeNull();
return entry;
});
} as unknown as MediaUpload;
jest.spyOn(mediaRepo, 'save').mockImplementationOnce(async (entry) => {
expect(await entry.note).toBeNull();
return entry as MediaUpload;
});
await service.removeNoteFromMediaUpload(mockMediaUploadEntry);
expect(mediaRepo.save).toHaveBeenCalled();
});

78
backend/src/media/media.service.ts
View file

@ -1,22 +1,20 @@
/*
* SPDX-FileCopyrightText: 2021 The HedgeDoc developers (see AUTHORS file)
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable } from '@nestjs/common';
import { ModuleRef } from '@nestjs/core';
import { InjectRepository } from '@nestjs/typeorm';
import crypto from 'crypto';
import * as FileType from 'file-type';
import { Repository } from 'typeorm';
import { v4 as uuidV4 } from 'uuid';
import mediaConfiguration, { MediaConfig } from '../config/media.config';
import { ClientError, NotInDBError } from '../errors/errors';
import { ConsoleLoggerService } from '../logger/console-logger.service';
import { Note } from '../notes/note.entity';
import { NotesService } from '../notes/notes.service';
import { User } from '../users/user.entity';
import { UsersService } from '../users/users.service';
import { AzureBackend } from './backends/azure-backend';
import { BackendType } from './backends/backend-type.enum';
import { FilesystemBackend } from './backends/filesystem-backend';
@ -36,8 +34,6 @@ export class MediaService {
private readonly logger: ConsoleLoggerService,
@InjectRepository(MediaUpload)
private mediaUploadRepository: Repository<MediaUpload>,
private notesService: NotesService,
private usersService: UsersService,
private moduleRef: ModuleRef,
@Inject(mediaConfiguration.KEY)
private mediaConfig: MediaConfig,
@ -68,15 +64,17 @@ export class MediaService {
/**
* @async
* Save the given buffer to the configured MediaBackend and create a MediaUploadEntity to track where the file is, who uploaded it and to which note.
* @param {string} fileName - the original file name
* @param {Buffer} fileBuffer - the buffer of the file to save.
* @param {User} user - the user who uploaded this file
* @param {Note} note - the note which will be associated with the new file.
* @return {string} the url of the saved file
* @return {MediaUpload} the created MediaUpload entity
* @throws {ClientError} the MIME type of the file is not supported.
* @throws {NotInDBError} - the note or user is not in the database
* @throws {MediaBackendError} - there was an error saving the file
*/
async saveFile(
fileName: string,
fileBuffer: Buffer,
user: User | null,
note: Note,
@ -99,19 +97,20 @@ export class MediaService {
if (!MediaService.isAllowedMimeType(fileTypeResult.mime)) {
throw new ClientError('MIME type not allowed.');
}
const randomBytes = crypto.randomBytes(16);
const id = randomBytes.toString('hex') + '.' + fileTypeResult.ext;
this.logger.debug(`Generated filename: '${id}'`, 'saveFile');
const [url, backendData] = await this.mediaBackend.saveFile(fileBuffer, id);
const uuid = uuidV4(); // TODO replace this with uuid-v7 in a later PR
const backendData = await this.mediaBackend.saveFile(
uuid,
fileBuffer,
fileTypeResult,
);
const mediaUpload = MediaUpload.create(
id,
uuid,
fileName,
note,
user,
fileTypeResult.ext,
this.mediaBackendType,
url,
backendData,
);
mediaUpload.backendData = backendData;
return await this.mediaUploadRepository.save(mediaUpload);
}
@ -122,10 +121,26 @@ export class MediaService {
* @throws {MediaBackendError} - there was an error deleting the file
*/
async deleteFile(mediaUpload: MediaUpload): Promise<void> {
await this.mediaBackend.deleteFile(mediaUpload.id, mediaUpload.backendData);
await this.mediaBackend.deleteFile(
mediaUpload.uuid,
mediaUpload.backendData,
);
await this.mediaUploadRepository.remove(mediaUpload);
}
/**
* @async
* Get the URL of the file.
* @param {MediaUpload} mediaUpload - the file to get the URL for.
* @return {string} the URL of the file.
* @throws {MediaBackendError} - there was an error retrieving the url
*/
async getFileUrl(mediaUpload: MediaUpload): Promise<string> {
const backendName = mediaUpload.backendType as BackendType;
const backend = this.getBackendFromType(backendName);
return await backend.getFileUrl(mediaUpload.uuid, mediaUpload.backendData);
}
/**
* @async
* Find a file entry by its filename.
@ -136,7 +151,7 @@ export class MediaService {
*/
async findUploadByFilename(filename: string): Promise<MediaUpload> {
const mediaUpload = await this.mediaUploadRepository.findOne({
where: { id: filename },
where: { fileName: filename },
relations: ['user'],
});
if (mediaUpload === null) {
@ -147,6 +162,24 @@ export class MediaService {
return mediaUpload;
}
/**
* @async
* Find a file entry by its UUID.
* @param {string} uuid - The UUID of the MediaUpload entity to find.
* @returns {MediaUpload} - the MediaUpload entity if found.
* @throws {NotInDBError} - the MediaUpload entity with the provided UUID is not found in the database.
*/
async findUploadByUuid(uuid: string): Promise<MediaUpload> {
const mediaUpload = await this.mediaUploadRepository.findOne({
where: { uuid },
relations: ['user'],
});
if (mediaUpload === null) {
throw new NotInDBError(`MediaUpload with uuid '${uuid}' not found`);
}
return mediaUpload;
}
/**
* @async
* List all uploads by a specific user
@ -166,9 +199,9 @@ export class MediaService {
/**
* @async
* List all uploads by a specific note
* List all uploads to a specific note
* @param {Note} note - the specific user
* @return {MediaUpload[]} arary of media uploads owned by the user
* @return {MediaUpload[]} array of media uploads owned by the user
*/
async listUploadsByNote(note: Note): Promise<MediaUpload[]> {
const mediaUploads = await this.mediaUploadRepository
@ -188,7 +221,7 @@ export class MediaService {
*/
async removeNoteFromMediaUpload(mediaUpload: MediaUpload): Promise<void> {
this.logger.debug(
'Setting note to null for mediaUpload: ' + mediaUpload.id,
'Setting note to null for mediaUpload: ' + mediaUpload.uuid,
'removeNoteFromMediaUpload',
);
mediaUpload.note = Promise.resolve(null);
@ -232,8 +265,9 @@ export class MediaService {
async toMediaUploadDto(mediaUpload: MediaUpload): Promise<MediaUploadDto> {
const user = await mediaUpload.user;
return {
id: mediaUpload.id,
notePublicId: (await mediaUpload.note)?.publicId ?? null,
uuid: mediaUpload.uuid,
fileName: mediaUpload.fileName,
noteId: (await mediaUpload.note)?.publicId ?? null,
createdAt: mediaUpload.createdAt,
username: user?.username ?? null,
};

11
backend/src/migrations/mariadb-1725266569705-init.ts → backend/src/migrations/mariadb-1726084491570-init.ts
View file

@ -1,19 +1,14 @@
/*
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { MigrationInterface, QueryRunner } from 'typeorm';
export class Init1725266569705 implements MigrationInterface {
name = 'Init1725266569705';
export class Init1726084491570 implements MigrationInterface {
name = 'Init1726084491570';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE TABLE \`history_entry\` (\`id\` int NOT NULL AUTO_INCREMENT, \`pinStatus\` tinyint NOT NULL, \`updatedAt\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), \`userId\` int NULL, \`noteId\` int NULL, UNIQUE INDEX \`IDX_928dd947355b0837366470a916\` (\`noteId\`, \`userId\`), PRIMARY KEY (\`id\`)) ENGINE=InnoDB`,
);
await queryRunner.query(
`CREATE TABLE \`media_upload\` (\`id\` varchar(255) NOT NULL, \`backendType\` varchar(255) NOT NULL, \`fileUrl\` varchar(255) NOT NULL, \`backendData\` text NULL, \`createdAt\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), \`noteId\` int NULL, \`userId\` int NULL, PRIMARY KEY (\`id\`)) ENGINE=InnoDB`,
`CREATE TABLE \`media_upload\` (\`uuid\` varchar(255) NOT NULL, \`fileName\` varchar(255) NOT NULL, \`backendType\` varchar(255) NOT NULL, \`backendData\` text NULL, \`createdAt\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), \`noteId\` int NULL, \`userId\` int NULL, PRIMARY KEY (\`uuid\`)) ENGINE=InnoDB`,
);
await queryRunner.query(
`CREATE TABLE \`note_group_permission\` (\`id\` int NOT NULL AUTO_INCREMENT, \`canEdit\` tinyint NOT NULL, \`groupId\` int NULL, \`noteId\` int NULL, UNIQUE INDEX \`IDX_ee1744842a9ef3ffbc05a7016a\` (\`groupId\`, \`noteId\`), PRIMARY KEY (\`id\`)) ENGINE=InnoDB`,

11
backend/src/migrations/postgres-1725266697932-init.ts → backend/src/migrations/postgres-1726084117959-init.ts
View file

@ -1,12 +1,7 @@
/*
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { MigrationInterface, QueryRunner } from 'typeorm';
export class Init1725266697932 implements MigrationInterface {
name = 'Init1725266697932';
export class Init1726084117959 implements MigrationInterface {
name = 'Init1726084117959';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
@ -16,7 +11,7 @@ export class Init1725266697932 implements MigrationInterface {
`CREATE UNIQUE INDEX "IDX_928dd947355b0837366470a916" ON "history_entry" ("noteId", "userId") `,
);
await queryRunner.query(
`CREATE TABLE "media_upload" ("id" character varying NOT NULL, "backendType" character varying NOT NULL, "fileUrl" character varying NOT NULL, "backendData" text, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "noteId" integer, "userId" integer, CONSTRAINT "PK_b406d9cee56e253dfd3b3d52706" PRIMARY KEY ("id"))`,
`CREATE TABLE "media_upload" ("uuid" character varying NOT NULL, "fileName" character varying NOT NULL, "backendType" character varying NOT NULL, "backendData" text, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "noteId" integer, "userId" integer, CONSTRAINT "PK_573c2a4f2a8f8382f2a8758444e" PRIMARY KEY ("uuid"))`,
);
await queryRunner.query(
`CREATE TABLE "note_group_permission" ("id" SERIAL NOT NULL, "canEdit" boolean NOT NULL, "groupId" integer, "noteId" integer, CONSTRAINT "PK_6327989190949e6a55d02a080c3" PRIMARY KEY ("id"))`,

19
backend/src/migrations/sqlite-1725268109950-init.ts → backend/src/migrations/sqlite-1726084595852-init.ts
View file

@ -1,12 +1,7 @@
/*
* SPDX-FileCopyrightText: 2024 The HedgeDoc developers (see AUTHORS file)
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { MigrationInterface, QueryRunner } from 'typeorm';
export class Init1725268109950 implements MigrationInterface {
name = 'Init1725268109950';
export class Init1726084595852 implements MigrationInterface {
name = 'Init1726084595852';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
@ -16,7 +11,7 @@ export class Init1725268109950 implements MigrationInterface {
`CREATE UNIQUE INDEX "IDX_928dd947355b0837366470a916" ON "history_entry" ("noteId", "userId") `,
);
await queryRunner.query(
`CREATE TABLE "media_upload" ("id" varchar PRIMARY KEY NOT NULL, "backendType" varchar NOT NULL, "fileUrl" varchar NOT NULL, "backendData" text, "createdAt" datetime NOT NULL DEFAULT (datetime('now')), "noteId" integer, "userId" integer)`,
`CREATE TABLE "media_upload" ("uuid" varchar PRIMARY KEY NOT NULL, "fileName" varchar NOT NULL, "backendType" varchar NOT NULL, "backendData" text, "createdAt" datetime NOT NULL DEFAULT (datetime('now')), "noteId" integer, "userId" integer)`,
);
await queryRunner.query(
`CREATE TABLE "note_group_permission" ("id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "canEdit" boolean NOT NULL, "groupId" integer, "noteId" integer)`,
@ -108,10 +103,10 @@ export class Init1725268109950 implements MigrationInterface {
`CREATE UNIQUE INDEX "IDX_928dd947355b0837366470a916" ON "history_entry" ("noteId", "userId") `,
);
await queryRunner.query(
`CREATE TABLE "temporary_media_upload" ("id" varchar PRIMARY KEY NOT NULL, "backendType" varchar NOT NULL, "fileUrl" varchar NOT NULL, "backendData" text, "createdAt" datetime NOT NULL DEFAULT (datetime('now')), "noteId" integer, "userId" integer, CONSTRAINT "FK_edba6d4e0f3bcf6605772f0af6b" FOREIGN KEY ("noteId") REFERENCES "note" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION, CONSTRAINT "FK_73ce66b082df1df2003e305e9ac" FOREIGN KEY ("userId") REFERENCES "user" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION)`,
`CREATE TABLE "temporary_media_upload" ("uuid" varchar PRIMARY KEY NOT NULL, "fileName" varchar NOT NULL, "backendType" varchar NOT NULL, "backendData" text, "createdAt" datetime NOT NULL DEFAULT (datetime('now')), "noteId" integer, "userId" integer, CONSTRAINT "FK_edba6d4e0f3bcf6605772f0af6b" FOREIGN KEY ("noteId") REFERENCES "note" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION, CONSTRAINT "FK_73ce66b082df1df2003e305e9ac" FOREIGN KEY ("userId") REFERENCES "user" ("id") ON DELETE NO ACTION ON UPDATE NO ACTION)`,
);
await queryRunner.query(
`INSERT INTO "temporary_media_upload"("id", "backendType", "fileUrl", "backendData", "createdAt", "noteId", "userId") SELECT "id", "backendType", "fileUrl", "backendData", "createdAt", "noteId", "userId" FROM "media_upload"`,
`INSERT INTO "temporary_media_upload"("uuid", "fileName", "backendType", "backendData", "createdAt", "noteId", "userId") SELECT "uuid", "fileName", "backendType", "backendData", "createdAt", "noteId", "userId" FROM "media_upload"`,
);
await queryRunner.query(`DROP TABLE "media_upload"`);
await queryRunner.query(
@ -444,10 +439,10 @@ export class Init1725268109950 implements MigrationInterface {
`ALTER TABLE "media_upload" RENAME TO "temporary_media_upload"`,
);
await queryRunner.query(
`CREATE TABLE "media_upload" ("id" varchar PRIMARY KEY NOT NULL, "backendType" varchar NOT NULL, "fileUrl" varchar NOT NULL, "backendData" text, "createdAt" datetime NOT NULL DEFAULT (datetime('now')), "noteId" integer, "userId" integer)`,
`CREATE TABLE "media_upload" ("uuid" varchar PRIMARY KEY NOT NULL, "fileName" varchar NOT NULL, "backendType" varchar NOT NULL, "backendData" text, "createdAt" datetime NOT NULL DEFAULT (datetime('now')), "noteId" integer, "userId" integer)`,
);
await queryRunner.query(
`INSERT INTO "media_upload"("id", "backendType", "fileUrl", "backendData", "createdAt", "noteId", "userId") SELECT "id", "backendType", "fileUrl", "backendData", "createdAt", "noteId", "userId" FROM "temporary_media_upload"`,
`INSERT INTO "media_upload"("uuid", "fileName", "backendType", "backendData", "createdAt", "noteId", "userId") SELECT "uuid", "fileName", "backendType", "backendData", "createdAt", "noteId", "userId" FROM "temporary_media_upload"`,
);
await queryRunner.query(`DROP TABLE "temporary_media_upload"`);
await queryRunner.query(`DROP INDEX "IDX_928dd947355b0837366470a916"`);