From b0b9b75e658481bfb37f53cd20120bfa282b51aa Mon Sep 17 00:00:00 2001 From: David Mehren Date: Mon, 12 Oct 2020 21:51:09 +0200 Subject: [PATCH 01/24] Public API: /media/upload returns the URL of the uploaded file Signed-off-by: David Mehren --- docs/dev/public_api.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/dev/public_api.yml b/docs/dev/public_api.yml index c6312844d..181b80a10 100644 --- a/docs/dev/public_api.yml +++ b/docs/dev/public_api.yml @@ -471,6 +471,13 @@ paths: responses: '200': description: The image was uploaded successfully. + content: + application/json: + schema: + type: object + properties: + link: + type: string '401': "$ref": "#/components/responses/UnauthorizedError" '403': From 0a0732049ab46615827cd01969ed9d8d20466a6b Mon Sep 17 00:00:00 2001 From: David Mehren Date: Tue, 13 Oct 2020 10:19:12 +0200 Subject: [PATCH 02/24] DB Schema: Add MediaUpload entity MediaUpload stores the uploading user, the note the media was uploaded to and backend data. Signed-off-by: David Mehren --- docs/dev/db-schema.plantuml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/dev/db-schema.plantuml b/docs/dev/db-schema.plantuml index 56063af26..280a71ece 100644 --- a/docs/dev/db-schema.plantuml +++ b/docs/dev/db-schema.plantuml @@ -115,6 +115,17 @@ entity "Group" { *canEdit : boolean } +entity "MediaUpload" { + *id : text <> + -- + *noteId : uuid <> + *userId : uuid <> + *extension : text + *backendType: text + backendData: text + *createdAt : date +} + Note "1" - "1..*" Revision Revision "0..*" - "0..*" Authorship (Revision, Authorship) .. RevisionAuthorship @@ -129,4 +140,6 @@ authToken "1..*" -- "1" User seesion "1..*" -- "1" User Note "0..*" -- "0..*" User : color (Note, User) .. AuthorColors +MediaUpload "0..*" -- "1" Note +MediaUpload "0..*" -- "1" User @enduml From f01c7dbbe2d8704d98b8f11f645c55e2a23cc597 Mon Sep 17 00:00:00 2001 From: David Mehren Date: Tue, 13 Oct 2020 10:33:54 +0200 Subject: [PATCH 03/24] Add MediaUpload entity & Media module Signed-off-by: David Mehren --- src/app.module.ts | 2 + src/media/media-upload.entity.ts | 65 ++++++++++++++++++++++++++++++++ src/media/media.module.ts | 8 ++++ 3 files changed, 75 insertions(+) create mode 100644 src/media/media-upload.entity.ts create mode 100644 src/media/media.module.ts diff --git a/src/app.module.ts b/src/app.module.ts index 138c0c615..5e912acb0 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -10,6 +10,7 @@ import { NotesModule } from './notes/notes.module'; import { PermissionsModule } from './permissions/permissions.module'; import { RevisionsModule } from './revisions/revisions.module'; import { UsersModule } from './users/users.module'; +import { MediaModule } from './media/media.module'; @Module({ imports: [ @@ -29,6 +30,7 @@ import { UsersModule } from './users/users.module'; PermissionsModule, GroupsModule, LoggerModule, + MediaModule, ], controllers: [], providers: [], diff --git a/src/media/media-upload.entity.ts b/src/media/media-upload.entity.ts new file mode 100644 index 000000000..485f9a883 --- /dev/null +++ b/src/media/media-upload.entity.ts @@ -0,0 +1,65 @@ +import * as crypto from 'crypto'; +import { + Column, + CreateDateColumn, + Entity, + ManyToOne, + PrimaryColumn, +} from 'typeorm'; +import { Note } from '../notes/note.entity'; +import { User } from '../users/user.entity'; + +@Entity() +export class MediaUpload { + @PrimaryColumn() + id: string; + + @ManyToOne(_ => Note, { nullable: false }) + note: Note; + + @ManyToOne(_ => User, { nullable: false }) + user: User; + + @Column({ + nullable: false, + }) + extension: string; + + @Column({ + nullable: false, + }) + backendType: string; + + @Column({ + nullable: true, + }) + backendData: string | null; + + @CreateDateColumn() + createdAt: Date; + + // eslint-disable-next-line @typescript-eslint/no-empty-function + private constructor() {} + + public static create( + note: Note, + user: User, + extension: string, + backendType: string, + backendData?: string, + ): MediaUpload { + const upload = new MediaUpload(); + const randomBytes = crypto.randomBytes(16); + upload.id = randomBytes.toString('hex'); + upload.note = note; + upload.user = user; + upload.extension = extension; + upload.backendType = backendType; + if (backendData) { + upload.backendData = backendData; + } else { + upload.backendData = null; + } + return upload; + } +} diff --git a/src/media/media.module.ts b/src/media/media.module.ts new file mode 100644 index 000000000..eebd3501a --- /dev/null +++ b/src/media/media.module.ts @@ -0,0 +1,8 @@ +import { Module } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { MediaUpload } from './media-upload.entity'; + +@Module({ + imports: [TypeOrmModule.forFeature([MediaUpload])], +}) +export class MediaModule {} From 9d8086bf3e1b17d6db40cfcacd4ec15008b2eee3 Mon Sep 17 00:00:00 2001 From: David Mehren Date: Fri, 16 Oct 2020 22:29:13 +0200 Subject: [PATCH 04/24] Define a MediaBackend interface This interface defines the functionality that all media backends (like S3 or Azure) must implement. Signed-off-by: David Mehren --- src/media/media-backend.interface.ts | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 src/media/media-backend.interface.ts diff --git a/src/media/media-backend.interface.ts b/src/media/media-backend.interface.ts new file mode 100644 index 000000000..52e6883d4 --- /dev/null +++ b/src/media/media-backend.interface.ts @@ -0,0 +1,25 @@ +import { BackendData } from './media-upload.entity'; + +export interface MediaBackend { + /** + * Saves a file according to backend internals. + * @param buffer File data + * @param fileName Name of the file to save. Can include a file extension. + * @return Tuple of file URL and internal backend data, which should be saved. + */ + saveFile(buffer: Buffer, fileName: string): Promise<[string, BackendData]>; + + /** + * Retrieve the URL of a previously saved file. + * @param fileName String to identify the file + * @param backendData Internal backend data + */ + getFileURL(fileName: string, backendData: BackendData): Promise; + + /** + * Delete a file from the backend + * @param fileName String to identify the file + * @param backendData Internal backend data + */ + deleteFile(fileName: string, backendData: BackendData): Promise; +} From 3689741ad9ab4f62a1799921877da087e36aa367 Mon Sep 17 00:00:00 2001 From: David Mehren Date: Fri, 16 Oct 2020 22:30:57 +0200 Subject: [PATCH 05/24] Implement filesystem media backend This backend stores uploaded media into files on the local filesystem. This commit also adds a `BackendType` enum, which can be used to distinguish different media backends. Signed-off-by: David Mehren --- src/media/backends/backend-type.enum.ts | 6 ++++++ src/media/backends/filesystem-backend.ts | 26 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 src/media/backends/backend-type.enum.ts create mode 100644 src/media/backends/filesystem-backend.ts diff --git a/src/media/backends/backend-type.enum.ts b/src/media/backends/backend-type.enum.ts new file mode 100644 index 000000000..60117b256 --- /dev/null +++ b/src/media/backends/backend-type.enum.ts @@ -0,0 +1,6 @@ +export enum BackendType { + FILEYSTEM = 'filesystem', + S3 = 's3', + IMGUR = 'imgur', + AZURE = 'azure', +} diff --git a/src/media/backends/filesystem-backend.ts b/src/media/backends/filesystem-backend.ts new file mode 100644 index 000000000..37cb440dd --- /dev/null +++ b/src/media/backends/filesystem-backend.ts @@ -0,0 +1,26 @@ +import { MediaBackend } from '../media-backend.interface'; +import { BackendData } from '../media-upload.entity'; +import { promises as fs } from 'fs'; +import { join } from 'path'; + +export class FilesystemBackend implements MediaBackend { + async saveFile( + buffer: Buffer, + fileName: string, + ): Promise<[string, BackendData]> { + // TODO: Get uploads directory from config + const uploadDirectory = './uploads'; + // TODO: Add server address to url + const filePath = join(uploadDirectory, fileName); + await fs.writeFile(filePath, buffer, null); + return ['/' + filePath, null]; + } + + deleteFile(fileName: string, backendData: BackendData): Promise { + return Promise.resolve(undefined); + } + + getFileURL(fileNam: string, backendData: BackendData): Promise { + return Promise.resolve(''); + } +} From f3e093c7150a4eeb234419508eb7031991dad292 Mon Sep 17 00:00:00 2001 From: David Mehren Date: Fri, 16 Oct 2020 22:32:58 +0200 Subject: [PATCH 06/24] Do not save file extension as a separate field. It turned out that saving the file extension in a separate field is not necessary. Instead, the extension is saved in the complete filename in the `id` field. Signed-off-by: David Mehren --- docs/dev/db-schema.plantuml | 1 - src/media/media-upload.entity.ts | 15 ++++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/docs/dev/db-schema.plantuml b/docs/dev/db-schema.plantuml index 280a71ece..3ec0fb4a7 100644 --- a/docs/dev/db-schema.plantuml +++ b/docs/dev/db-schema.plantuml @@ -120,7 +120,6 @@ entity "MediaUpload" { -- *noteId : uuid <> *userId : uuid <> - *extension : text *backendType: text backendData: text *createdAt : date diff --git a/src/media/media-upload.entity.ts b/src/media/media-upload.entity.ts index 485f9a883..5ec8739b1 100644 --- a/src/media/media-upload.entity.ts +++ b/src/media/media-upload.entity.ts @@ -8,6 +8,9 @@ import { } from 'typeorm'; import { Note } from '../notes/note.entity'; import { User } from '../users/user.entity'; +import { BackendType } from './backends/backend-type.enum'; + +export type BackendData = string | null; @Entity() export class MediaUpload { @@ -20,11 +23,6 @@ export class MediaUpload { @ManyToOne(_ => User, { nullable: false }) user: User; - @Column({ - nullable: false, - }) - extension: string; - @Column({ nullable: false, }) @@ -33,7 +31,7 @@ export class MediaUpload { @Column({ nullable: true, }) - backendData: string | null; + backendData: BackendData; @CreateDateColumn() createdAt: Date; @@ -45,15 +43,14 @@ export class MediaUpload { note: Note, user: User, extension: string, - backendType: string, + backendType: BackendType, backendData?: string, ): MediaUpload { const upload = new MediaUpload(); const randomBytes = crypto.randomBytes(16); - upload.id = randomBytes.toString('hex'); + upload.id = randomBytes.toString('hex') + '.' + extension; upload.note = note; upload.user = user; - upload.extension = extension; upload.backendType = backendType; if (backendData) { upload.backendData = backendData; From 7a6c06d0682246e344e19e2f2a97e2a1e3c41f78 Mon Sep 17 00:00:00 2001 From: David Mehren Date: Fri, 16 Oct 2020 22:35:53 +0200 Subject: [PATCH 07/24] Add `MediaService` This service is responsible for operations regarding uploaded media. It should perform save, get and delete operations with the configured backend. The service also checks, if the mime type of the uploaded media is allowed. Signed-off-by: David Mehren --- package.json | 1 + src/media/media.module.ts | 7 +++- src/media/media.service.spec.ts | 18 +++++++++++ src/media/media.service.ts | 52 ++++++++++++++++++++++++++++++ src/media/multer-file.interface.ts | 32 ++++++++++++++++++ 5 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 src/media/media.service.spec.ts create mode 100644 src/media/media.service.ts create mode 100644 src/media/multer-file.interface.ts diff --git a/package.json b/package.json index 4b6dd5c77..1bfd9eb58 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "class-transformer": "^0.2.3", "class-validator": "^0.12.2", "connect-typeorm": "^1.1.4", + "file-type": "^15.0.1", "raw-body": "^2.4.1", "reflect-metadata": "^0.1.13", "rimraf": "^3.0.2", diff --git a/src/media/media.module.ts b/src/media/media.module.ts index eebd3501a..f791218e4 100644 --- a/src/media/media.module.ts +++ b/src/media/media.module.ts @@ -1,8 +1,13 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; +import { NotesModule } from '../notes/notes.module'; +import { UsersModule } from '../users/users.module'; import { MediaUpload } from './media-upload.entity'; +import { MediaService } from './media.service'; @Module({ - imports: [TypeOrmModule.forFeature([MediaUpload])], + imports: [TypeOrmModule.forFeature([MediaUpload]), NotesModule, UsersModule], + providers: [MediaService], + exports: [MediaService], }) export class MediaModule {} diff --git a/src/media/media.service.spec.ts b/src/media/media.service.spec.ts new file mode 100644 index 000000000..5009e4a57 --- /dev/null +++ b/src/media/media.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { MediaService } from './media.service'; + +describe('MediaService', () => { + let service: MediaService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [MediaService], + }).compile(); + + service = module.get(MediaService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/media/media.service.ts b/src/media/media.service.ts new file mode 100644 index 000000000..c8dfa2c3e --- /dev/null +++ b/src/media/media.service.ts @@ -0,0 +1,52 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import * as FileType from 'file-type'; +import { Repository } from 'typeorm'; +import { NotesService } from '../notes/notes.service'; +import { UsersService } from '../users/users.service'; +import { BackendType } from './backends/backend-type.enum'; +import { FilesystemBackend } from './backends/filesystem-backend'; +import { MediaUpload } from './media-upload.entity'; +import { MulterFile } from './multer-file.interface'; + +@Injectable() +export class MediaService { + constructor( + @InjectRepository(MediaUpload) + private mediaUploadRepository: Repository, + private notesService: NotesService, + private usersService: UsersService, + ) {} + + public async saveFile(file: MulterFile, username: string, noteId: string) { + const note = await this.notesService.getNoteByIdOrAlias(noteId); + const user = await this.usersService.getUserByUsername(username); + const fileTypeResult = await FileType.fromBuffer(file.buffer); + if (!fileTypeResult) { + throw new Error('Could not detect file type.'); + } + if (!MediaService.isAllowedMimeType(fileTypeResult.mime)) { + throw new Error('MIME type not allowed'); + } + //TODO: Choose backend according to config + const mediaUpload = MediaUpload.create( + note, + user, + fileTypeResult.ext, + BackendType.FILEYSTEM, + ); + const backend = new FilesystemBackend(); + const [url, backendData] = await backend.saveFile( + file.buffer, + mediaUpload.id, + ); + mediaUpload.backendData = backendData; + await this.mediaUploadRepository.save(mediaUpload); + return url; + } + + private static isAllowedMimeType(mimeType: string): boolean { + //TODO: Which mimetypes are allowed? + return true; + } +} diff --git a/src/media/multer-file.interface.ts b/src/media/multer-file.interface.ts new file mode 100644 index 000000000..a602a5008 --- /dev/null +++ b/src/media/multer-file.interface.ts @@ -0,0 +1,32 @@ +import { Readable } from 'stream'; + +// Type from https://github.com/DefinitelyTyped/DefinitelyTyped/blob/master/types/multer/index.d.ts +export interface MulterFile { + /** Name of the form field associated with this file. */ + fieldname: string; + /** Name of the file on the uploader's computer. */ + originalname: string; + /** + * Value of the `Content-Transfer-Encoding` header for this file. + * @deprecated since July 2015 + * @see RFC 7578, Section 4.7 + */ + encoding: string; + /** Value of the `Content-Type` header for this file. */ + mimetype: string; + /** Size of the file in bytes. */ + size: number; + /** + * A readable stream of this file. Only available to the `_handleFile` + * callback for custom `StorageEngine`s. + */ + stream: Readable; + /** `DiskStorage` only: Directory to which this file has been uploaded. */ + destination: string; + /** `DiskStorage` only: Name of this file within `destination`. */ + filename: string; + /** `DiskStorage` only: Full path to the uploaded file. */ + path: string; + /** `MemoryStorage` only: A Buffer containing the entire file. */ + buffer: Buffer; +} From 8e234962d67af28d89775a1205fcf575be7bd7df Mon Sep 17 00:00:00 2001 From: David Mehren Date: Fri, 16 Oct 2020 22:37:20 +0200 Subject: [PATCH 08/24] MediaController: Use `MediaService` to store media Signed-off-by: David Mehren --- src/api/public/media/media.controller.ts | 20 +++++++++++++++++--- src/api/public/public-api.module.ts | 2 ++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/api/public/media/media.controller.ts b/src/api/public/media/media.controller.ts index 57aa18121..4322aaf4e 100644 --- a/src/api/public/media/media.controller.ts +++ b/src/api/public/media/media.controller.ts @@ -6,16 +6,30 @@ import { } from '@nestjs/common'; import { FileInterceptor } from '@nestjs/platform-express'; import { ConsoleLoggerService } from '../../../logger/console-logger.service'; +import { MediaService } from '../../../media/media.service'; +import { MulterFile } from '../../../media/multer-file.interface'; @Controller('media') export class MediaController { - constructor(private readonly logger: ConsoleLoggerService) { + constructor( + private readonly logger: ConsoleLoggerService, + private mediaService: MediaService, + ) { this.logger.setContext(MediaController.name); } @Post('upload') @UseInterceptors(FileInterceptor('file')) - uploadImage(@UploadedFile() file) { - this.logger.debug('Recieved file: ' + file); + async uploadImage(@UploadedFile() file: MulterFile) { + this.logger.debug('Recieved file: ' + file.originalname); + //TODO: Get user and note from request + const url = await this.mediaService.saveFile( + file, + 'hardcoded', + 'hardcoded', + ); + return { + link: url, + }; } } diff --git a/src/api/public/public-api.module.ts b/src/api/public/public-api.module.ts index 26646d4cf..bc1f09c79 100644 --- a/src/api/public/public-api.module.ts +++ b/src/api/public/public-api.module.ts @@ -1,6 +1,7 @@ import { Module } from '@nestjs/common'; import { HistoryModule } from '../../history/history.module'; import { LoggerModule } from '../../logger/logger.module'; +import { MediaModule } from '../../media/media.module'; import { MonitoringModule } from '../../monitoring/monitoring.module'; import { NotesModule } from '../../notes/notes.module'; import { RevisionsModule } from '../../revisions/revisions.module'; @@ -18,6 +19,7 @@ import { MonitoringController } from './monitoring/monitoring.controller'; RevisionsModule, MonitoringModule, LoggerModule, + MediaModule, ], controllers: [ MeController, From 9743018591013faa3df11208912a4b10ae95837c Mon Sep 17 00:00:00 2001 From: David Mehren Date: Fri, 16 Oct 2020 22:38:31 +0200 Subject: [PATCH 09/24] Use `serve-static` to serve uploaded files. Add `@nestjs/serve-static` to serve uploaded media from the upload directory on the local filesystem. Signed-off-by: David Mehren --- package.json | 1 + src/app.module.ts | 9 +++++++- yarn.lock | 56 ++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 1bfd9eb58..012173c8d 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "@nestjs/common": "^7.0.0", "@nestjs/core": "^7.0.0", "@nestjs/platform-express": "^7.0.0", + "@nestjs/serve-static": "^2.1.3", "@nestjs/swagger": "^4.5.12", "@nestjs/typeorm": "^7.1.0", "class-transformer": "^0.2.3", diff --git a/src/app.module.ts b/src/app.module.ts index 5e912acb0..57a842aa4 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -1,16 +1,18 @@ import { Module } from '@nestjs/common'; +import { ServeStaticModule } from '@nestjs/serve-static'; import { TypeOrmModule } from '@nestjs/typeorm'; +import { join } from 'path'; import { PublicApiModule } from './api/public/public-api.module'; import { AuthorsModule } from './authors/authors.module'; import { GroupsModule } from './groups/groups.module'; import { HistoryModule } from './history/history.module'; import { LoggerModule } from './logger/logger.module'; +import { MediaModule } from './media/media.module'; import { MonitoringModule } from './monitoring/monitoring.module'; import { NotesModule } from './notes/notes.module'; import { PermissionsModule } from './permissions/permissions.module'; import { RevisionsModule } from './revisions/revisions.module'; import { UsersModule } from './users/users.module'; -import { MediaModule } from './media/media.module'; @Module({ imports: [ @@ -20,6 +22,11 @@ import { MediaModule } from './media/media.module'; autoLoadEntities: true, synchronize: true, }), + ServeStaticModule.forRoot({ + rootPath: join(__dirname, '..'), + // TODO: Get uploads directory from config + renderPath: 'uploads', + }), NotesModule, UsersModule, RevisionsModule, diff --git a/yarn.lock b/yarn.lock index bfe54b370..2cdf87f00 100644 --- a/yarn.lock +++ b/yarn.lock @@ -614,6 +614,13 @@ "@angular-devkit/schematics" "9.1.7" fs-extra "9.0.0" +"@nestjs/serve-static@^2.1.3": + version "2.1.3" + resolved "https://registry.yarnpkg.com/@nestjs/serve-static/-/serve-static-2.1.3.tgz#bdcb6d3463d193153b334212facc24a9767046e9" + integrity sha512-9xyysggaOdfbABWqhty+hAkauDWv/Q8YKHm4OMXdQbQei5tquFuTjiSx8IFDOZeSOKlA9fjBq/2MXCJRSo23SQ== + dependencies: + path-to-regexp "0.1.7" + "@nestjs/swagger@^4.5.12": version "4.5.12" resolved "https://registry.yarnpkg.com/@nestjs/swagger/-/swagger-4.5.12.tgz#e8aa65fbb0033007ece1d494b002f47ff472c20b" @@ -669,6 +676,11 @@ dependencies: "@sinonjs/commons" "^1.7.0" +"@tokenizer/token@^0.1.0", "@tokenizer/token@^0.1.1": + version "0.1.1" + resolved "https://registry.yarnpkg.com/@tokenizer/token/-/token-0.1.1.tgz#f0d92c12f87079ddfd1b29f614758b9696bc29e3" + integrity sha512-XO6INPbZCxdprl+9qa/AAbFFOMzzwqYxpjPgLICrMD6C2FCw6qfJOPcBk6JqqPLSaZ/Qx87qn4rpPmPMwaAK6w== + "@types/anymatch@*": version "1.3.1" resolved "https://registry.yarnpkg.com/@types/anymatch/-/anymatch-1.3.1.tgz#336badc1beecb9dacc38bea2cf32adf627a8421a" @@ -737,6 +749,11 @@ resolved "https://registry.yarnpkg.com/@types/debug/-/debug-0.0.31.tgz#bac8d8aab6a823e91deb7f79083b2a35fa638f33" integrity sha512-LS1MCPaQKqspg7FvexuhmDbWUhE2yIJ+4AgVIyObfc06/UKZ8REgxGNjZc82wPLWmbeOm7S+gSsLgo75TanG4A== +"@types/debug@^4.1.5": + version "4.1.5" + resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.5.tgz#b14efa8852b7768d898906613c23f688713e02cd" + integrity sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ== + "@types/eslint-visitor-keys@^1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz#1ee30d79544ca84d68d4b3cdb0af4f205663dd2d" @@ -3118,6 +3135,16 @@ file-entry-cache@^5.0.1: dependencies: flat-cache "^2.0.1" +file-type@^15.0.1: + version "15.0.1" + resolved "https://registry.yarnpkg.com/file-type/-/file-type-15.0.1.tgz#54175484953d48b970c095ba8737d4e0c3a9b407" + integrity sha512-0LieQlSA3bWUdErNrxzxfI4rhsvNAVPBO06R8pTc1hp9SE6nhqlVyvhcaXoMmtXkBTPnQenbMPLW9X76hH76oQ== + dependencies: + readable-web-to-node-stream "^2.0.0" + strtok3 "^6.0.3" + token-types "^2.0.0" + typedarray-to-buffer "^3.1.5" + file-uri-to-path@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" @@ -3659,7 +3686,7 @@ iconv-lite@0.4.24, iconv-lite@^0.4.24, iconv-lite@^0.4.4: dependencies: safer-buffer ">= 2.1.2 < 3" -ieee754@^1.1.4: +ieee754@^1.1.13, ieee754@^1.1.4: version "1.1.13" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== @@ -5698,6 +5725,11 @@ pbkdf2@^3.0.3: safe-buffer "^5.0.1" sha.js "^2.4.8" +peek-readable@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/peek-readable/-/peek-readable-3.1.0.tgz#250b08b7de09db8573d7fd8ea475215bbff14348" + integrity sha512-KGuODSTV6hcgdZvDrIDBUkN0utcAVj1LL7FfGbM0viKTtCHmtZcuEJ+lGqsp0fTFkGqesdtemV2yUSMeyy3ddA== + performance-now@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" @@ -6028,6 +6060,11 @@ readable-stream@1.1.x: isarray "0.0.1" string_decoder "~0.10.x" +readable-web-to-node-stream@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/readable-web-to-node-stream/-/readable-web-to-node-stream-2.0.0.tgz#751e632f466552ac0d5c440cc01470352f93c4b7" + integrity sha512-+oZJurc4hXpaaqsN68GoZGQAQIA3qr09Or4fqEsargABnbe5Aau8hFn6ISVleT3cpY/0n/8drn7huyyEvTbghA== + readdirp@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-2.2.1.tgz#0e87622a3325aa33e892285caf8b4e846529a525" @@ -6841,6 +6878,15 @@ strip-json-comments@~2.0.1: resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" integrity sha1-PFMZQukIwml8DsNEhYwobHygpgo= +strtok3@^6.0.3: + version "6.0.4" + resolved "https://registry.yarnpkg.com/strtok3/-/strtok3-6.0.4.tgz#ede0d20fde5aa9fda56417c3558eaafccc724694" + integrity sha512-rqWMKwsbN9APU47bQTMEYTPcwdpKDtmf1jVhHzNW2cL1WqAxaM9iBb9t5P2fj+RV2YsErUWgQzHD5JwV0uCTEQ== + dependencies: + "@tokenizer/token" "^0.1.1" + "@types/debug" "^4.1.5" + peek-readable "^3.1.0" + superagent@^3.8.3: version "3.8.3" resolved "https://registry.yarnpkg.com/superagent/-/superagent-3.8.3.tgz#460ea0dbdb7d5b11bc4f78deba565f86a178e128" @@ -7103,6 +7149,14 @@ toidentifier@1.0.0: resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553" integrity sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw== +token-types@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/token-types/-/token-types-2.0.0.tgz#b23618af744818299c6fbf125e0fdad98bab7e85" + integrity sha512-WWvu8sGK8/ZmGusekZJJ5NM6rRVTTDO7/bahz4NGiSDb/XsmdYBn6a1N/bymUHuWYTWeuLUg98wUzvE4jPdCZw== + dependencies: + "@tokenizer/token" "^0.1.0" + ieee754 "^1.1.13" + tough-cookie@^2.3.3: version "2.4.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781" From 3da16baeaec9a8a89f85c6924087e688422be376 Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sat, 17 Oct 2020 16:24:30 +0200 Subject: [PATCH 10/24] FilesystemBackend: Implement `deleteFile` and `getFileURL`. We use `fs.unlink` instead of `fs.rm`, as the latter is only available in the fsPromises API since Node 14.14 Signed-off-by: David Mehren --- src/media/backends/filesystem-backend.ts | 33 +++++++++++++++++------- src/media/media.module.ts | 11 ++++++-- src/media/media.service.ts | 4 ++- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/media/backends/filesystem-backend.ts b/src/media/backends/filesystem-backend.ts index 37cb440dd..88c8f7fce 100644 --- a/src/media/backends/filesystem-backend.ts +++ b/src/media/backends/filesystem-backend.ts @@ -1,26 +1,39 @@ -import { MediaBackend } from '../media-backend.interface'; -import { BackendData } from '../media-upload.entity'; +import { Injectable } from '@nestjs/common'; import { promises as fs } from 'fs'; import { join } from 'path'; +import { ConsoleLoggerService } from '../../logger/console-logger.service'; +import { MediaBackend } from '../media-backend.interface'; +import { BackendData } from '../media-upload.entity'; +@Injectable() export class FilesystemBackend implements MediaBackend { + constructor(private readonly logger: ConsoleLoggerService) { + this.logger.setContext(FilesystemBackend.name); + } + async saveFile( buffer: Buffer, fileName: string, ): Promise<[string, BackendData]> { - // TODO: Get uploads directory from config - const uploadDirectory = './uploads'; - // TODO: Add server address to url - const filePath = join(uploadDirectory, fileName); + const filePath = FilesystemBackend.getFilePath(fileName); + this.logger.debug(`Writing file to: ${filePath}`, 'saveFile'); await fs.writeFile(filePath, buffer, null); return ['/' + filePath, null]; } - deleteFile(fileName: string, backendData: BackendData): Promise { - return Promise.resolve(undefined); + async deleteFile(fileName: string, backendData: BackendData): Promise { + return fs.unlink(FilesystemBackend.getFilePath(fileName)); } - getFileURL(fileNam: string, backendData: BackendData): Promise { - return Promise.resolve(''); + getFileURL(fileName: string, backendData: BackendData): Promise { + const filePath = FilesystemBackend.getFilePath(fileName); + // TODO: Add server address to url + return Promise.resolve('/' + filePath); + } + + private static getFilePath(fileName: string): string { + // TODO: Get uploads directory from config + const uploadDirectory = './uploads'; + return join(uploadDirectory, fileName); } } diff --git a/src/media/media.module.ts b/src/media/media.module.ts index f791218e4..d24cb2bcb 100644 --- a/src/media/media.module.ts +++ b/src/media/media.module.ts @@ -1,13 +1,20 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; +import { LoggerModule } from '../logger/logger.module'; import { NotesModule } from '../notes/notes.module'; import { UsersModule } from '../users/users.module'; +import { FilesystemBackend } from './backends/filesystem-backend'; import { MediaUpload } from './media-upload.entity'; import { MediaService } from './media.service'; @Module({ - imports: [TypeOrmModule.forFeature([MediaUpload]), NotesModule, UsersModule], - providers: [MediaService], + imports: [ + TypeOrmModule.forFeature([MediaUpload]), + NotesModule, + UsersModule, + LoggerModule, + ], + providers: [MediaService, FilesystemBackend], exports: [MediaService], }) export class MediaModule {} diff --git a/src/media/media.service.ts b/src/media/media.service.ts index c8dfa2c3e..3beec6f42 100644 --- a/src/media/media.service.ts +++ b/src/media/media.service.ts @@ -1,4 +1,5 @@ import { Injectable } from '@nestjs/common'; +import { ModuleRef } from '@nestjs/core'; import { InjectRepository } from '@nestjs/typeorm'; import * as FileType from 'file-type'; import { Repository } from 'typeorm'; @@ -16,6 +17,7 @@ export class MediaService { private mediaUploadRepository: Repository, private notesService: NotesService, private usersService: UsersService, + private moduleRef: ModuleRef, ) {} public async saveFile(file: MulterFile, username: string, noteId: string) { @@ -35,7 +37,7 @@ export class MediaService { fileTypeResult.ext, BackendType.FILEYSTEM, ); - const backend = new FilesystemBackend(); + const backend = this.moduleRef.get(FilesystemBackend); const [url, backendData] = await backend.saveFile( file.buffer, mediaUpload.id, From dea3c1d3932d295931ad6bf8b019ddfe54506f01 Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sat, 17 Oct 2020 16:44:00 +0200 Subject: [PATCH 11/24] MediaController: Get parent note from `HedgeDoc-Note` header Signed-off-by: David Mehren --- src/api/public/media/media.controller.ts | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/api/public/media/media.controller.ts b/src/api/public/media/media.controller.ts index 4322aaf4e..ce82247dc 100644 --- a/src/api/public/media/media.controller.ts +++ b/src/api/public/media/media.controller.ts @@ -1,5 +1,6 @@ import { Controller, + Headers, Post, UploadedFile, UseInterceptors, @@ -8,26 +9,32 @@ import { FileInterceptor } from '@nestjs/platform-express'; import { ConsoleLoggerService } from '../../../logger/console-logger.service'; import { MediaService } from '../../../media/media.service'; import { MulterFile } from '../../../media/multer-file.interface'; +import { NotesService } from '../../../notes/notes.service'; @Controller('media') export class MediaController { constructor( private readonly logger: ConsoleLoggerService, private mediaService: MediaService, + private notesService: NotesService, ) { this.logger.setContext(MediaController.name); } @Post('upload') @UseInterceptors(FileInterceptor('file')) - async uploadImage(@UploadedFile() file: MulterFile) { - this.logger.debug('Recieved file: ' + file.originalname); - //TODO: Get user and note from request - const url = await this.mediaService.saveFile( - file, - 'hardcoded', - 'hardcoded', + async uploadImage( + @UploadedFile() file: MulterFile, + @Headers('HedgeDoc-Note') noteId: string, + ) { + //TODO: Get user from request + const username = 'hardcoded'; + this.logger.debug( + `Recieved filename '${file.originalname}' for note '${noteId}' from user '${username}'`, + 'uploadImage', ); + const note = await this.notesService.getNoteByIdOrAlias(noteId); + const url = await this.mediaService.saveFile(file, username, note.id); return { link: url, }; From ec8cf6d33e6e0f499758b2bb56dfb5a060f33916 Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sat, 17 Oct 2020 18:47:10 +0200 Subject: [PATCH 12/24] NotesService: Throw `NotInDBError` when the note wasn't found Signed-off-by: David Mehren --- src/errors/errors.ts | 7 +++++++ src/notes/notes.service.ts | 19 ++++++++++++++++--- test/public-api/notes.e2e-spec.ts | 3 ++- 3 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 src/errors/errors.ts diff --git a/src/errors/errors.ts b/src/errors/errors.ts new file mode 100644 index 000000000..6e1d00da6 --- /dev/null +++ b/src/errors/errors.ts @@ -0,0 +1,7 @@ +export class NotInDBError extends Error { + name = 'NotInDBError'; +} + +export class ClientError extends Error { + name = 'ClientError'; +} diff --git a/src/notes/notes.service.ts b/src/notes/notes.service.ts index 2233fc87f..f6d5e4994 100644 --- a/src/notes/notes.service.ts +++ b/src/notes/notes.service.ts @@ -1,6 +1,7 @@ import { forwardRef, Inject, Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; +import { NotInDBError } from '../errors/errors'; import { ConsoleLoggerService } from '../logger/console-logger.service'; import { Revision } from '../revisions/revision.entity'; import { RevisionsService } from '../revisions/revisions.service'; @@ -132,8 +133,19 @@ export class NotesService { } async getNoteByIdOrAlias(noteIdOrAlias: string): Promise { + this.logger.debug( + `Trying to find note '${noteIdOrAlias}'`, + 'getNoteByIdOrAlias', + ); const note = await this.noteRepository.findOne({ - where: [{ id: noteIdOrAlias }, { alias: noteIdOrAlias }], + where: [ + { + id: noteIdOrAlias, + }, + { + alias: noteIdOrAlias, + }, + ], relations: [ 'authorColors', 'owner', @@ -142,8 +154,9 @@ export class NotesService { ], }); if (note === undefined) { - //TODO: Improve error handling - throw new Error('Note not found'); + throw new NotInDBError( + `Note with id/alias '${noteIdOrAlias}' not found.`, + ); } return note; } diff --git a/test/public-api/notes.e2e-spec.ts b/test/public-api/notes.e2e-spec.ts index a27e0ef7b..09678dbb6 100644 --- a/test/public-api/notes.e2e-spec.ts +++ b/test/public-api/notes.e2e-spec.ts @@ -3,6 +3,7 @@ import { Test } from '@nestjs/testing'; import { TypeOrmModule } from '@nestjs/typeorm'; import * as request from 'supertest'; import { PublicApiModule } from '../../src/api/public/public-api.module'; +import { NotInDBError } from '../../src/errors/errors'; import { GroupsModule } from '../../src/groups/groups.module'; import { LoggerModule } from '../../src/logger/logger.module'; import { NotesModule } from '../../src/notes/notes.module'; @@ -82,7 +83,7 @@ describe('Notes', () => { .delete('/notes/test3') .expect(200); return expect(notesService.getNoteByIdOrAlias('test3')).rejects.toEqual( - Error('Note not found'), + new NotInDBError("Note with id/alias 'test3' not found."), ); }); From 7997a0955ad3cbf2f3dbd84940de44466bbc6e20 Mon Sep 17 00:00:00 2001 From: David Mehren Date: Mon, 12 Oct 2020 21:46:53 +0200 Subject: [PATCH 13/24] UsersService: Add methods to find, create and delete users Signed-off-by: David Mehren --- src/api/public/me/me.controller.ts | 6 +++-- src/users/user.entity.ts | 16 ++++++++++++ src/users/users.service.ts | 40 ++++++++++++++++++++++-------- 3 files changed, 49 insertions(+), 13 deletions(-) diff --git a/src/api/public/me/me.controller.ts b/src/api/public/me/me.controller.ts index 645fcaa0b..92eb52ba7 100644 --- a/src/api/public/me/me.controller.ts +++ b/src/api/public/me/me.controller.ts @@ -29,8 +29,10 @@ export class MeController { } @Get() - getMe(): UserInfoDto { - return this.usersService.getUserInfo(); + async getMe(): Promise { + return this.usersService.toUserDto( + await this.usersService.getUserByUsername('hardcoded'), + ); } @Get('history') diff --git a/src/users/user.entity.ts b/src/users/user.entity.ts index 939ffd11c..9b20e6444 100644 --- a/src/users/user.entity.ts +++ b/src/users/user.entity.ts @@ -48,4 +48,20 @@ export class User { identity => identity.user, ) identities: Identity[]; + + // eslint-disable-next-line @typescript-eslint/no-empty-function + private constructor() {} + + public static create( + userName: string, + displayName: string, + ): Pick< + User, + 'userName' | 'displayName' | 'ownedNotes' | 'authToken' | 'identities' + > { + const newUser = new User(); + newUser.userName = userName; + newUser.displayName = displayName; + return newUser; + } } diff --git a/src/users/users.service.ts b/src/users/users.service.ts index cc4fa1880..193ca397d 100644 --- a/src/users/users.service.ts +++ b/src/users/users.service.ts @@ -1,26 +1,44 @@ import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { NotInDBError } from '../errors/errors'; import { ConsoleLoggerService } from '../logger/console-logger.service'; import { UserInfoDto } from './user-info.dto'; import { User } from './user.entity'; @Injectable() export class UsersService { - constructor(private readonly logger: ConsoleLoggerService) { + constructor( + private readonly logger: ConsoleLoggerService, + @InjectRepository(User) private userRepository: Repository, + ) { this.logger.setContext(UsersService.name); } - getUserInfo(): UserInfoDto { - //TODO: Use the database - this.logger.warn('Using hardcoded data!'); - return { - displayName: 'foo', - userName: 'fooUser', - email: 'foo@example.com', - photo: '', - }; + createUser(userName: string, displayName: string): Promise { + const user = User.create(userName, displayName); + return this.userRepository.save(user); } - getPhotoUrl(user: User) { + async deleteUser(userName: string) { + //TOOD: Handle owned notes and edits + const user = await this.userRepository.findOne({ + where: { userName: userName }, + }); + await this.userRepository.delete(user); + } + + async getUserByUsername(userName: string): Promise { + const user = this.userRepository.findOne({ + where: { userName: userName }, + }); + if (user === undefined) { + throw new NotInDBError(`User with username '${userName}' not found`); + } + return user; + } + + getPhotoUrl(user: User): string { if (user.photo) { return user.photo; } else { From 8e662167dc30fb82dbaf42fe7285b0a19808d995 Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sat, 17 Oct 2020 18:50:53 +0200 Subject: [PATCH 14/24] MediaService: Improve error handling and logging Add debug logging to `saveFile` method and throw the proper errors when problems with the mime type are encountered Signed-off-by: David Mehren --- src/media/media.service.ts | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/media/media.service.ts b/src/media/media.service.ts index 3beec6f42..c9fc6ccd9 100644 --- a/src/media/media.service.ts +++ b/src/media/media.service.ts @@ -3,6 +3,8 @@ import { ModuleRef } from '@nestjs/core'; import { InjectRepository } from '@nestjs/typeorm'; import * as FileType from 'file-type'; import { Repository } from 'typeorm'; +import { ClientError } from '../errors/errors'; +import { ConsoleLoggerService } from '../logger/console-logger.service'; import { NotesService } from '../notes/notes.service'; import { UsersService } from '../users/users.service'; import { BackendType } from './backends/backend-type.enum'; @@ -13,22 +15,34 @@ import { MulterFile } from './multer-file.interface'; @Injectable() export class MediaService { constructor( + private readonly logger: ConsoleLoggerService, @InjectRepository(MediaUpload) private mediaUploadRepository: Repository, private notesService: NotesService, private usersService: UsersService, private moduleRef: ModuleRef, - ) {} + ) { + this.logger.setContext(MediaService.name); + } + + private static isAllowedMimeType(mimeType: string): boolean { + //TODO: Which mimetypes are allowed? + return true; + } public async saveFile(file: MulterFile, username: string, noteId: string) { + this.logger.debug( + `Saving '${file.originalname}' for note '${noteId}' and user '${username}'`, + 'saveFile', + ); const note = await this.notesService.getNoteByIdOrAlias(noteId); const user = await this.usersService.getUserByUsername(username); const fileTypeResult = await FileType.fromBuffer(file.buffer); if (!fileTypeResult) { - throw new Error('Could not detect file type.'); + throw new ClientError('Could not detect file type.'); } if (!MediaService.isAllowedMimeType(fileTypeResult.mime)) { - throw new Error('MIME type not allowed'); + throw new ClientError('MIME type not allowed.'); } //TODO: Choose backend according to config const mediaUpload = MediaUpload.create( @@ -37,6 +51,7 @@ export class MediaService { fileTypeResult.ext, BackendType.FILEYSTEM, ); + this.logger.debug(`Generated filename: '${mediaUpload.id}'`, 'saveFile'); const backend = this.moduleRef.get(FilesystemBackend); const [url, backendData] = await backend.saveFile( file.buffer, @@ -46,9 +61,4 @@ export class MediaService { await this.mediaUploadRepository.save(mediaUpload); return url; } - - private static isAllowedMimeType(mimeType: string): boolean { - //TODO: Which mimetypes are allowed? - return true; - } } From 4cd80a321226c2b9ae9dd58e9326c79043038134 Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sat, 17 Oct 2020 18:51:29 +0200 Subject: [PATCH 15/24] MediaController: Handle errors when trying to save file Signed-off-by: David Mehren --- src/api/public/media/media.controller.ts | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/api/public/media/media.controller.ts b/src/api/public/media/media.controller.ts index ce82247dc..a2dc65ac4 100644 --- a/src/api/public/media/media.controller.ts +++ b/src/api/public/media/media.controller.ts @@ -1,4 +1,5 @@ import { + BadRequestException, Controller, Headers, Post, @@ -6,6 +7,7 @@ import { UseInterceptors, } from '@nestjs/common'; import { FileInterceptor } from '@nestjs/platform-express'; +import { ClientError, NotInDBError } from '../../../errors/errors'; import { ConsoleLoggerService } from '../../../logger/console-logger.service'; import { MediaService } from '../../../media/media.service'; import { MulterFile } from '../../../media/multer-file.interface'; @@ -33,10 +35,16 @@ export class MediaController { `Recieved filename '${file.originalname}' for note '${noteId}' from user '${username}'`, 'uploadImage', ); - const note = await this.notesService.getNoteByIdOrAlias(noteId); - const url = await this.mediaService.saveFile(file, username, note.id); - return { - link: url, - }; + try { + const url = await this.mediaService.saveFile(file, username, noteId); + return { + link: url, + }; + } catch (e) { + if (e instanceof ClientError || e instanceof NotInDBError) { + throw new BadRequestException(e.message); + } + throw e; + } } } From ed142815e3b5ccecf7631bc7d5ad407bf78f7ba2 Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sat, 17 Oct 2020 20:21:22 +0200 Subject: [PATCH 16/24] Add various missing imports and provider ovverides in unit tests Signed-off-by: David Mehren --- src/api/public/media/media.controller.spec.ts | 32 ++++++++++++++- src/media/media.service.spec.ts | 40 ++++++++++++++++++- src/users/users.service.spec.ts | 15 ++++++- 3 files changed, 81 insertions(+), 6 deletions(-) diff --git a/src/api/public/media/media.controller.spec.ts b/src/api/public/media/media.controller.spec.ts index fa364d9b5..c5cf4914e 100644 --- a/src/api/public/media/media.controller.spec.ts +++ b/src/api/public/media/media.controller.spec.ts @@ -1,5 +1,16 @@ import { Test, TestingModule } from '@nestjs/testing'; +import { getRepositoryToken } from '@nestjs/typeorm'; import { LoggerModule } from '../../../logger/logger.module'; +import { MediaUpload } from '../../../media/media-upload.entity'; +import { MediaModule } from '../../../media/media.module'; +import { AuthorColor } from '../../../notes/author-color.entity'; +import { Note } from '../../../notes/note.entity'; +import { NotesModule } from '../../../notes/notes.module'; +import { Authorship } from '../../../revisions/authorship.entity'; +import { Revision } from '../../../revisions/revision.entity'; +import { AuthToken } from '../../../users/auth-token.entity'; +import { Identity } from '../../../users/identity.entity'; +import { User } from '../../../users/user.entity'; import { MediaController } from './media.controller'; describe('Media Controller', () => { @@ -8,8 +19,25 @@ describe('Media Controller', () => { beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ controllers: [MediaController], - imports: [LoggerModule], - }).compile(); + imports: [LoggerModule, MediaModule, NotesModule], + }) + .overrideProvider(getRepositoryToken(AuthorColor)) + .useValue({}) + .overrideProvider(getRepositoryToken(Authorship)) + .useValue({}) + .overrideProvider(getRepositoryToken(AuthToken)) + .useValue({}) + .overrideProvider(getRepositoryToken(Identity)) + .useValue({}) + .overrideProvider(getRepositoryToken(MediaUpload)) + .useValue({}) + .overrideProvider(getRepositoryToken(Note)) + .useValue({}) + .overrideProvider(getRepositoryToken(Revision)) + .useValue({}) + .overrideProvider(getRepositoryToken(User)) + .useValue({}) + .compile(); controller = module.get(MediaController); }); diff --git a/src/media/media.service.spec.ts b/src/media/media.service.spec.ts index 5009e4a57..19369f72d 100644 --- a/src/media/media.service.spec.ts +++ b/src/media/media.service.spec.ts @@ -1,4 +1,16 @@ import { Test, TestingModule } from '@nestjs/testing'; +import { getRepositoryToken } from '@nestjs/typeorm'; +import { LoggerModule } from '../logger/logger.module'; +import { AuthorColor } from '../notes/author-color.entity'; +import { Note } from '../notes/note.entity'; +import { NotesModule } from '../notes/notes.module'; +import { Authorship } from '../revisions/authorship.entity'; +import { Revision } from '../revisions/revision.entity'; +import { AuthToken } from '../users/auth-token.entity'; +import { Identity } from '../users/identity.entity'; +import { User } from '../users/user.entity'; +import { UsersModule } from '../users/users.module'; +import { MediaUpload } from './media-upload.entity'; import { MediaService } from './media.service'; describe('MediaService', () => { @@ -6,8 +18,32 @@ describe('MediaService', () => { beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ - providers: [MediaService], - }).compile(); + providers: [ + MediaService, + { + provide: getRepositoryToken(MediaUpload), + useValue: {}, + }, + ], + imports: [LoggerModule, NotesModule, UsersModule], + }) + .overrideProvider(getRepositoryToken(AuthorColor)) + .useValue({}) + .overrideProvider(getRepositoryToken(MediaUpload)) + .useValue({}) + .overrideProvider(getRepositoryToken(Authorship)) + .useValue({}) + .overrideProvider(getRepositoryToken(AuthToken)) + .useValue({}) + .overrideProvider(getRepositoryToken(Identity)) + .useValue({}) + .overrideProvider(getRepositoryToken(Note)) + .useValue({}) + .overrideProvider(getRepositoryToken(Revision)) + .useValue({}) + .overrideProvider(getRepositoryToken(User)) + .useValue({}) + .compile(); service = module.get(MediaService); }); diff --git a/src/users/users.service.spec.ts b/src/users/users.service.spec.ts index 3adc1fd92..4b43f74c4 100644 --- a/src/users/users.service.spec.ts +++ b/src/users/users.service.spec.ts @@ -1,5 +1,7 @@ import { Test, TestingModule } from '@nestjs/testing'; +import { getRepositoryToken } from '@nestjs/typeorm'; import { LoggerModule } from '../logger/logger.module'; +import { User } from './user.entity'; import { UsersService } from './users.service'; describe('UsersService', () => { @@ -7,9 +9,18 @@ describe('UsersService', () => { beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ - providers: [UsersService], + providers: [ + UsersService, + { + provide: getRepositoryToken(User), + useValue: {}, + }, + ], imports: [LoggerModule], - }).compile(); + }) + .overrideProvider(getRepositoryToken(User)) + .useValue({}) + .compile(); service = module.get(UsersService); }); From ffef4425f5f6d4c023ac5ae78e40e84cb24f4b5b Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sat, 17 Oct 2020 20:50:20 +0200 Subject: [PATCH 17/24] MediaService: Only allow upload of common image formats and PDFs Signed-off-by: David Mehren --- src/media/media.service.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/media/media.service.ts b/src/media/media.service.ts index c9fc6ccd9..197a1e44c 100644 --- a/src/media/media.service.ts +++ b/src/media/media.service.ts @@ -26,8 +26,22 @@ export class MediaService { } private static isAllowedMimeType(mimeType: string): boolean { - //TODO: Which mimetypes are allowed? - return true; + const allowedTypes = [ + 'application/pdf', + 'image/apng', + 'image/bmp', + 'image/gif', + 'image/heif', + 'image/heic', + 'image/heif-sequence', + 'image/heic-sequence', + 'image/jpeg', + 'image/png', + 'image/svg+xml', + 'image/tiff', + 'image/webp', + ]; + return allowedTypes.includes(mimeType); } public async saveFile(file: MulterFile, username: string, noteId: string) { From e0f8031fabf73a4e882047c72c72576d1022e1f3 Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sat, 17 Oct 2020 20:55:40 +0200 Subject: [PATCH 18/24] Public API: Update `/media/upload` route with supported content-types Signed-off-by: David Mehren --- docs/dev/public_api.yml | 65 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/docs/dev/public_api.yml b/docs/dev/public_api.yml index 181b80a10..be897596f 100644 --- a/docs/dev/public_api.yml +++ b/docs/dev/public_api.yml @@ -458,19 +458,74 @@ paths: post: tags: - media - summary: Uploads an image to the backend storage - description: Uploads an image to be processed by the backend. + summary: Uploads a media file to the backend storage + description: Uploads a file to be processed by the backend. requestBody: required: true - description: The binary image to upload. + description: The binary file to upload. content: - image/*: + application/pdf: schema: type: string format: binary + image/apng: + schema: + type: string + format: binary + image/bmp: + schema: + type: string + format: binary + image/gif: + schema: + type: string + format: binary + image/heif: + schema: + type: string + format: binary + image/heic: + schema: + type: string + format: binary + image/heif-sequence: + schema: + type: string + format: binary + image/heic-sequence: + schema: + type: string + format: binary + image/jpeg: + schema: + type: string + format: binary + image/png: + schema: + type: string + format: binary + image/svg+xml: + schema: + type: string + format: binary + image/tiff: + schema: + type: string + format: binary + image/webp: + schema: + type: string + format: binary + parameters: + - in: header + name: HedgeDoc-Note + schema: + type: string + required: true + description: ID or alias of the parent note responses: '200': - description: The image was uploaded successfully. + description: The file was uploaded successfully. content: application/json: schema: From 16b5f3a5c82f625cfd94a3357cd43c13494d474a Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sat, 17 Oct 2020 20:58:10 +0200 Subject: [PATCH 19/24] Use `POST /media` for file upload The old `/media/upload` subpath does not follow the convention of REST APIs. Signed-off-by: David Mehren --- docs/dev/public_api.yml | 2 +- src/api/public/media/media.controller.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dev/public_api.yml b/docs/dev/public_api.yml index be897596f..62a131b85 100644 --- a/docs/dev/public_api.yml +++ b/docs/dev/public_api.yml @@ -454,7 +454,7 @@ paths: content: text/plain: example: my-note - /media/upload: + /media: post: tags: - media diff --git a/src/api/public/media/media.controller.ts b/src/api/public/media/media.controller.ts index a2dc65ac4..9627a5054 100644 --- a/src/api/public/media/media.controller.ts +++ b/src/api/public/media/media.controller.ts @@ -23,7 +23,7 @@ export class MediaController { this.logger.setContext(MediaController.name); } - @Post('upload') + @Post() @UseInterceptors(FileInterceptor('file')) async uploadImage( @UploadedFile() file: MulterFile, From 6e6ab84391d004117ae0f5fe127d1d2a996b376b Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sat, 17 Oct 2020 21:52:59 +0200 Subject: [PATCH 20/24] UsersService: Wait for the DB to find a user Signed-off-by: David Mehren --- src/users/users.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/users/users.service.ts b/src/users/users.service.ts index 193ca397d..2c596905d 100644 --- a/src/users/users.service.ts +++ b/src/users/users.service.ts @@ -29,7 +29,7 @@ export class UsersService { } async getUserByUsername(userName: string): Promise { - const user = this.userRepository.findOne({ + const user = await this.userRepository.findOne({ where: { userName: userName }, }); if (user === undefined) { From db869418d4f4d19b3a64717b3594e52a5e38892a Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sat, 17 Oct 2020 21:53:34 +0200 Subject: [PATCH 21/24] FilesystemBackend: ESLint fixes Signed-off-by: David Mehren --- src/media/backends/filesystem-backend.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/media/backends/filesystem-backend.ts b/src/media/backends/filesystem-backend.ts index 88c8f7fce..5069fa969 100644 --- a/src/media/backends/filesystem-backend.ts +++ b/src/media/backends/filesystem-backend.ts @@ -21,11 +21,11 @@ export class FilesystemBackend implements MediaBackend { return ['/' + filePath, null]; } - async deleteFile(fileName: string, backendData: BackendData): Promise { + async deleteFile(fileName: string, _: BackendData): Promise { return fs.unlink(FilesystemBackend.getFilePath(fileName)); } - getFileURL(fileName: string, backendData: BackendData): Promise { + getFileURL(fileName: string, _: BackendData): Promise { const filePath = FilesystemBackend.getFilePath(fileName); // TODO: Add server address to url return Promise.resolve('/' + filePath); From 9e7e15a20aa044d98fddb676e16da0ad56317b91 Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sat, 17 Oct 2020 21:54:08 +0200 Subject: [PATCH 22/24] MediaService: Implement delete feature Signed-off-by: David Mehren --- src/errors/errors.ts | 4 ++++ src/media/media.service.ts | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/errors/errors.ts b/src/errors/errors.ts index 6e1d00da6..5f7885ce1 100644 --- a/src/errors/errors.ts +++ b/src/errors/errors.ts @@ -5,3 +5,7 @@ export class NotInDBError extends Error { export class ClientError extends Error { name = 'ClientError'; } + +export class PermissionError extends Error { + name = 'PermissionError'; +} diff --git a/src/media/media.service.ts b/src/media/media.service.ts index 197a1e44c..92c96ee49 100644 --- a/src/media/media.service.ts +++ b/src/media/media.service.ts @@ -3,7 +3,7 @@ import { ModuleRef } from '@nestjs/core'; import { InjectRepository } from '@nestjs/typeorm'; import * as FileType from 'file-type'; import { Repository } from 'typeorm'; -import { ClientError } from '../errors/errors'; +import { ClientError, NotInDBError, PermissionError } from '../errors/errors'; import { ConsoleLoggerService } from '../logger/console-logger.service'; import { NotesService } from '../notes/notes.service'; import { UsersService } from '../users/users.service'; @@ -75,4 +75,36 @@ export class MediaService { await this.mediaUploadRepository.save(mediaUpload); return url; } + + public async deleteFile(filename: string, username: string) { + this.logger.debug( + `Deleting '${filename}' for user '${username}'`, + 'deleteFile', + ); + const mediaUpload = await this.findUploadByFilename(filename); + if (mediaUpload.user.userName !== username) { + this.logger.warn( + `${username} tried to delete '${filename}', but is not the owner`, + 'deleteFile', + ); + throw new PermissionError( + `File '${filename}' is not owned by '${username}'`, + ); + } + const backend = this.moduleRef.get(FilesystemBackend); + await backend.deleteFile(filename, mediaUpload.backendData); + await this.mediaUploadRepository.remove(mediaUpload); + } + + public async findUploadByFilename(filename: string): Promise { + const mediaUpload = await this.mediaUploadRepository.findOne(filename, { + relations: ['user'], + }); + if (mediaUpload === undefined) { + throw new NotInDBError( + `MediaUpload with filename '${filename}' not found`, + ); + } + return mediaUpload; + } } From 3686685f082bb2969e5ea4966660da8788ad76d0 Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sat, 17 Oct 2020 21:54:44 +0200 Subject: [PATCH 23/24] MediaController: Add `DELETE /{filename}` route Signed-off-by: David Mehren --- src/api/public/media/media.controller.ts | 29 ++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/api/public/media/media.controller.ts b/src/api/public/media/media.controller.ts index 9627a5054..109a2d1c3 100644 --- a/src/api/public/media/media.controller.ts +++ b/src/api/public/media/media.controller.ts @@ -1,13 +1,21 @@ import { BadRequestException, Controller, + Delete, Headers, + NotFoundException, + Param, Post, + UnauthorizedException, UploadedFile, UseInterceptors, } from '@nestjs/common'; import { FileInterceptor } from '@nestjs/platform-express'; -import { ClientError, NotInDBError } from '../../../errors/errors'; +import { + ClientError, + NotInDBError, + PermissionError, +} from '../../../errors/errors'; import { ConsoleLoggerService } from '../../../logger/console-logger.service'; import { MediaService } from '../../../media/media.service'; import { MulterFile } from '../../../media/multer-file.interface'; @@ -25,7 +33,7 @@ export class MediaController { @Post() @UseInterceptors(FileInterceptor('file')) - async uploadImage( + async uploadMedia( @UploadedFile() file: MulterFile, @Headers('HedgeDoc-Note') noteId: string, ) { @@ -47,4 +55,21 @@ export class MediaController { throw e; } } + + @Delete(':filename') + async deleteMedia(@Param('filename') filename: string) { + //TODO: Get user from request + const username = 'hardcoded'; + try { + await this.mediaService.deleteFile(filename, username); + } catch (e) { + if (e instanceof PermissionError) { + throw new UnauthorizedException(e.message); + } + if (e instanceof NotInDBError) { + throw new NotFoundException(e.message); + } + throw e; + } + } } From a728866ebb2ade33e5ae4d0921b867743b787f98 Mon Sep 17 00:00:00 2001 From: David Mehren Date: Sat, 17 Oct 2020 21:55:05 +0200 Subject: [PATCH 24/24] Public API: Add media deletion Signed-off-by: David Mehren --- docs/dev/public_api.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/dev/public_api.yml b/docs/dev/public_api.yml index 62a131b85..fecc46375 100644 --- a/docs/dev/public_api.yml +++ b/docs/dev/public_api.yml @@ -537,6 +537,29 @@ paths: "$ref": "#/components/responses/UnauthorizedError" '403': "$ref": "#/components/responses/ForbiddenError" + /media/{filename}: + delete: + tags: + - media + summary: Delete the specified file + operationId: deleteMedia + parameters: + - name: filename + in: path + required: true + description: The name of the file to be deleted. + content: + text/plain: + example: e18d1b83e1821128615bad849ad0655a.jpg + responses: + '204': + "$ref": "#/components/responses/SuccessfullyDeleted" + '401': + "$ref": "#/components/responses/UnauthorizedError" + '403': + "$ref": "#/components/responses/ForbiddenError" + '404': + "$ref": "#/components/responses/NotFoundError" /monitoring: get: tags: