refactor: save ydoc state in the database, so it can be restored easier

By storing the ydoc state in the database we can reconnect lost clients easier
and enable offline editing because we continue using the crdt data that has been
used by the client before the connection loss.

Signed-off-by: Tilman Vatteroth <git@tilmanvatteroth.de>

backend/src/realtime/realtime-note/realtime-connection.spec.ts

@@ -27,11 +27,7 @@ jest.mock(
     ({
       ...jest.requireActual('@hedgedoc/commons'),
       // eslint-disable-next-line @typescript-eslint/naming-convention
-      YDocSyncServerAdapter: jest.fn(() =>
-        Mock.of<YDocSyncServerAdapter>({
-          setYDoc: jest.fn(),
-        }),
-      ),
+      YDocSyncServerAdapter: jest.fn(() => Mock.of<YDocSyncServerAdapter>({})),
     } as Record<string, unknown>),
 );
 
@@ -86,9 +82,7 @@ describe('websocket connection', () => {
   });
 
   it('returns the correct sync adapter', () => {
-    const yDocSyncServerAdapter = Mock.of<YDocSyncServerAdapter>({
-      setYDoc: jest.fn(),
-    });
+    const yDocSyncServerAdapter = Mock.of<YDocSyncServerAdapter>({});
     jest
       .spyOn(HedgeDocCommonsModule, 'YDocSyncServerAdapter')
       .mockImplementation(() => yDocSyncServerAdapter);

backend/src/realtime/realtime-note/realtime-connection.ts

@@ -41,8 +41,10 @@ export class RealtimeConnection {
     this.transporter.on('disconnected', () => {
       realtimeNote.removeClient(this);
     });
-    this.yDocSyncAdapter = new YDocSyncServerAdapter(this.transporter);
-    this.yDocSyncAdapter.setYDoc(realtimeNote.getRealtimeDoc());
+    this.yDocSyncAdapter = new YDocSyncServerAdapter(
+      this.transporter,
+      realtimeNote.getRealtimeDoc(),
+    );
     this.realtimeUserStateAdapter = new RealtimeUserStatusAdapter(
       this.user?.username ?? null,
       this.getDisplayName(),

backend/src/realtime/realtime-note/realtime-note-store.spec.ts

@@ -41,6 +41,23 @@ describe('RealtimeNoteStore', () => {
     expect(realtimeNoteConstructorSpy).toHaveBeenCalledWith(
       mockedNote,
       mockedContent,
+      undefined,
+    );
+    expect(realtimeNoteStore.find(mockedNoteId)).toBe(mockedRealtimeNote);
+    expect(realtimeNoteStore.getAllRealtimeNotes()).toStrictEqual([
+      mockedRealtimeNote,
+    ]);
+  });
+
+  it("can create a new realtime note with a yjs state if it doesn't exist yet", () => {
+    const initialYjsState = [1, 2, 3];
+    expect(
+      realtimeNoteStore.create(mockedNote, mockedContent, initialYjsState),
+    ).toBe(mockedRealtimeNote);
+    expect(realtimeNoteConstructorSpy).toHaveBeenCalledWith(
+      mockedNote,
+      mockedContent,
+      initialYjsState,
     );
     expect(realtimeNoteStore.find(mockedNoteId)).toBe(mockedRealtimeNote);
     expect(realtimeNoteStore.getAllRealtimeNotes()).toStrictEqual([

backend/src/realtime/realtime-note/realtime-note-store.ts

@@ -16,15 +16,24 @@ export class RealtimeNoteStore {
    * Creates a new {@link RealtimeNote} for the given {@link Note} and memorizes it.
    *
    * @param note The note for which the realtime note should be created
-   * @param initialContent The initial content for the realtime note
+   * @param initialTextContent the initial text content of realtime doc
+   * @param initialYjsState the initial yjs state. If provided this will be used instead of the text content
    * @throws Error if there is already an realtime note for the given note.
    * @return The created realtime note
    */
-  public create(note: Note, initialContent: string): RealtimeNote {
+  public create(
+    note: Note,
+    initialTextContent: string,
+    initialYjsState?: number[],
+  ): RealtimeNote {
     if (this.noteIdToRealtimeNote.has(note.id)) {
       throw new Error(`Realtime note for note ${note.id} already exists.`);
     }
-    const realtimeNote = new RealtimeNote(note, initialContent);
+    const realtimeNote = new RealtimeNote(
+      note,
+      initialTextContent,
+      initialYjsState,
+    );
     realtimeNote.on('destroy', () => {
       this.noteIdToRealtimeNote.delete(note.id);
     });

backend/src/realtime/realtime-note/realtime-note.service.spec.ts

@@ -17,6 +17,7 @@ import { RealtimeNoteService } from './realtime-note.service';
 
 describe('RealtimeNoteService', () => {
   const mockedContent = 'mockedContent';
+  const mockedYjsState = [1, 2, 3];
   const mockedNoteId = 4711;
   let note: Note;
   let realtimeNote: RealtimeNote;
@@ -37,7 +38,10 @@ describe('RealtimeNoteService', () => {
     jest.useFakeTimers();
   });
 
-  function mockGetLatestRevision(latestRevisionExists: boolean) {
+  function mockGetLatestRevision(
+    latestRevisionExists: boolean,
+    hasYjsState = false,
+  ) {
     jest
       .spyOn(revisionsService, 'getLatestRevision')
       .mockImplementation((note: Note) =>
@@ -45,6 +49,7 @@ describe('RealtimeNoteService', () => {
           ? Promise.resolve(
               Mock.of<Revision>({
                 content: mockedContent,
+                ...(hasYjsState ? { yjsStateVector: mockedYjsState } : {}),
               }),
             )
           : Promise.reject('Revision for note mockedNoteId not found.'),
@@ -106,7 +111,32 @@ describe('RealtimeNoteService', () => {
     ).resolves.toBe(realtimeNote);
 
     expect(realtimeNoteStore.find).toHaveBeenCalledWith(mockedNoteId);
-    expect(realtimeNoteStore.create).toHaveBeenCalledWith(note, mockedContent);
+    expect(realtimeNoteStore.create).toHaveBeenCalledWith(
+      note,
+      mockedContent,
+      undefined,
+    );
+    expect(setIntervalSpy).not.toHaveBeenCalled();
+  });
+
+  it("creates a new realtime note with a yjs state if it doesn't exist yet", async () => {
+    mockGetLatestRevision(true, true);
+    jest.spyOn(realtimeNoteStore, 'find').mockImplementation(() => undefined);
+    jest
+      .spyOn(realtimeNoteStore, 'create')
+      .mockImplementation(() => realtimeNote);
+    mockedAppConfig.persistInterval = 0;
+
+    await expect(
+      realtimeNoteService.getOrCreateRealtimeNote(note),
+    ).resolves.toBe(realtimeNote);
+
+    expect(realtimeNoteStore.find).toHaveBeenCalledWith(mockedNoteId);
+    expect(realtimeNoteStore.create).toHaveBeenCalledWith(
+      note,
+      mockedContent,
+      mockedYjsState,
+    );
     expect(setIntervalSpy).not.toHaveBeenCalled();
   });
 
@@ -192,7 +222,11 @@ describe('RealtimeNoteService', () => {
       .mockImplementation(() => Promise.resolve(Mock.of<Revision>()));
 
     realtimeNote.emit('beforeDestroy');
-    expect(createRevisionSpy).toHaveBeenCalledWith(note, mockedContent);
+    expect(createRevisionSpy).toHaveBeenCalledWith(
+      note,
+      mockedContent,
+      expect.any(Array),
+    );
   });
 
   it('destroys every realtime note on application shutdown', () => {

backend/src/realtime/realtime-note/realtime-note.service.ts

@@ -43,6 +43,7 @@ export class RealtimeNoteService implements BeforeApplicationShutdown {
       .createRevision(
         realtimeNote.getNote(),
         realtimeNote.getRealtimeDoc().getCurrentContent(),
+        realtimeNote.getRealtimeDoc().encodeStateAsUpdate(),
       )
       .catch((reason) => this.logger.error(reason));
   }
@@ -68,9 +69,12 @@ export class RealtimeNoteService implements BeforeApplicationShutdown {
    * @return The created realtime note
    */
   private async createNewRealtimeNote(note: Note): Promise<RealtimeNote> {
-    const initialContent = (await this.revisionsService.getLatestRevision(note))
-      .content;
-    const realtimeNote = this.realtimeNoteStore.create(note, initialContent);
+    const lastRevision = await this.revisionsService.getLatestRevision(note);
+    const realtimeNote = this.realtimeNoteStore.create(
+      note,
+      lastRevision.content,
+      lastRevision.yjsStateVector ?? undefined,
+    );
     realtimeNote.on('beforeDestroy', () => {
       this.saveRealtimeNote(realtimeNote);
     });

backend/src/realtime/realtime-note/realtime-note.spec.ts

@@ -4,15 +4,12 @@
  * SPDX-License-Identifier: AGPL-3.0-only
  */
 import { MessageType, RealtimeDoc } from '@hedgedoc/commons';
-import * as hedgedocCommonsModule from '@hedgedoc/commons';
 import { Mock } from 'ts-mockery';
 
 import { Note } from '../../notes/note.entity';
 import { RealtimeNote } from './realtime-note';
 import { MockConnectionBuilder } from './test-utils/mock-connection';
 
-jest.mock('@hedgedoc/commons');
-
 describe('realtime note', () => {
   let mockedNote: Note;
 
@@ -40,18 +37,13 @@ describe('realtime note', () => {
     expect(sut.hasConnections()).toBeFalsy();
   });
 
-  it('creates a y-doc', () => {
+  it('creates a realtime doc', () => {
     const initialContent = 'nothing';
-    const mockedDoc = new RealtimeDoc(initialContent);
-    const docSpy = jest
-      .spyOn(hedgedocCommonsModule, 'RealtimeDoc')
-      .mockReturnValue(mockedDoc);
     const sut = new RealtimeNote(mockedNote, initialContent);
-    expect(docSpy).toHaveBeenCalledWith(initialContent);
-    expect(sut.getRealtimeDoc()).toBe(mockedDoc);
+    expect(sut.getRealtimeDoc() instanceof RealtimeDoc).toBeTruthy();
   });
 
-  it('destroys y-doc on self-destruction', () => {
+  it('destroys realtime doc on self-destruction', () => {
     const sut = new RealtimeNote(mockedNote, 'nothing');
     const docDestroy = jest.spyOn(sut.getRealtimeDoc(), 'destroy');
     sut.destroy();

backend/src/realtime/realtime-note/realtime-note.ts

@@ -28,12 +28,17 @@ export class RealtimeNote extends EventEmitter2<RealtimeNoteEventMap> {
   private readonly clients = new Set<RealtimeConnection>();
   private isClosing = false;
 
-  constructor(private readonly note: Note, initialContent: string) {
+  constructor(
+    private readonly note: Note,
+    initialTextContent: string,
+    initialYjsState?: number[],
+  ) {
     super();
     this.logger = new Logger(`${RealtimeNote.name} ${note.id}`);
-    this.doc = new RealtimeDoc(initialContent);
+    this.doc = new RealtimeDoc(initialTextContent, initialYjsState);
+    const length = this.doc.getCurrentContent().length;
     this.logger.debug(
-      `New realtime session for note ${note.id} created. Length of initial content: ${initialContent.length} characters`,
+      `New realtime session for note ${note.id} created. Length of initial content: ${length} characters`,
     );
   }
 

backend/src/revisions/revision.entity.ts

@@ -48,6 +48,9 @@ export class Revision {
   @Column()
   length: number;
 
+  @Column('simple-array', { nullable: true })
+  yjsStateVector: null | number[];
+
   /**
    * Date at which the revision was created.
    */
@@ -74,6 +77,7 @@ export class Revision {
     content: string,
     patch: string,
     note: Note,
+    yjsStateVector?: number[],
   ): Omit<Revision, 'id' | 'createdAt'> {
     const newRevision = new Revision();
     newRevision.patch = patch;
@@ -81,6 +85,7 @@ export class Revision {
     newRevision.length = content.length;
     newRevision.note = Promise.resolve(note);
     newRevision.edits = Promise.resolve([]);
+    newRevision.yjsStateVector = yjsStateVector ?? null;
     return newRevision;
   }
 }

backend/src/revisions/revisions.service.ts

@@ -145,6 +145,7 @@ export class RevisionsService {
   async createRevision(
     note: Note,
     newContent: string,
+    yjsStateVector?: number[],
   ): Promise<Revision | undefined> {
     // TODO: Save metadata
     const latestRevision = await this.getLatestRevision(note);
@@ -157,7 +158,7 @@ export class RevisionsService {
       latestRevision.content,
       newContent,
     );
-    const revision = Revision.create(newContent, patch, note);
+    const revision = Revision.create(newContent, patch, note, yjsStateVector);
     return await this.revisionRepository.save(revision);
   }
 }