Improve handling of termination signals

Previously, upon receiving a termination signal, the process tries to
flush all changes to the database, retrying every 0.1s until it
succeeds. However, if the database is not set up properly, this always
fails, and spams the terminal/logging with the error message 10 times a
second.

If the user sends another termination signal, the handleTermSignal
function is called once again, and we get twice the number of error
messages.

This commit changes the behaviour in various ways.

(1) It lowers the retry rate to 0.5s, and aborts after 30 seconds.

(2) If the write to the database errored, the error message explains
    that this is due to us flushing the final changes.

(3) We replace realtime.maintenance with realtime.state, which is an
    Enum with three possible states --- Starting, Running, and Stopping.
    If a termination signal is received in the starting state, the
    process simply aborts because there is nothing to clean up. This is
    the case when the database is misconfigured, since the application
    starts up only after connecting to the databse. If it is in the
    Stopping state, the handleTermSignal function returns because
    another instance of handleTermSignal is already running.

Fixes #408

Signed-off-by: Dexter Chua <dec41@srcf.net>

src/lib/app.ts

@@ -25,7 +25,7 @@ import { addNonceToLocals, computeDirectives } from './csp'
 import { errors } from './errors'
 import { logger } from './logger'
 import { Revision, sequelize } from './models'
-import { realtime } from './realtime'
+import { realtime, State } from './realtime'
 import { handleTermSignals } from './utils/functions'
 import { AuthRouter, BaseRouter, HistoryRouter, ImageRouter, NoteRouter, StatusRouter, UserRouter } from './web/'
 import { tooBusy, checkURI, redirectWithoutTrailingSlashes, codiMDVersion } from './web/middleware'
@@ -253,7 +253,7 @@ function startListen (): void {
   const listenCallback = function (): void {
     const schema = config.useSSL ? 'HTTPS' : 'HTTP'
     logger.info('%s Server listening at %s', schema, address)
-    realtime.maintenance = false
+    realtime.state = State.Running
   }
 
   const unixCallback = function (): void {

src/lib/realtime.ts

@@ -20,6 +20,11 @@ export type SocketWithNoteId = Socket & { noteId: string }
 
 const chance = new Chance()
 
+export enum State {
+  Starting,
+  Running,
+  Stopping
+}
 /* eslint-disable @typescript-eslint/no-use-before-define */
 const realtime: {
   onAuthorizeSuccess: (data, accept) => void;
@@ -27,7 +32,7 @@ const realtime: {
   io: SocketIO.Server; isReady: () => boolean;
   connection: (socket: SocketWithNoteId) => void;
   secure: (socket: SocketIO.Socket, next: (err?: Error) => void) => void;
-  getStatus: (callback) => void; maintenance: boolean;
+  getStatus: (callback) => void; state: State;
 } = {
   io: SocketIO(),
   onAuthorizeSuccess: onAuthorizeSuccess,
@@ -36,7 +41,7 @@ const realtime: {
   connection: connection,
   getStatus: getStatus,
   isReady: isReady,
-  maintenance: true
+  state: State.Starting
 }
 /* eslint-enable @typescript-eslint/no-use-before-define */
 
@@ -751,7 +756,7 @@ function updateUserData (socket: Socket, user): void {
 }
 
 function connection (socket: SocketWithNoteId): void {
-  if (realtime.maintenance) return
+  if (realtime.state !== State.Running) return
   parseNoteIdFromSocket(socket, function (err, noteId) {
     if (err) {
       return failConnection(500, err, socket)

src/lib/utils/functions.ts

@@ -2,7 +2,7 @@ import fs from 'fs'
 import { config } from '../config'
 import { logger } from '../logger'
 import { Revision } from '../models'
-import { realtime } from '../realtime'
+import { realtime, State } from '../realtime'
 
 /*
 Converts a map from string to something into a plain JS object for transmitting via a websocket
@@ -51,8 +51,15 @@ export function processData<T> (data: T, _default: T, process?: (T) => T): T | u
 }
 
 export function handleTermSignals (io): void {
+  if (realtime.state === State.Starting) {
+    process.exit(0)
+  }
+  if (realtime.state === State.Stopping) {
+    // The function is already running. Do nothing
+    return
+  }
   logger.info('CodiMD has been killed by signal, try to exit gracefully...')
-  realtime.maintenance = true
+  realtime.state = State.Stopping
   // disconnect all socket.io clients
   Object.keys(io.sockets.sockets).forEach(function (key) {
     const socket = io.sockets.sockets[key]
@@ -72,7 +79,7 @@ export function handleTermSignals (io): void {
     if (realtime.isReady()) {
       Revision.checkAllNotesRevision(function (err, notes) {
         if (err) {
-          return logger.error(err)
+          return logger.error('Error while writing changes to database. We will abort after trying for 30 seconds.\n' + err)
         }
         if (!notes || notes.length <= 0) {
           clearInterval(checkCleanTimer)
@@ -80,5 +87,10 @@ export function handleTermSignals (io): void {
         }
       })
     }
-  }, 100)
+  }, 500)
+  setTimeout(function () {
+    logger.error('Failed to write changes to database. Aborting')
+    clearInterval(checkCleanTimer)
+    process.exit(1)
+  }, 30000)
 }