From c8033f9a3a028bc97672efdab01db14f0b1b7895 Mon Sep 17 00:00:00 2001 From: Dexter Chua Date: Sat, 27 Jun 2020 11:06:48 +0800 Subject: [PATCH] Improve handling of termination signals Previously, upon receiving a termination signal, the process tries to flush all changes to the database, retrying every 0.1s until it succeeds. However, if the database is not set up properly, this always fails, and spams the terminal/logging with the error message 10 times a second. If the user sends another termination signal, the handleTermSignal function is called once again, and we get twice the number of error messages. This commit changes the behaviour in various ways. (1) It lowers the retry rate to 0.5s, and aborts after 30 seconds. (2) If the write to the database errored, the error message explains that this is due to us flushing the final changes. (3) We replace realtime.maintenance with realtime.state, which is an Enum with three possible states --- Starting, Running, and Stopping. If a termination signal is received in the starting state, the process simply aborts because there is nothing to clean up. This is the case when the database is misconfigured, since the application starts up only after connecting to the databse. If it is in the Stopping state, the handleTermSignal function returns because another instance of handleTermSignal is already running. Fixes #408 Signed-off-by: Dexter Chua --- src/lib/app.ts | 4 ++-- src/lib/realtime.ts | 11 ++++++++--- src/lib/utils/functions.ts | 20 ++++++++++++++++---- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/lib/app.ts b/src/lib/app.ts index 8ad68dfc8..3aa576637 100644 --- a/src/lib/app.ts +++ b/src/lib/app.ts @@ -25,7 +25,7 @@ import { addNonceToLocals, computeDirectives } from './csp' import { errors } from './errors' import { logger } from './logger' import { Revision, sequelize } from './models' -import { realtime } from './realtime' +import { realtime, State } from './realtime' import { handleTermSignals } from './utils/functions' import { AuthRouter, BaseRouter, HistoryRouter, ImageRouter, NoteRouter, StatusRouter, UserRouter } from './web/' import { tooBusy, checkURI, redirectWithoutTrailingSlashes, codiMDVersion } from './web/middleware' @@ -253,7 +253,7 @@ function startListen (): void { const listenCallback = function (): void { const schema = config.useSSL ? 'HTTPS' : 'HTTP' logger.info('%s Server listening at %s', schema, address) - realtime.maintenance = false + realtime.state = State.Running } const unixCallback = function (): void { diff --git a/src/lib/realtime.ts b/src/lib/realtime.ts index 8a2034282..98d83257f 100644 --- a/src/lib/realtime.ts +++ b/src/lib/realtime.ts @@ -20,6 +20,11 @@ export type SocketWithNoteId = Socket & { noteId: string } const chance = new Chance() +export enum State { + Starting, + Running, + Stopping +} /* eslint-disable @typescript-eslint/no-use-before-define */ const realtime: { onAuthorizeSuccess: (data, accept) => void; @@ -27,7 +32,7 @@ const realtime: { io: SocketIO.Server; isReady: () => boolean; connection: (socket: SocketWithNoteId) => void; secure: (socket: SocketIO.Socket, next: (err?: Error) => void) => void; - getStatus: (callback) => void; maintenance: boolean; + getStatus: (callback) => void; state: State; } = { io: SocketIO(), onAuthorizeSuccess: onAuthorizeSuccess, @@ -36,7 +41,7 @@ const realtime: { connection: connection, getStatus: getStatus, isReady: isReady, - maintenance: true + state: State.Starting } /* eslint-enable @typescript-eslint/no-use-before-define */ @@ -751,7 +756,7 @@ function updateUserData (socket: Socket, user): void { } function connection (socket: SocketWithNoteId): void { - if (realtime.maintenance) return + if (realtime.state !== State.Running) return parseNoteIdFromSocket(socket, function (err, noteId) { if (err) { return failConnection(500, err, socket) diff --git a/src/lib/utils/functions.ts b/src/lib/utils/functions.ts index 45ff70a59..1ddac94e3 100644 --- a/src/lib/utils/functions.ts +++ b/src/lib/utils/functions.ts @@ -2,7 +2,7 @@ import fs from 'fs' import { config } from '../config' import { logger } from '../logger' import { Revision } from '../models' -import { realtime } from '../realtime' +import { realtime, State } from '../realtime' /* Converts a map from string to something into a plain JS object for transmitting via a websocket @@ -51,8 +51,15 @@ export function processData (data: T, _default: T, process?: (T) => T): T | u } export function handleTermSignals (io): void { + if (realtime.state === State.Starting) { + process.exit(0) + } + if (realtime.state === State.Stopping) { + // The function is already running. Do nothing + return + } logger.info('CodiMD has been killed by signal, try to exit gracefully...') - realtime.maintenance = true + realtime.state = State.Stopping // disconnect all socket.io clients Object.keys(io.sockets.sockets).forEach(function (key) { const socket = io.sockets.sockets[key] @@ -72,7 +79,7 @@ export function handleTermSignals (io): void { if (realtime.isReady()) { Revision.checkAllNotesRevision(function (err, notes) { if (err) { - return logger.error(err) + return logger.error('Error while writing changes to database. We will abort after trying for 30 seconds.\n' + err) } if (!notes || notes.length <= 0) { clearInterval(checkCleanTimer) @@ -80,5 +87,10 @@ export function handleTermSignals (io): void { } }) } - }, 100) + }, 500) + setTimeout(function () { + logger.error('Failed to write changes to database. Aborting') + clearInterval(checkCleanTimer) + process.exit(1) + }, 30000) }