logging: Remove support for Papertrail and LogDNA.

Author: freezy

.snyk

@@ -52,9 +52,6 @@ ignore:
     - google-auth-library > gcp-metadata > extend:
         reason: None given
         expires: '2018-09-04T08:34:40.448Z'
-    - logdna > request > extend:
-        reason: None given
-        expires: '2018-09-04T08:34:40.448Z'
     - nodegit > node-gyp > request > extend:
         reason: None given
         expires: '2018-09-04T08:34:40.448Z'
@@ -106,20 +103,12 @@ patch:
   'npm:hoek:20180212':
     - nodegit > node-pre-gyp > hawk > sntp > hoek:
         patched: '2018-07-13T21:22:51.632Z'
-      logdna > request > hawk > boom > hoek:
-        patched: '2018-08-05T08:32:40.897Z'
     - nodegit > node-gyp > request > hawk > hoek:
         patched: '2018-07-13T21:22:51.632Z'
-      logdna > request > hawk > sntp > hoek:
-        patched: '2018-08-05T08:32:40.897Z'
     - nodegit > node-gyp > request > hawk > sntp > hoek:
         patched: '2018-07-13T21:22:51.632Z'
-      logdna > request > hawk > cryptiles > boom > hoek:
-        patched: '2018-08-05T08:32:40.897Z'
     - nodegit > node-gyp > request > hawk > cryptiles > boom > hoek:
         patched: '2018-07-13T21:22:51.632Z'
-      logdna > request > hawk > hoek:
-        patched: '2018-08-05T08:32:40.897Z'
     - nodegit > node-pre-gyp > hawk > hoek:
         patched: '2018-07-13T21:22:51.632Z'
     - nodegit > node-pre-gyp > hawk > boom > hoek:

SAAS.md

@@ -12,12 +12,6 @@ settings, more precisely at `vpdb.services.raygun`.
 [Rollbar](https://rollbar.com) is another crash reporter. Like Raygun, it's configured in the
 settings at `vpdb.services.rollbar`.
 
-## LogDNA
-
-[LogDNA](https://logdna.com/) is a log management platform. VPDB sends structured logs,
-which makes it possible to filter by request, user, or IP. LogDNA is enabled in the settings
-under `vpdb.logging.logdna`.
-
 ## Sqreen
 
 [Sqreen](https://www.sqreen.io) provides several security-related features.  

config/settings.browserstack.js

@@ -27,9 +27,7 @@ module.exports = {
 		logging: {
 			level: 'info',
 			console: { access: true,  app: true },
-			file: { access: null, app: null },
-			papertrail: { access: false, app: false, options: { } },
-			logdna: { apiKey: '', env: '', hostname: '' },
+			file: { access: null, app: null, json: null },
 		},
 		skipImageOptimizations: true,
 		quota: {

config/settings.dist.js

@@ -203,57 +203,11 @@ module.exports = {
 				/**
 				 * File location of the application log. Null or empty for no logging.
 				 */
-				app: '/var/www/staging/shared/logs/app.log'
-			},
-
-			/**
-			 * Sends logs to Papertrail.
-			 */
-			papertrail: {
-
-				/**
-				 * Whether to send HTTP access log to Papertrail
-				 */
-				access: false,
-
-				/**
-				 * Whether to send application log to Papertrail
-				 */
-				app: false,
-
+				app: '/var/www/staging/shared/logs/app.log',
 				/**
-				 * Papertrail configuration. Must be set if at least
-				 * one of the two above is set to true.
-				 *
-				 * @see https://github.com/kenperkins/winston-papertrail#usage
-				 */
-				options: {
-					host: '',
-					port: '',
-					program: 'vpdb-staging',
-					colorize: true
-				}
-			},
-
-			/**
-			 * Sends logs to LogDNA
-			 */
-			logdna: {
-
-				/**
-				 * The Ingestion Key
-				 */
-				apiKey: '',
-
-				/**
-				 * The environment passed along for each line
-				 */
-				env: '',
-
-				/**
-				 * The hostname passed along
+				 * File location of the application log. Null or empty for no logging.
 				 */
-				hostname: '',
+				json: '/var/www/staging/shared/logs/json.log'
 			},
 
 			slack: {

config/settings.protractor.js

@@ -27,9 +27,7 @@ module.exports = {
 		logging: {
 			level: 'info',
 			console: { access: true,  app: true },
-			file: { access: null, app: null },
-			papertrail: { access: false, app: false, options: { } },
-			logdna: { apiKey: '', env: '', hostname: '' },
+			file: { access: null, app: null, json: null },
 		},
 		skipImageOptimizations: true,
 		quota: {

config/settings.test.js

@@ -38,9 +38,7 @@ module.exports = {
 		logging: {
 			level: process.env.LOGLEVEL || 'info',
 			console: { access: true,  app: true },
-			file: { access: null, app: null },
-			papertrail: { access: false, app: false, options: { } },
-			logdna: { apiKey: '', env: '', hostname: '' },
+			file: { access: null, app: null, json: null },
 		},
 		skipImageOptimizations: true,
 		quota: {

package.json

@@ -44,7 +44,7 @@
 	"mod-project-name": "vpdb",
 	"dependencies": {
 		"@koa/cors": "2.2.2",
-		"@slack/client": "4.3.1",
+		"@slack/client": "^4.4.0",
 		"acl": "0.4.11",
 		"adm-zip": "0.4.11",
 		"archiver": "2.1.1",
@@ -71,7 +71,6 @@
 		"koa-router": "7.4.0",
 		"koa-send": "5.0.0",
 		"lodash": "4.17.10",
-		"logdna": "2.0.4",
 		"logform": "1.9.1",
 		"mongoose": "5.2.6",
 		"mongoose-paginate": "5.0.3",
@@ -176,7 +175,6 @@
 			"build/app/files/processor/processor.js",
 			"build/app/common/settings.validator.js",
 			"build/app/common/middleware/website.middleware.js",
-			"build/app/common/logger.logdna.transport.js",
 			"build/app/authentication/strategies/**"
 		],
 		"reporter": [

src/app/common/logger.ts

@@ -27,11 +27,8 @@ import winston from 'winston';
 import { config } from './settings';
 import { RequestState } from './typings/context';
 
-const LogDna = require('logdna');
-
 class Logger {
 	private logger: winston.Logger;
-	private readonly logDnaLogger: any;
 
 	constructor(private type: LogType) {
 		const alignedWithColorsAndTime = logFormat.combine(
@@ -46,16 +43,6 @@ class Logger {
 			level: config.vpdb.logging.level,
 		});
 
-		/* istanbul ignore next */
-		if (config.vpdb.logging.logdna.apiKey) {
-			this.logDnaLogger = LogDna.createLogger(config.vpdb.logging.logdna.apiKey, {
-				app: 'vpdb',
-				env: config.vpdb.logging.logdna.env,
-				hostname: config.vpdb.logging.logdna.hostname,
-				index_meta: true,
-			});
-		}
-
 		if (type === 'app') {
 			this.setupAppLogger();
 		}
@@ -103,15 +90,6 @@ class Logger {
 
 	private log(requestState: RequestState | null, level: string, message: string) {
 		this.logger.log({ level: this.getWinstonLevel(level), message });
-		/* istanbul ignore if */
-		if (this.logDnaLogger) {
-			const opts = {
-				timestamp: Date.now(),
-				level: this.getLogDnaLevel(level),
-				meta: this.getMeta(requestState),
-			};
-			this.logDnaLogger.log(message, opts);
-		}
 	}
 
 	private setupAppLogger(): void {
@@ -179,18 +157,6 @@ class Logger {
 		} : { logType: this.type };
 	}
 
-	private getLogDnaLevel(level: string): string {
-		const map: Map<string, string> = new Map([
-			['info', 'info'],
-			['error', 'error'],
-			['warn', 'warn'],
-			['verbose', 'debug'],
-			['debug', 'debug'],
-			['wtf', 'fatal'],
-		]);
-		return map.get(level) || 'info';
-	}
-
 	private getWinstonLevel(level: string): string {
 		const map: Map<string, string> = new Map([
 			['info', 'info'],

src/app/common/settings.validator.ts

@@ -136,38 +136,16 @@ export const settingValidations = {
 						return 'App log path is not a folder.';
 					}
 				},
-			},
-			papertrail: {
-				access: (bool: any) => {
-					if (!isBoolean(bool)) {
-						return 'Papertrail access log must be either true or false';
-					}
-				},
-				app: (bool: any) => {
-					if (!isBoolean(bool)) {
-						return 'Papertrail application log must be either true or false';
-					}
-				},
-				options: (bool: any) => {
-					if (!isObject(bool)) {
-						return 'Papertrail config must be at least an object, even if it\'s empty.';
-					}
-				},
-			},
-			logdna: {
-				apiKey: (str: any) => {
-					if (!isString(str)) {
-						return 'API Key must be a string.';
+				json: (logPath: any) => {
+					if (!logPath) {
+						return null;
 					}
-				},
-				env:  (str: any) => {
-					if (!isString(str)) {
-						return 'Environment must be a string.';
+					const logDir = dirname(logPath);
+					if (!existsSync(logDir)) {
+						return 'Json log path does not exist.';
 					}
-				},
-				hostname:  (str: any) => {
-					if (!isString(str)) {
-						return 'Hostname must be a string.';
+					if (!lstatSync(logDir).isDirectory()) {
+						return 'Json log path is not a folder.';
 					}
 				},
 			},

src/app/common/typings/config.d.ts

@@ -221,58 +221,13 @@ export interface VpdbConfig {
 				 * File location of the application log. Null or empty for no logging.
 				 */
 				app: string,
-			},
-
-			/**
-			 * Sends logs to Papertrail.
-			 */
-			papertrail: {
 
 				/**
-				 * Whether to send HTTP access log to Papertrail
+				 * File location where both access and app log are written.
 				 */
-				access: boolean,
-
-				/**
-				 * Whether to send application log to Papertrail
-				 */
-				app: boolean,
-
-				/**
-				 * Papertrail configuration. Must be set if at least
-				 * one of the two above is set to true.
-				 *
-				 * @see https://github.com/kenperkins/winston-papertrail#usage
-				 */
-				options: {
-					host: string,
-					port: number,
-					program: string,
-					colorize: boolean,
-				},
+				json: string,
 			},
 
-			/**
-			 * Sends logs to LogDNA
-			 */
-			logdna: {
-
-				/**
-				 * The Ingestion Key
-				 */
-				apiKey: string,
-
-				/**
-				 * The environment passed along for each line
-				 */
-				env: string,
-
-				/**
-				 * The hostname passed along
-				 */
-				hostname: string,
-			}
-
 			slack: {
 				enabled: boolean,
 				token: string,