Merge pull request #7772 from dannyzaken/danny-5.14

[Backport to 5.14] ssl cert reload - run in endpoint bg (bz 2237903)

Author: dannyzaken

src/endpoint/endpoint.js

@@ -80,6 +80,7 @@ dbg.log0('endpoint: replacing old umask: ', old_umask.toString(8), 'with new uma
 /**
  * @param {EndpointOptions} options
  */
+/* eslint-disable max-statements */
 async function main(options = {}) {
     try {
         // the primary just forks and returns, workers will continue to serve
@@ -144,11 +145,17 @@ async function main(options = {}) {
         const endpoint_request_handler = create_endpoint_handler(init_request_sdk, virtual_hosts);
         const endpoint_request_handler_sts = create_endpoint_handler(init_request_sdk, virtual_hosts, true);
 
-        const ssl_cert = await ssl_utils.get_ssl_certificate('S3');
-        const ssl_options = { ...ssl_cert, honorCipherOrder: true };
+        const ssl_cert_info = await ssl_utils.get_ssl_cert_info('S3');
+        const ssl_options = { ...ssl_cert_info.cert, honorCipherOrder: true };
         const http_server = http.createServer(endpoint_request_handler);
         const https_server = https.createServer(ssl_options, endpoint_request_handler);
         const https_server_sts = https.createServer(ssl_options, endpoint_request_handler_sts);
+        ssl_cert_info.on('update', updated_ssl_cert_info => {
+            dbg.log0("Setting updated S3 ssl certs for endpoint.");
+            const updated_ssl_options = { ...updated_ssl_cert_info.cert, honorCipherOrder: true };
+            https_server.setSecureContext(updated_ssl_options);
+            https_server_sts.setSecureContext(updated_ssl_options);
+        });
 
         if (http_port > 0) {
             dbg.log0('Starting S3 HTTP', http_port);

src/rpc/rpc_http_server.js

@@ -48,10 +48,17 @@ class RpcHttpServer extends events.EventEmitter {
         const logging = options.logging;
         dbg.log0('HTTP SERVER:', 'port', port, 'secure', secure, 'logging', logging);
 
-        const ssl_cert = await ssl_utils.get_ssl_certificate('MGMT');
-        const server = secure ?
-            https.createServer({ ...ssl_cert, honorCipherOrder: true }) :
-            http.createServer();
+        let server;
+        if (secure) {
+            const ssl_cert_info = await ssl_utils.get_ssl_cert_info('MGMT');
+            server = https.createServer({ ...ssl_cert_info.cert, honorCipherOrder: true });
+            ssl_cert_info.on('update', updated_cert_info => {
+                dbg.log0("Setting updated MGMT ssl certs for rpc server.");
+                server.setSecureContext({ ...updated_cert_info.cert, honorCipherOrder: true });
+            });
+        } else {
+            server = http.createServer();
+        }
         this.install_on_server(server, options.default_handler);
         return P.fromCallback(callback => server.listen(port, callback))
             .then(() => server);

src/server/bg_services/server_monitor.js

@@ -9,7 +9,6 @@ const os_utils = require('../../util/os_utils');
 const Dispatcher = require('../notifications/dispatcher');
 const server_rpc = require('../server_rpc');
 const system_store = require('../system_services/system_store').get_instance();
-const ssl_utils = require('../../util/ssl_utils');
 const db_client = require('../../util/db_client');
 
 
@@ -53,7 +52,6 @@ async function run_monitors() {
 
     _check_dns_and_phonehome();
     await _check_internal_ips();
-    await _verify_ssl_certs();
     await _check_db_disk_usage();
     await _check_address_changes(CONTAINER_PLATFORM);
 }
@@ -80,17 +78,6 @@ function _check_internal_ips() {
         });
 }
 
-async function _verify_ssl_certs() {
-    dbg.log2('_verify_ssl_certs');
-    const updated = await ssl_utils.update_certs_from_disk();
-    if (updated) {
-        dbg.log0('_verify_ssl_certs: SSL certificates changed, restarting relevant services');
-        await os_utils.restart_services([
-            'webserver'
-        ]);
-    }
-}
-
 async function _check_db_disk_usage() {
     dbg.log2('_check_db_disk_usage');
     const { fsUsedSize, fsTotalSize } = await db_client.instance().get_db_stats();

src/server/web_server.js

@@ -90,8 +90,12 @@ async function main() {
         server_rpc.rpc.register_ws_transport(http_server);
         await P.ninvoke(http_server, 'listen', http_port);
 
-        const ssl_cert = await ssl_utils.get_ssl_certificate('MGMT');
-        const https_server = https.createServer({ ...ssl_cert, honorCipherOrder: true }, app);
+        const ssl_cert_info = await ssl_utils.get_ssl_cert_info('MGMT');
+        const https_server = https.createServer({ ...ssl_cert_info.cert, honorCipherOrder: true }, app);
+        ssl_cert_info.on('update', updated_cert_info => {
+            dbg.log0("Setting updated MGMT ssl certs for web server.");
+            https_server.setSecureContext({...updated_cert_info.cert, honorCipherOrder: true });
+        });
         server_rpc.rpc.register_ws_transport(https_server);
         await P.ninvoke(https_server, 'listen', https_port);
 

src/util/ssl_utils.js

@@ -8,18 +8,41 @@ const https = require('https');
 const Semaphore = require('../util/semaphore');
 const dbg = require('./debug_module')(__filename);
 const nb_native = require('./nb_native');
+const { EventEmitter } = require('events');
 
-const init_cert_info = dir => ({
-    dir,
-    cert: null,
-    is_loaded: false,
-    is_generated: false,
-    sem: new Semaphore(1)
-});
+class CertInfo extends EventEmitter {
+
+    constructor(dir) {
+        super();
+        this.dir = dir;
+        this.cert = null;
+        this.is_loaded = false;
+        this.is_generated = false;
+        this.sem = new Semaphore(1);
+    }
+
+    async file_notification(event, filename) {
+        try {
+            const cert_on_disk = await _read_ssl_certificate(this.dir);
+            if (this.cert.key === cert_on_disk.key) {
+                return;
+            }
+
+            this.cert = cert_on_disk;
+            this.is_generated = false;
+
+            this.emit('update', this);
+        } catch (err) {
+            if (err.code !== 'ENOENT') {
+                dbg.warn(`SSL certificate failed to update from dir ${this.dir}:`, err.message);
+            }
+        }
+    }
+}
 
 const certs = {
-    MGMT: init_cert_info('/etc/mgmt-secret'),
-    S3: init_cert_info('/etc/s3-secret'),
+    MGMT: new CertInfo('/etc/mgmt-secret'),
+    S3: new CertInfo('/etc/s3-secret'),
 };
 
 function generate_ssl_certificate() {
@@ -36,19 +59,19 @@ function verify_ssl_certificate(certificate) {
 }
 
 // Get SSL certificate (load once then serve from cache)
-function get_ssl_certificate(service) {
+function get_ssl_cert_info(service) {
     const cert_info = certs[service];
     if (!cert_info) {
         throw new Error(`Invalid service name, got: ${service}`);
     }
 
     if (cert_info.is_loaded) {
-        return cert_info.cert;
+        return cert_info;
     }
 
     return cert_info.sem.surround(async () => {
         if (cert_info.is_loaded) {
-            return cert_info.cert;
+            return cert_info;
         }
 
         try {
@@ -68,40 +91,19 @@ function get_ssl_certificate(service) {
         }
 
         cert_info.is_loaded = true;
-        return cert_info.cert;
-    });
-}
-
-// For each cert that was loaded into memory we check if the cert was changed on disk.
-// If so we update it. If any of the certs was updated we return true else we return false.
-async function update_certs_from_disk() {
-    const promiseList = Object.values(certs).map(cert_info =>
-        cert_info.sem.surround(async () => {
-            if (!cert_info.is_loaded) {
-                return false;
-            }
 
-            try {
-                const cert_on_disk = await _read_ssl_certificate(cert_info.dir);
-                if (cert_info.cert.key === cert_on_disk.key) {
-                    return false;
-                }
-
-                cert_info.cert = cert_on_disk;
-                cert_info.is_generated = false;
-                return true;
-
-            } catch (err) {
-                if (err.code !== 'ENOENT') {
-                    dbg.warn(`SSL certificate failed to update from dir ${cert_info.dir}:`, err.message);
-                }
-                return false;
+        try {
+            fs.watch(cert_info.dir, {}, cert_info.file_notification.bind(cert_info));
+        } catch (err) {
+            if (err.code === 'ENOENT') {
+                dbg.warn("Certificate folder ", cert_info.dir, " does not exist. New certificate won't be loaded.");
+            } else {
+                dbg.error("Failed to watch certificate dir ", cert_info.dir, ". err = ", err);
             }
-        })
-    );
+        }
 
-    const updatedList = await Promise.all(promiseList);
-    return updatedList.some(Boolean);
+        return cert_info;
+    });
 }
 
 // Read SSL certificate form disk
@@ -125,6 +127,15 @@ function is_using_generated_certs() {
     );
 }
 
+function get_cert_dir(service) {
+    const cert_info = certs[service];
+    if (!cert_info) {
+        throw new Error(`Invalid service name, got: ${service}`);
+    }
+
+    return cert_info.dir;
+}
+
 // create a default certificate and start an https server to test it in the browser
 function run_https_test_server() {
     const server = https.createServer(generate_ssl_certificate());
@@ -144,8 +155,8 @@ function run_https_test_server() {
 
 exports.generate_ssl_certificate = generate_ssl_certificate;
 exports.verify_ssl_certificate = verify_ssl_certificate;
-exports.get_ssl_certificate = get_ssl_certificate;
+exports.get_ssl_cert_info = get_ssl_cert_info;
 exports.is_using_generated_certs = is_using_generated_certs;
-exports.update_certs_from_disk = update_certs_from_disk;
+exports.get_cert_dir = get_cert_dir;
 
 if (require.main === module) run_https_test_server();