Persist MemoryMonitoring for defunct instances (#413)

* Persist MemoryMonitoring for defunct instances

* Persist count of snapshots, not a particular time period

CHANGELOG.md

@@ -13,6 +13,7 @@
   request, helping to reduce network overhead for chatty apps.
 * Improved the handling of track log timestamps - these can now be supplied by the client and are no
   longer bound to insert time of DB record. Latest Hoist React uses *start* of the tracked activity.
+* Support for persisting of memory monitoring results
 
 ### ⚙️ Technical
 

grails-app/controllers/io/xh/hoist/admin/cluster/MemoryMonitorAdminController.groovy

@@ -16,6 +16,8 @@ import static io.xh.hoist.util.Utils.appContext
 @Access(['HOIST_ADMIN_READER'])
 class MemoryMonitorAdminController extends BaseController {
 
+    def memoryMonitoringService
+
     def snapshots(String instance) {
         runOnInstance(new Snapshots(), instance)
     }
@@ -46,7 +48,6 @@ class MemoryMonitorAdminController extends BaseController {
         }
     }
 
-
     @Access(['HOIST_ADMIN'])
     def dumpHeap(String filename, String instance) {
         runOnInstance(new DumpHeap(filename: filename), instance)
@@ -59,4 +60,12 @@ class MemoryMonitorAdminController extends BaseController {
             return [success: true]
         }
     }
+
+    def availablePastInstances() {
+        renderJSON(memoryMonitoringService.availablePastInstances())
+    }
+
+    def snapshotsForPastInstance(String instance) {
+        renderJSON(memoryMonitoringService.snapshotsForPastInstance(instance))
+    }
 }

grails-app/init/io/xh/hoist/BootStrap.groovy

@@ -268,7 +268,9 @@ class BootStrap implements LogSupport {
                     enabled: true,
                     snapshotInterval: 60,
                     maxSnapshots: 1440,
-                    heapDumpDir: null
+                    heapDumpDir: null,
+                    preservePastInstances: true,
+                    maxPastInstances: 10
                 ],
                 clientVisible: true,
                 groupName: 'xh.io',

grails-app/services/io/xh/hoist/admin/MemoryMonitoringService.groovy

@@ -8,35 +8,54 @@
 package io.xh.hoist.admin
 
 import com.sun.management.HotSpotDiagnosticMXBean
+import grails.gorm.transactions.Transactional
 import io.xh.hoist.BaseService
 import io.xh.hoist.util.DateTimeUtils
 
 import java.lang.management.GarbageCollectorMXBean
 import java.lang.management.ManagementFactory
 import java.util.concurrent.ConcurrentHashMap
 
+import static io.xh.hoist.json.JSONParser.parseObject
+import static io.xh.hoist.util.DateTimeUtils.MINUTES
 import static io.xh.hoist.util.DateTimeUtils.intervalElapsed
+import static io.xh.hoist.util.Utils.getAppEnvironment
+import static io.xh.hoist.util.Utils.isProduction
 import static io.xh.hoist.util.Utils.startupTime
+import static io.xh.hoist.util.DateTimeUtils.HOURS
 import static java.lang.Runtime.getRuntime
 import static java.lang.System.currentTimeMillis
 
+
 /**
  * Service to sample and return simple statistics on heap (memory) usage from the JVM runtime.
  * Collects rolling history of snapshots on a configurable timer.
  */
 class MemoryMonitoringService extends BaseService {
 
     def configService
+    def jsonBlobService
 
     private Map<Long, Map> _snapshots = new ConcurrentHashMap()
     private Date _lastInfoLogged
+    private final String blobOwner = 'xhMemoryMonitoringService'
+    private final static String blobType =  isProduction ? 'xhMemorySnapshots' : "xhMemorySnapshots_$appEnvironment"
+    private String blobToken
 
     void init() {
         createTimer(
             name: 'takeSnapshot',
             runFn: this.&takeSnapshot,
             interval: {this.enabled ? config.snapshotInterval * DateTimeUtils.SECONDS: -1}
         )
+
+        createTimer(
+            name: 'cullPersisted',
+            runFn: this.&cullPersisted,
+            interval: 1 * HOURS,
+            delay: 5 * MINUTES,
+            primaryOnly: true
+        )
     }
 
     boolean getEnabled() {
@@ -86,13 +105,15 @@ class MemoryMonitoringService extends BaseService {
             _snapshots.remove(oldest.key)
         }
 
-        if (intervalElapsed(1 * DateTimeUtils.HOURS, _lastInfoLogged)) {
+        if (intervalElapsed(1 * HOURS, _lastInfoLogged)) {
             logInfo(newSnap)
             _lastInfoLogged = new Date()
         } else {
             logDebug(newSnap)
         }
 
+        if (config.preservePastInstances) persistSnapshots()
+
         return newSnap
     }
 
@@ -108,6 +129,25 @@ class MemoryMonitoringService extends BaseService {
         ]
     }
 
+    /**
+     * Get list of past instances for which snapshots are available.
+     */
+    List<Map> availablePastInstances() {
+        if (!config.preservePastInstances) return []
+        jsonBlobService
+            .list(blobType, blobOwner)
+            .findAll { !clusterService.isMember(it.name) }
+            .collect { [name: it.name, lastUpdated: it.lastUpdated] }
+    }
+
+    /**
+     * Get snapshots for a past instance.
+     */
+    Map snapshotsForPastInstance(String instanceName) {
+        def blob = jsonBlobService.list(blobType, blobOwner).find { it.name == instanceName }
+        blob ? parseObject(blob.value) : [:]
+    }
+
     //------------------------
     // Implementation
     //------------------------
@@ -169,6 +209,37 @@ class MemoryMonitoringService extends BaseService {
         return Math.round(v * 100) / 100
     }
 
+    private void persistSnapshots() {
+        try {
+            if (blobToken) {
+                jsonBlobService.update(blobToken, [value: snapshots], blobOwner)
+            } else {
+                def blob = jsonBlobService.create([
+                    name : clusterService.instanceName,
+                    type : blobType,
+                    value: snapshots
+                ], blobOwner)
+                blobToken = blob.token
+            }
+        } catch (Exception e) {
+            logError('Failed to persist memory snapshots', e)
+            blobToken = null
+        }
+    }
+
+    @Transactional
+    private cullPersisted() {
+        def all = jsonBlobService.list(blobType, blobOwner).sort { it.lastUpdated },
+            maxKeep = config.maxPastInstances != null ? Math.max(config.maxPastInstances, 0) : 5,
+            toDelete = all.dropRight(maxKeep)
+
+        if (toDelete) {
+            withInfo(['Deleting memory snapshots', [count: toDelete.size()]]) {
+                toDelete.each { it.delete() }
+            }
+        }
+    }
+
     void clearCaches() {
         _snapshots.clear()
         super.clearCaches()