+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 | 2x + +2x + + + + + + + + + + + + + + + + + + + + +2x +2x +1x + + +2x +2x +2x +2x +2x +2x +2x + + + +2x +2x +2x + + + + +2x + + + + + + +2x +1x + +1x + + + +2x + | const { queryPrometheus } = require('../services/prometheusService');
+
+let config = {
+ cpu: {
+ label: 'cpu',
+ threshold: 80,
+ minutes: 30,
+ queryString: `
+ avg(rate(container_cpu_usage_seconds_total[30m])) by (pod, namespace)/
+ sum(kube_pod_container_resource_requests{resource="cpu"}) by (pod, namespace) * 100
+ `,
+ },
+ memory: {
+ label: 'memory',
+ threshold: 80,
+ minutes: 30,
+ queryString: `sum(avg_over_time(container_memory_usage_bytes[30m])) by (pod, namespace)
+ /
+ sum(kube_pod_container_resource_requests{resource="memory"}) by (pod, namespace) * 100
+ `,
+ },
+};
+
+const updateQueries = () => {
+ queryPrometheus(config.cpu.queryString);
+ queryPrometheus(config.memory.queryString);
+};
+
+const configController = {};
+configController.saveConfig = async (req, res, next) => {
+ try {
+ const { memory, memTimeFrame, cpu, cpuTimeFrame } = req.body;
+ config.cpu.threshold = cpu;
+ config.cpu.minutes = cpuTimeFrame;
+ config.cpu.queryString = `
+ avg(rate(container_cpu_usage_seconds_total[${cpuTimeFrame}m])) by (pod, namespace)/
+ sum(kube_pod_container_resource_requests{resource="cpu"}) by (pod, namespace) * 100
+ `;
+ config.memory.threshold = memory;
+ config.memory.minutes = memTimeFrame;
+ config.memory.queryString = `sum(avg_over_time(container_memory_usage_bytes[${memTimeFrame}m])) by (pod, namespace)
+ /
+ sum(kube_pod_container_resource_requests{resource="memory"}) by (pod, namespace) * 100
+ `;
+
+ res.locals.savedConfig = {
+ cpu: { threshold: config.cpu.threshold, minutes: config.cpu.minutes },
+ memory: {
+ threshold: config.memory.threshold,
+ minutes: config.memory.minutes,
+ },
+ };
+ await updateQueries();
+ return next();
+ } catch (err) {
+ return next(err);
+ }
+};
+
+module.exports = { config, configController };
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| File | ++ | Statements | ++ | Branches | ++ | Functions | ++ | Lines | ++ |
|---|---|---|---|---|---|---|---|---|---|
| configController.js | +
+
+ |
+ 100% | +20/20 | +100% | +0/0 | +100% | +2/2 | +100% | +20/20 | +
| kubeController.js | +
+
+ |
+ 61.53% | +8/13 | +100% | +0/0 | +33.33% | +1/3 | +61.53% | +8/13 | +
| prometheusController.js | +
+
+ |
+ 87.17% | +34/39 | +83.33% | +10/12 | +60% | +3/5 | +89.47% | +34/38 | +
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 | 1x + +1x +1x + +1x + + + + + + + + + + + + + +2x +2x + +1x + + + +1x + | const k8s = require('@kubernetes/client-node');
+
+const kubeConfigFile = new k8s.KubeConfig();
+kubeConfigFile.loadFromDefault();
+
+const k8sApi = kubeConfigFile.makeApiClient(k8s.CoreV1Api);
+
+async function getPods() {
+ try {
+ const res = await k8sApi.listPodForAllNamespaces();
+ res.body.items.forEach((pod) => {
+ console.log(`${pod.metadata.namespace} - ${pod.metadata.name}`);
+ });
+ } catch (err) {
+ console.error(`Error getting pods: ${err}`);
+ }
+}
+
+async function deletePod(podName, podNamespace) {
+ try {
+ const res = await k8sApi.deleteNamespacedPod(podName, podNamespace);
+ } catch (err) {
+ console.error(`Error deleting pod: ${JSON.stringify(err)}`);
+ }
+}
+
+module.exports = deletePod;
+ |
+ Press n or j to go to the next uncovered block, b, p or k for the previous block. +
+ +| 1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 +35 +36 +37 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +57 +58 +59 +60 +61 +62 +63 +64 +65 +66 +67 +68 +69 +70 +71 +72 +73 +74 +75 +76 +77 +78 | 1x +1x +1x +1x + +1x + +1x + +1x + +1x +3x +3x + + + +3x +3x + + + +3x +2x + + + +2x +1x + +2x +1x + + + +1x +1x + + +2x + +1x + + + +1x +2x +2x +2x +2x +2x +2x + + + +1x + + + + + + + +1x + + + + + + + +1x + + + +1x +1x + | const { config } = require('./configController');
+const { queryPrometheus, runDemo } = require('../services/prometheusService');
+const deletePod = require('./kubeController');
+console.log('Prometheus Controller Running!');
+
+const callInterval = 0.3;
+
+const restartedPods = [];
+
+const prometheusController = {};
+
+prometheusController.fetchGraphData = async (req, res, next) => {
+ try {
+ Iif (runDemo) {
+ checkRestart(config.cpu);
+ checkRestart(config.memory);
+ }
+ const cpuGraphMinutes = req.query.cpuGraphMinutes;
+ const memoryGraphMinutes = req.query.memoryGraphMinutes;
+
+ let cpuData, memData;
+
+ if (cpuGraphMinutes) {
+ const cpuQuery = `
+ avg(rate(container_cpu_usage_seconds_total[${cpuGraphMinutes}m])) by (pod, namespace)/
+ sum(kube_pod_container_resource_requests{resource="cpu"}) by (pod, namespace) * 100
+ `;
+ cpuData = await queryPrometheus(cpuQuery, config.cpu.threshold);
+ res.locals.data = { cpuData };
+ }
+ if (memoryGraphMinutes) {
+ const memQuery = `
+ sum(avg_over_time(container_memory_usage_bytes[${memoryGraphMinutes}m])) by (pod, namespace) /
+ sum(kube_pod_container_resource_requests{resource="memory"}) by (pod, namespace) * 100
+ `;
+ memData = await queryPrometheus(memQuery, config.memory.threshold);
+ res.locals.data = { memData };
+ }
+
+ return next();
+ } catch (err) {
+ return next(err);
+ }
+};
+
+const checkRestart = async (obj) => {
+ const { threshold, queryString, label } = obj;
+ const data = await queryPrometheus(queryString, threshold);
+ if (data.status === 'success') {
+ const results = data.data.result;
+ results.forEach((pod) => {
+ if (
+ pod.value[1] > threshold &&
+ pod.metric.pod !== 'prometheus-prometheus-kube-prometheus-prometheus-0'
+ ) {
+ restartedPods.push({
+ timestamp: new Date(),
+ namespace: pod.metric.namespace,
+ podName: pod.metric.pod,
+ label,
+ value: pod.value[1],
+ threshold,
+ });
+ deletePod(pod.metric.pod, pod.metric.namespace);
+ }
+ });
+ } else E{
+ console.error(`PromQL ${label} query failed:`, data.error);
+ }
+};
+
+const restartChecks = async (config) => {
+ // invoke Promise.all and pass in the function invocations with arguments in the order it needs to be run in an array
+ await Promise.all([checkRestart(config.cpu), checkRestart(config.memory)]);
+};
+setInterval(() => restartChecks(config), 1000 * 60 * callInterval);
+module.exports = { restartedPods, prometheusController, checkRestart };
+ |