Merge pull request #5585 from nimrod-becker/nimrod_backport_4_0

Backport Prometheus Metrics phase2 into 4.0

Author: nimrod-becker

package.json

@@ -1,6 +1,6 @@
 {
   "name": "noobaa-core",
-  "version": "4.0.1",
+  "version": "4.0.2",
   "private": true,
   "license": "SEE LICENSE IN LICENSE",
   "description": "",

src/api/stats_api.js

@@ -33,6 +33,16 @@ module.exports = {
             }
         },
 
+        get_partial_accounts_stats: {
+            method: 'GET',
+            reply: {
+                $ref: '#/definitions/partial_accounts_stats'
+            },
+            auth: {
+                system: 'admin'
+            }
+        },
+
         get_nodes_stats: {
             method: 'GET',
             reply: {
@@ -483,14 +493,17 @@ module.exports = {
 
         partial_stats: {
             type: 'object',
-            required: ['systems_stats', 'cloud_pool_stats'],
+            required: ['systems_stats', 'cloud_pool_stats', 'accounts_stats'],
             properties: {
                 systems_stats: {
                     $ref: '#/definitions/partial_systems_stats'
                 },
                 cloud_pool_stats: {
                     $ref: '#/definitions/cloud_pool_stats'
                 },
+                accounts_stats: {
+                    $ref: '#/definitions/partial_accounts_stats'
+                },
             }
         },
 
@@ -502,20 +515,61 @@ module.exports = {
                     type: 'array',
                     items: {
                         type: 'object',
-                        required: ['name', 'free_space', 'total_space', 'buckets_stats'],
+                        required: ['name', 'capacity', 'reduction_ratio', 'savings', 'buckets_stats', 'usage_by_project', 'usage_by_bucket_class'],
                         properties: {
                             name: {
                                 type: 'string'
                             },
-                            free_space: {
-                                $ref: 'common_api#/definitions/bigint'
+                            capacity: {
+                                type: 'number'
                             },
-                            total_space: {
-                                $ref: 'common_api#/definitions/bigint'
+                            savings: {
+                                type: 'number'
+                            },
+                            reduction_ratio: {
+                                type: 'number'
                             },
                             buckets_stats: {
                                 $ref: '#/definitions/partial_buckets_stats'
                             },
+                            usage_by_project: {
+                                type: 'object',
+                                additionalProperties: true,
+                                properties: {},
+                            },
+                            usage_by_bucket_class: {
+                                type: 'object',
+                                additionalProperties: true,
+                                properties: {},
+                            },
+                        }
+                    }
+                }
+            }
+        },
+
+        partial_accounts_stats: {
+            type: 'object',
+            required: ['accounts'],
+            properties: {
+                accounts: {
+                    type: 'array',
+                    items: {
+                        type: 'object',
+                        required: ['account', 'read_count', 'write_count', 'read_write_bytes'],
+                        properties: {
+                            account: {
+                                type: 'string'
+                            },
+                            read_count: {
+                                type: 'number'
+                            },
+                            write_count: {
+                                type: 'number'
+                            },
+                            read_write_bytes: {
+                                type: 'number'
+                            },
                         }
                     }
                 }

src/deploy/NVA_build/noobaa_core.yaml

@@ -126,7 +126,7 @@ spec:
             tcpSocket:
               port: 6001
             timeoutSeconds: 5
-          image: noobaa/noobaa-core:4.0.1
+          image: noobaa/noobaa-core:4.0.2
           imagePullPolicy: IfNotPresent
           resources:
             # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/

src/server/analytic_services/prometheus_reporting.js

@@ -17,6 +17,30 @@ const METRIC_RECORDS = Object.freeze([{
         help: 'Cloud Resource Types in the System',
         labelNames: ['type', 'count']
     }
+}, {
+    metric_type: 'Gauge',
+    metric_variable: 'projects_capacity_usage',
+    configuration: {
+        name: get_metric_name('projects_capacity_usage'),
+        help: 'Projects Capacity Usage',
+        labelNames: ['project', 'count']
+    }
+}, {
+    metric_type: 'Gauge',
+    metric_variable: 'accounts_io_usage',
+    configuration: {
+        name: get_metric_name('accounts_io_usage'),
+        help: 'Accounts I/O Usage',
+        labelNames: ['account', 'read_count', 'write_count']
+    }
+}, {
+    metric_type: 'Gauge',
+    metric_variable: 'bucket_class_capacity_usage',
+    configuration: {
+        name: get_metric_name('bucket_class_capacity_usage'),
+        help: 'Bucket Class Capacity Usage',
+        labelNames: ['bucket_class', 'count']
+    }
 }, {
     metric_type: 'Gauge',
     metric_variable: 'unhealthy_cloud_types',
@@ -105,6 +129,22 @@ const METRIC_RECORDS = Object.freeze([{
         help: 'Objects On Object Bucket Claims',
     },
     generate_default_set: true,
+}, {
+    metric_type: 'Gauge',
+    metric_variable: 'reduction_ratio',
+    configuration: {
+        name: get_metric_name('reduction_ratio'),
+        help: 'Object Efficiency Ratio',
+    },
+    generate_default_set: true,
+}, {
+    metric_type: 'Gauge',
+    metric_variable: 'object_savings',
+    configuration: {
+        name: get_metric_name('object_savings'),
+        help: 'Object Savings',
+    },
+    generate_default_set: true,
 }]);
 
 
@@ -173,6 +213,31 @@ class PrometheusReporting {
         this._metrics.unhealthy_cloud_types.set({ type: 'S3_Compatible' }, types.unhealthy_cloud_pool_target.s3_comp_unhealthy);
     }
 
+    set_bucket_class_capacity_usage(usage_info) {
+        if (!this.enabled()) return;
+        this._metrics.bucket_class_capacity_usage.reset();
+        for (let [key, value] of Object.entries(usage_info)) {
+            this._metrics.bucket_class_capacity_usage.set({ bucket_class: key }, value);
+        }
+    }
+
+    set_projects_capacity_usage(usage_info) {
+        if (!this.enabled()) return;
+        this._metrics.projects_capacity_usage.reset();
+        for (let [key, value] of Object.entries(usage_info)) {
+            this._metrics.projects_capacity_usage.set({ project: key }, value);
+        }
+    }
+
+    set_accounts_io_usage(accounts_info) {
+        if (!this.enabled()) return;
+        this._metrics.accounts_io_usage.reset();
+        accounts_info.accounts.forEach(account_info => {
+            const { account, read_count, write_count, read_write_bytes } = account_info;
+            this._metrics.accounts_io_usage.set({ account, read_count, write_count }, read_write_bytes);
+        });
+    }
+
     set_object_sizes(sizes) {
         if (!this.enabled()) return;
         for (const bin of sizes) {

src/server/system_services/account_server.js

@@ -706,31 +706,26 @@ function check_external_connection(req) {
     return P.resolve()
         .then(() => {
             switch (endpoint_type) {
-                case 'AZURE':
-                    {
-                        return check_azure_connection(params);
-                    }
+                case 'AZURE': {
+                    return check_azure_connection(params);
+                }
 
                 case 'AWS':
                 case 'S3_COMPATIBLE':
-                case 'FLASHBLADE':
-                    {
-                        return check_aws_connection(params);
-                    }
+                case 'FLASHBLADE': {
+                    return check_aws_connection(params);
+                }
 
-                case 'NET_STORAGE':
-                    {
-                        return check_net_storage_connection(params);
-                    }
-                case 'GOOGLE':
-                    {
-                        return check_google_connection(params);
-                    }
+                case 'NET_STORAGE': {
+                    return check_net_storage_connection(params);
+                }
+                case 'GOOGLE': {
+                    return check_google_connection(params);
+                }
 
-                default:
-                    {
-                        throw new Error('Unknown endpoint type');
-                    }
+                default: {
+                    throw new Error('Unknown endpoint type');
+                }
             }
         });
 }