Feature opensearch

assets/config-default.yaml

@@ -218,6 +218,10 @@ loader:
             options:
               name: 'added_v1'
 
+  opensearch:
+    client: {}
+    indexes_prefix: ~
+
   file_queue_path: 'messages-queue'
   schemas_git_upstream: 'https://pagure.io/fedora-ci/messages.git'
   schemas_local_git_repo_path: 'schemas.git'
@@ -247,6 +251,12 @@ env_to_config_map:
     schemas_git_upstream: LOADER_SCHEMAS_GIT_UPSTREAM
     schemas_local_git_repo_path: LOADER_SCHEMAS_GIT_REPO_PATH
     schemas_local_dir_unpacked: LOADER_SCHEMAS_DIR_UNPACKED
+    opensearch:
+      client:
+        node: LOADER_OPENSEARCH_CLIENT_NODE
+        ssl:
+          ca: LOADER_OPENSEARCH_CLIENT_SSL_CA
+      indexes_prefix: LOADER_OPENSEARCH_INDEXES_PREFIX
   listener:
     broker_umb:
       client_name: LISTENER_BROKER_UMB_CLIENT_NAME

dev-loader-opensearch.json

@@ -0,0 +1,6 @@
+{
+  "watch": ["src"],
+  "ext": ".ts,.js",
+  "ignore": [],
+  "exec": "source ./env.sh && ts-node ./src/loader_opensearch.ts | ./node_modules/.bin/pino-pretty"
+}

package.json

@@ -10,12 +10,15 @@
     "dev:listener-fedora": "nodemon --config dev-listener-fedora.json",
     "start:listener-fedora": "source ./env.sh && node build/listener_fedora.js | ./node_modules/.bin/pino-pretty",
     "dev:loader": "nodemon --config dev-loader.json",
+    "dev:loader-opensearch": "nodemon --config dev-loader-opensearch.json",
     "start:loader": "source ./env.sh && node build/loader.js | ./node_modules/.bin/pino-pretty",
+    "start:loader-opensearch": "source ./env.sh && node build/loader_opensearch.js | ./node_modules/.bin/pino-pretty",
     "build": "rimraf ./build && tsc"
   },
   "author": "Andrei Stepanov",
   "license": "LGPLv3+",
   "dependencies": {
+    "@opensearch-project/opensearch": "^2.2.0",
     "@types/amqplib": "^0.10.1",
     "@types/debug": "^4.1.8",
     "@types/lodash": "^4.14.195",

rhcachain.crt

@@ -68,3 +68,69 @@ X6KXw/3RH9f5YgKqydtEZtZ0isA4+55gf0m7I0O5lNK3mgY/uBmIk/jSI9WqczrD
 WGf78pvkTQ2PcYg/WiCv+AVsaSaiEDUf4rDj55wQ30h78Ox5J2izd4I6QylB9Lpu
 fQEw+cWRxwFPJujSOTSKRHZDo1UwOIQbxqkbznSHlLCICEXxuvQ=
 -----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIF6TCCA9GgAwIBAgIFICIEEF4wDQYJKoZIhvcNAQEMBQAwgaMxCzAJBgNVBAYT
+AlVTMRcwFQYDVQQIDA5Ob3J0aCBDYXJvbGluYTEQMA4GA1UEBwwHUmFsZWlnaDEW
+MBQGA1UECgwNUmVkIEhhdCwgSW5jLjETMBEGA1UECwwKUmVkIEhhdCBJVDEZMBcG
+A1UEAwwQSW50ZXJuYWwgUm9vdCBDQTEhMB8GCSqGSIb3DQEJARYSaW5mb3NlY0By
+ZWRoYXQuY29tMB4XDTIzMDQwNjE4MjI0M1oXDTQzMDQwMjE4MjI0M1owTTEQMA4G
+A1UEChMHUmVkIEhhdDENMAsGA1UECxMEcHJvZDEqMCgGA1UEAxMhMjAyMyBDZXJ0
+aWZpY2F0ZSBBdXRob3JpdHkgUkhDU3YyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8A
+MIICCgKCAgEAukugmlnSPsZbxx3MTj+mwB7bkppCtk7ch9ggh+47gWQsMALO1CG8
+BNUlWBJWNnK5qtufMPXrKk6X06akpo4NDCzVR8Wqs/NltIR+o9R0+4rum/M9Nxa5
+oMk8Xgj5h/SyDu8x02NBJeiZV1E5oxVJjqb33AcoHGkVO3O3fPW0qNyYCsm9rWXW
+WJ2snjC9Nb9WYUMhRrsGPzUUGmRgLruET2lQOGIYqbwnyG5E6ajnDypbhxhIykyv
+vwtJMQ1Y+jBxA48ZI5pRPS/c4rWyl1i/7gftiJVk8QxpGYd6+Thi2q9sadK9JWYJ
+eWgvZwMuYO0eC12TDS4sKqmok9eZ9EIhuS1/Yuf2sodUBfwp9Sm/L6BzN/6kjADR
++Sv9Dl2TTYLlVNGWhtiWZCNGZ+I/808KanLtqC7uJX8pr7DIErvo+RGqc8TVfCYo
+eu1Aj2mRBnn1s4ZYwnxR5JtDAIhRSiv6VjWe7aS6cBywvkffT5pUzR+XQUxzYHmv
+RWFRZrXQJdZBJzq4d71bUwE3DVTODSJqc+LB6mcDNOPDtm9kVNyGewMMBz6cjhdq
+UV4t9DrFed5YELm8J+tfc/hJE+8w0waBW8f6dYAE3aIikr/k+XqxW4mm+4sTJ+UR
+2U+ts9ipOZNPVrv9Mriw/S3rq+sMRqjI9pU9T7oVfS5AgfDUVuan588CAwEAAaN5
+MHcwHQYDVR0OBBYEFAdOi8mRwVrZOwwUaZkP6ocgloX6MB8GA1UdIwQYMBaAFLX6
+jeUKeKEJldtNIYaVallPSciLMBIGA1UdEwEB/wQIMAYBAf8CAQEwDgYDVR0PAQH/
+BAQDAgGGMBEGCWCGSAGG+EIBAQQEAwIBBjANBgkqhkiG9w0BAQwFAAOCAgEAQQxq
+wobjk1YHzCy5faItBt6Q9PwEV7vtYTWIUoOLVCU7XNw/397YPWw7MtzBIDocwYY6
+wChlmcrj3IFDb12gGW/bU761f6QdWbhHIBkxaK+s2S14wNvTRk3cT9Sm+wBCcnb/
+ncE0bPpLyfFxe80+PaIdk0Cy2uGrk+53Ute0fPQJtP93dn2kZK8Iobg1A6Mj+MOM
+qAouQ/7ZcOHvkmfAEonYPc9ERGd6uTwRVfHWfncRbqBkiiK8CHH6nEx7tpI2vP4Z
+GcKRM6htMdW/PUJ3fcss+12SXBWWvkniVnQv9xl7E12uKni1VF9XthCflz9BrM5L
+/fCidOyAdYSe8yjgYOyDWjgsTb2R/nbmmS2NNVagnK6XP0nqQHnQPSjhh27W/QnN
+ob9H5CfiZe62l5o/7aMaEQqXaAmU8hqxPDo4RpduVR/mUWlQd2n4XWxKvSgMLCUR
+YanJeYbNI/S1ie7CCOdSDedcrAUc3LC7LOGj/8901qlUt19eiUA4Z3h5CTaaXmd+
+8p5sPP4CPmUw+BtvEupLYhJ3MpWm8J7VBdHMi9bxOq5OCfJgArN39nXoM1CDBc73
+QL5ldqlwCXC/V4rOCKgcHuDg9vt6DDm3itRWHHjqAWZDMGzoCuZ/8PONWtHLqQjR
+hsOZNhLn4WtaIBdt98nhjIL/ymLVhv1b5VZESTA=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIFbjCCBFagAwIBAgIBFjANBgkqhkiG9w0BAQwFADCBpTELMAkGA1UEBhMCVVMx
+FzAVBgNVBAgMDk5vcnRoIENhcm9saW5hMRAwDgYDVQQHDAdSYWxlaWdoMRYwFAYD
+VQQKDA1SZWQgSGF0LCBJbmMuMRMwEQYDVQQLDApSZWQgSGF0IElUMRswGQYDVQQD
+DBJSZWQgSGF0IElUIFJvb3QgQ0ExITAfBgkqhkiG9w0BCQEWEmluZm9zZWNAcmVk
+aGF0LmNvbTAeFw0yMzA0MDUxODUyMzJaFw0yNTExMDYxODUyMzJaMIGjMQswCQYD
+VQQGEwJVUzEXMBUGA1UECAwOTm9ydGggQ2Fyb2xpbmExEDAOBgNVBAcMB1JhbGVp
+Z2gxFjAUBgNVBAoMDVJlZCBIYXQsIEluYy4xEzARBgNVBAsMClJlZCBIYXQgSVQx
+GTAXBgNVBAMMEEludGVybmFsIFJvb3QgQ0ExITAfBgkqhkiG9w0BCQEWEmluZm9z
+ZWNAcmVkaGF0LmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALG6
+WgRWCXNZdn0UUVQ5JV2lEgHaNblgGnCAx6bZ89l5Ygi+tVDo8v1c16cM5e4EdtKE
+P88CnGL+6NJnI4iMuw2HtYM77Q2qmR9PIH3BRgCHHcZMgZjvlFKjJnLXIptkNMq/
+6tJ+6L0iWy0AzPovc5AtkRL3MBgrwgKINTBN41nuq4Dqp/QpqbYvK4Fz9uUEjtYU
+s4YZZjXfk/U5RcmCclSwyGdgxOC9lDInY/t4tCmJHxM6vlkjoJhqmLIbrgueSv+u
+wAuNLGhSjT1hqLUJU7rpUUn9eAw23ebNC0sMw9eIpS7CwGyC+jhC8uORdgiKL79h
+DJBrKmwpy0byZ58qRNPWREMqPgs11NFGB3m1yj5vj47/i6m3yYizHX61t0ws0YTP
+cmp3SyPwWXhHO6z5b56fNeYx9kfzpfptTm0y+564V3ktX4z1fOWKxxoRAwoRDsIL
+vaV2s4rYrXYaNvtu7x0qr5pKU25Yr4bPU29vBiloIFinQmivK8cSrmOsIs+VOS4l
+DcdpoB/7gtoGbyej3ErZVsN/qX/se1vkjkucABmLT/lPMfTs2Eegh4xKZMQRrTuL
++LmVuEzapvHql8u6SDbgcsIpN2LgWjr8mo9Yfr/d4jnk2yhZKagN1OIuDi/Ub+uB
+RWvY3oXfoZNgwaqIhO+93hCbeL1c5NC+zHxEnHglAgMBAAGjgagwgaUwHQYDVR0O
+BBYEFLX6jeUKeKEJldtNIYaVallPSciLMB8GA1UdIwQYMBaAFH7R4yC+UehIIPeu
+L8Zqw3PzbgcZMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMBEGCWCG
+SAGG+EIBAQQEAwIBBjAvBgNVHR8EKDAmMCSgIqAghh5odHRwOi8vb2NzcC5yZWRo
+YXQuY29tL2NybC5wZW0wDQYJKoZIhvcNAQEMBQADggEBAJFAsY4gVbXrrX1OONOk
+9ynU6ZkThgymEr4+UOqHiYeQM36rMqG5/eI16o0PHmziaIHe+Oeesx+Xi/WHUeDc
+ZO0IzixnBuhKcBYhaw1ckL61bePWs5eiomLjgUlAZB2+ViH7u+v+/AqeFjdyTCYn
+GHLS7CTprBUBnIotIN5G9Uy1QlkrxxQAIZKxeUmqZ/SMXR0hSdOVrnmjUaRdNOke
+cz13SFjq0y2+PcsFaptAmFGbi4MiiKk6ofssBR66q/Hmb65qWOijwy24UhNmbiWZ
+YyoEjnCcv9rLzuv4hUqAQsk9BwzLbFaDWNAxBUvFBe6tcU7LroZAGE4ohMOdOD3y
+jTE=
+-----END CERTIFICATE-----

src/cfg.ts

@@ -25,6 +25,7 @@ import debug from 'debug';
 import yaml from 'js-yaml';
 import assert from 'assert';
 import { ConnectionOptions } from 'rhea-promise';
+import { ClientOptions } from '@opensearch-project/opensearch/.';
 
 const log = debug('kaijs:cfg');
 /** Default config must present */
@@ -164,7 +165,7 @@ class Config {
     /**
      * Uncomment to print whole active config
      */
-    // log('Active config: %s', '\n' + yaml.dump(this.config_active));
+    log('Active config: %s', '\n' + yaml.dump(this.config_active));
     /** constructor in javascript returns this object automatically
      * constructor returns the type of the class, the constructor implicitly returns 'this'
      * Even though you technically can't extend a proxy, there is a way to force a class
@@ -230,6 +231,10 @@ export interface Cfg {
         };
       };
     };
+    opensearch: {
+      client: ClientOptions;
+      indexes_prefix: string;
+    };
     schemas_git_upstream: string;
     schemas_local_git_repo_path: string;
     schemas_local_dir_unpacked: string;

src/db_interface.ts

@@ -68,8 +68,8 @@ export type ArtifactTypes =
   | 'redhat-container-image';
 
 export const atype_to_hub_map = {
-  'koji-build': 'fedoraproject',
-  'koji-build-cs': 'centos-stream',
+  'koji-build': 'fedora',
+  'koji-build-cs': 'centos',
 };
 
 export interface KaiEtaState {

src/fqueue.ts

@@ -52,9 +52,9 @@ export interface FileQueueMessage {
   /** Timestamp from message header */
   header_timestamp?: number;
   /** Payload of message */
-  body: any;
+  body: { [key: string]: any };
   /** Message headers */
-  broker_extra?: any;
+  broker_extra: { [key: string]: any };
 }
 
 async function make(path: string, opts = { poll: false, optimizeList: false }) {

src/koji.ts

@@ -28,8 +28,8 @@ const log = debug('kaijs:kojibrew');
 const cfg = getcfg();
 
 export const koji_clients = {
-  fedoraproject: xmlrpc.createSecureClient(cfg.koji_fp),
-  'centos-stream': xmlrpc.createSecureClient(cfg.koji_cs),
+  fedora: xmlrpc.createSecureClient(cfg.koji_fp),
+  centos: xmlrpc.createSecureClient(cfg.koji_cs),
 };
 
 interface BuildInfo {
@@ -148,7 +148,7 @@ export const koji_query = async (
             log("Response for '%s':\n%o", method, value);
             resolve(value);
           }
-        }
+        },
       );
     });
   });

src/listener_fedora.ts

@@ -112,7 +112,7 @@ const process_msg = (
     broker_extra: { ...headers },
   };
   /** 'sent-at': '2021-07-30T13:10:14+00:00' */
-  let provider_timestamp = Math.floor(Date.parse(headers['sent-at']) / 1000);
+  let provider_timestamp = Math.floor(Date.parse(headers!['sent-at']) / 1000);
   if (!_.isNaN(provider_timestamp)) {
     payload_obj.header_timestamp = provider_timestamp;
   }