Add quickstart mode (#50)

* Add quickstart mode

* Fix drone

* fix drone

* fix preprocessing

* fix delete and comments

* fix wait for ns

* fix images

* Use upstream images

* Update images

.drone.yml

@@ -1,238 +1,43 @@
 ---
 kind: pipeline
-name: amd64
+type: docker
+name: build-amd64
 
 platform:
   os: linux
   arch: amd64
 
 steps:
-- name: build
-  image: rancher/dapper:v0.4.1
+- name: build-and-package
+  image: rancher/dapper:v0.5.5
   commands:
   - dapper ci
   volumes:
   - name: docker
     path: /var/run/docker.sock
 
-- name: github_binary_release
+- name: publish-bin-artifacts
   image: plugins/github-release
   settings:
     api_key:
       from_secret: github_token
-    prerelease: true
     checksum:
     - sha256
     checksum_file: CHECKSUMsum-amd64.txt
     checksum_flatten: true
     files:
-    - "dist/artifacts/*"
-  when:
-    instance:
-    - drone-publish.rancher.io
-    ref:
-    - refs/head/master
-    - refs/tags/*
-    event:
-    - tag
-
-- name: docker-publish
-  image: plugins/docker
-  settings:
-    dockerfile: package/Dockerfile
-    password:
-      from_secret: docker_password
-    repo: "rancher/opnictl"
-    tag: "${DRONE_TAG}-amd64"
-    username:
-      from_secret: docker_username
-  when:
-    instance:
-    - drone-publish.rancher.io
-    ref:
-    - refs/head/master
-    - refs/tags/*
-    event:
-    - tag
-
-volumes:
-- name: docker
-  host:
-    path: /var/run/docker.sock
-
----
-kind: pipeline
-name: arm64
-
-platform:
-  os: linux
-  arch: arm64
-
-steps:
-- name: build
-  image: rancher/dapper:v0.4.1
-  commands:
-  - dapper ci
-  volumes:
-  - name: docker
-    path: /var/run/docker.sock
-
-- name: github_binary_release
-  image: plugins/github-release
-  settings:
-    api_key:
-      from_secret: github_token
+    - bin/*
     prerelease: true
-    checksum:
-    - sha256
-    checksum_file: CHECKSUMsum-arm64.txt
-    checksum_flatten: true
-    files:
-    - "dist/artifacts/*"
   when:
-    instance:
-    - drone-publish.rancher.io
-    ref:
-    - refs/head/master
-    - refs/tags/*
     event:
     - tag
-
-- name: docker-publish
-  image: plugins/docker
-  settings:
-    dockerfile: package/Dockerfile
-    password:
-      from_secret: docker_password
-    repo: "rancher/opnictl"
-    tag: "${DRONE_TAG}-arm64"
-    username:
-      from_secret: docker_username
-  when:
     instance:
     - drone-publish.rancher.io
     ref:
     - refs/head/master
     - refs/tags/*
-    event:
-    - tag
 
 volumes:
 - name: docker
   host:
-    path: /var/run/docker.sock
-
----
-kind: pipeline
-name: arm
-
-platform:
-  os: linux
-  arch: arm
-
-steps:
-- name: build
-  image: rancher/dapper:v0.4.1
-  commands:
-  - dapper ci
-  volumes:
-  - name: docker
-    path: /var/run/docker.sock
-
-- name: github_binary_release
-  image: plugins/github-release
-  settings:
-    api_key:
-      from_secret: github_token
-    prerelease: true
-    checksum:
-    - sha256
-    checksum_file: CHECKSUMsum-arm.txt
-    checksum_flatten: true
-    files:
-    - "dist/artifacts/*"
-  when:
-    instance:
-    - drone-publish.rancher.io
-    ref:
-    - refs/head/master
-    - refs/tags/*
-    event:
-    - tag
-
-- name: docker-publish
-  image: plugins/docker
-  settings:
-    dockerfile: package/Dockerfile
-    password:
-      from_secret: docker_password
-    repo: "rancher/opnictl"
-    tag: "${DRONE_TAG}-arm"
-    username:
-      from_secret: docker_username
-  when:
-    instance:
-    - drone-publish.rancher.io
-    ref:
-    - refs/head/master
-    - refs/tags/*
-    event:
-    - tag
-
-volumes:
-- name: docker
-  host:
-    path: /var/run/docker.sock
-
----
-kind: pipeline
-name: manifest
-
-platform:
-  os: linux
-  arch: amd64
-
-steps:
-- name: manifest
-  image: plugins/manifest:1.0.2
-  settings:
-    username:
-      from_secret: docker_username
-    password:
-      from_secret: docker_password
-    platforms:d64
-      - linux/arm64
-      - linux/arm
-    target: "rancher/opnictl:${DRONE_TAG}"
-    template: "rancher/opnictl:${DRONE_TAG}-ARCH"
-  when:
-    instance:
-    - drone-publish.rancher.io
-    ref:
-    - refs/head/master
-    - refs/tags/*
-    event:
-    - tag
-
-depends_on:
-- amd64
-- arm64
-- arm
-      - linux/amd64
-      - linux/arm64
-      - linux/arm
-    target: "rancher/opnictl:${DRONE_TAG}"
-    template: "rancher/opnictl:${DRONE_TAG}-ARCH"
-  when:
-    instance:
-    - drone-publish.rancher.io
-    ref:
-    - refs/head/master
-    - refs/tags/*
-    event:
-    - tag
-
-depends_on:
-- amd64
-- arm64
-- arm
+    path: /var/run/docker.sock

go.mod

@@ -38,6 +38,7 @@ replace (
 require (
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
 	github.com/k3s-io/helm-controller v0.8.4
+	github.com/mitchellh/go-homedir v1.1.0
 	github.com/rancher/wrangler v0.8.0
 	github.com/rancher/wrangler-api v0.6.0
 	github.com/sirupsen/logrus v1.6.0

go.sum

@@ -264,6 +264,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=

src/k8s/opni-stack/opendistro-es.yaml → manifests/opni-stack/opendistro-es.yaml

@@ -11,4 +11,5 @@ spec:
     elasticsearch.data.persistence.enabled: "true"
     elasticsearch.data.persistence.storageClass: "local-path"
     elasticsearch.username: "%ES_USER%"
-    elasticsearch.password: "%ES_PASSWORD%"
+    elasticsearch.password: "%ES_PASSWORD%"
+

manifests/opni-stack/rancher-logging.yaml

@@ -0,0 +1,21 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: rancher-logging-crd
+  namespace: opni-system
+spec:
+  chart: rancher-logging-crd
+  repo: http://charts.rancher.io/
+  version: "3.9.400"
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: rancher-logging 
+  namespace: opni-system
+spec:
+  chart: rancher-logging
+  repo: http://charts.rancher.io/
+  version: "3.9.400"
+  set:
+    additionalLoggingSources.rke.enabled: "true"

src/k8s/opni-stack/traefik.yaml → manifests/opni-stack/traefik.yaml

@@ -6,4 +6,6 @@ metadata:
 spec:
   chart: traefik
   repo: https://helm.traefik.io/traefik
-  version: "%TRAEFIK_VERSION%"
+  version: "%TRAEFIK_VERSION%"
+  set:
+    ports.websecure.nodePort: 32222 # hardcoding node port for traefik endpoint to be used later in rancher logging

src/k8s/services/drain-service.yaml → manifests/services/drain-service.yaml

@@ -2,6 +2,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: drain-service
+  namespace: opni-system
 spec:
   replicas: 1
   selector:
@@ -14,7 +15,7 @@ spec:
     spec:
       containers:
       - name: drain-service
-        image: amartyarancher/drain-service:v0.1
+        image: rancher/opni-drain-service:v0.1.1
         imagePullPolicy: Always
         env:
         - name: NATS_SERVER_URL
@@ -29,4 +30,13 @@ spec:
           value: "https://opendistro-es-client-service.opni-system.svc.cluster.local:9200"
         - name: FAIL_KEYWORDS
           value: "fail,error,missing,unable"
-
+        - name: MINIO_SERVER_URL
+          value: "http://minio.opni-system.svc.cluster.local:9000"
+        - name: MINIO_ACCESS_KEY
+          value: "%MINIO_ACCESS_KEY%"
+        - name: MINIO_SECRET_KEY
+          value: "%MINIO_SECRET_KEY%"
+        - name: ES_USERNAME
+          value: "%ES_USER%"
+        - name: ES_PASSWORD
+          value: "%ES_PASSWORD%"

manifests/services/log-output.yaml

@@ -0,0 +1,42 @@
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: ClusterOutput
+metadata:
+    name: "aiops-demo-log-output"
+    namespace: "opni-system"
+spec:
+  http:
+    endpoint: http://payload-receiver-service.opni-system.svc/
+    content_type: application/json
+    json_array: true
+    buffer:
+      tags: "[]"
+      flush_interval: 2s
+      chunk_limit_size: 1mb
+---
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: ClusterFlow
+metadata: 
+  name: "aiops-demo-log-flow"
+  namespace: "opni-system"
+spec: 
+  filters:
+    - dedot:
+       de_dot_separator: "-"
+       de_dot_nested: true
+    - grep:
+        exclude:
+        - key: log
+          pattern: ^\n$
+    - 
+      detectExceptions: 
+        languages: 
+          - java
+          - python
+          - go
+          - ruby
+          - js
+          - csharp
+          - php
+        multiline_flush_interval: "0.1"
+  globalOutputRefs: 
+    - "aiops-demo-log-output"