Revert "Revert "Add helm charts for airbyte-api-server (#7854)"" (#7948)

airbyte-proxy/Dockerfile

@@ -13,6 +13,7 @@ RUN apt-get update -y && apt-get install -y apache2-utils && rm -rf /var/lib/apt
 ENV PROXY_PASS_WEB "http://airbyte-webapp:80"
 ENV PROXY_PASS_API "http://airbyte-server:8001"
 ENV CONNECTOR_BUILDER_SERVER_API "http://airbyte-connector-builder-server:80"
+ENV PROXY_PASS_AIRBYTE_API_SERVER "http://airbyte-api-server:8006"
 
 # Nginx config file
 WORKDIR /

airbyte-proxy/nginx-auth.conf.template

@@ -79,4 +79,32 @@ http {
       }
     }
   }
+
+    server {
+      listen 8006;
+
+      location / {
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+        client_max_body_size 200M;
+
+        auth_basic "Welcome to Airbyte";
+        auth_basic_user_file /etc/nginx/.htpasswd;
+
+        proxy_pass "${PROXY_PASS_AIRBYTE_API_SERVER}";
+
+        proxy_connect_timeout       ${BASIC_AUTH_PROXY_TIMEOUT};
+        proxy_send_timeout          ${BASIC_AUTH_PROXY_TIMEOUT};
+        proxy_read_timeout          ${BASIC_AUTH_PROXY_TIMEOUT};
+        send_timeout                ${BASIC_AUTH_PROXY_TIMEOUT};
+
+        error_page 401 /etc/nginx/401.html;
+        location ~ (401.html)$ {
+          alias /etc/nginx/$1;
+          auth_basic off;
+        }
+      }
+    }
 }

airbyte-proxy/nginx-no-auth.conf.template

@@ -52,4 +52,24 @@ http {
       proxy_pass "${CONNECTOR_BUILDER_SERVER_API}";
     }
   }
+
+  server {
+    listen 8006;
+
+    location / {
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+      client_max_body_size 200M;
+
+      proxy_pass "${PROXY_PASS_AIRBYTE_API_SERVER}";
+
+      proxy_connect_timeout       ${BASIC_AUTH_PROXY_TIMEOUT};
+      proxy_send_timeout          ${BASIC_AUTH_PROXY_TIMEOUT};
+      proxy_read_timeout          ${BASIC_AUTH_PROXY_TIMEOUT};
+      send_timeout                ${BASIC_AUTH_PROXY_TIMEOUT};
+    }
+  }
+
 }

airbyte-proxy/run.sh

@@ -16,7 +16,7 @@ else
   TEMPLATE_PATH="/etc/nginx/templates/nginx-auth.conf.template"
 fi
 
-envsubst '${PROXY_PASS_WEB} ${PROXY_PASS_API} ${CONNECTOR_BUILDER_SERVER_API} ${PROXY_PASS_RESOLVER} ${BASIC_AUTH_PROXY_TIMEOUT}' < $TEMPLATE_PATH > /etc/nginx/nginx.conf
+envsubst '${PROXY_PASS_WEB} ${PROXY_PASS_API} ${CONNECTOR_BUILDER_SERVER_API} ${PROXY_PASS_AIRBYTE_API_SERVER} ${PROXY_PASS_RESOLVER} ${BASIC_AUTH_PROXY_TIMEOUT}' < $TEMPLATE_PATH > /etc/nginx/nginx.conf
 
 echo "starting nginx..."
 nginx -v

airbyte-proxy/test.sh

@@ -12,7 +12,7 @@ VERSION="${VERSION:-dev}" # defaults to "dev", otherwise it is set by environmen
 echo "testing with proxy container airbyte/proxy:$VERSION"
 
 function start_container () {
-  CMD="docker run -d -p $PORT:8000 --env BASIC_AUTH_USERNAME=$1 --env BASIC_AUTH_PASSWORD=$2 --env BASIC_AUTH_PROXY_TIMEOUT=$3 --env PROXY_PASS_WEB=http://localhost --env PROXY_PASS_API=http://localhost --env CONNECTOR_BUILDER_SERVER_API=http://localhost --name $NAME airbyte/proxy:$VERSION"
+  CMD="docker run -d -p $PORT:8000 --env BASIC_AUTH_USERNAME=$1 --env BASIC_AUTH_PASSWORD=$2 --env BASIC_AUTH_PROXY_TIMEOUT=$3 --env PROXY_PASS_WEB=http://localhost --env PROXY_PASS_API=http://localhost --env CONNECTOR_BUILDER_SERVER_API=http://localhost --env PROXY_PASS_AIRBYTE_API_SERVER=http://localhost --name $NAME airbyte/proxy:$VERSION"
   echo $CMD
   eval $CMD
   wait_for_docker;

charts/airbyte-api-server/.gitignore

@@ -0,0 +1,2 @@
+# Charts are downloaded at install time with `helm dep build`.
+charts

charts/airbyte-api-server/Chart.yaml

@@ -0,0 +1,31 @@
+apiVersion: v2
+name: airbyte-api-server
+description: Helm chart to deploy airbyte-api-server
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.47.2
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: 0.50.8
+
+dependencies:
+  - name: common
+    repository: https://charts.bitnami.com/bitnami
+    tags:
+      - bitnami-common
+    version: 1.x.x

charts/airbyte-api-server/templates/_helpers.tpl

@@ -0,0 +1,73 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "airbyte.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "airbyte.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "airbyte.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "airbyte.labels" -}}
+helm.sh/chart: {{ include "airbyte.chart" . }}
+{{ include "airbyte.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "airbyte.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "airbyte.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Define db secret
+*/}}
+
+{{- define "database.secret.name" -}}
+{{- printf "%s-postgresql" .Release.Name }}
+{{- end }}
+
+{{/* 
+Define imageTag
+*/}}
+
+{{- define "airbyte-api-server.imageTag" -}}
+{{- if .Values.image.tag }}
+    {{- printf "%s" .Values.image.tag }}
+{{- else if ((.Values.global.image).tag) }}
+    {{- printf "%s" .Values.global.image.tag }}
+{{- else }}
+    {{- printf "%s" .Chart.AppVersion }}
+{{- end }}
+{{- end }}

charts/airbyte-api-server/templates/deployment.yaml

@@ -0,0 +1,148 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "common.names.fullname" . }}
+  labels:
+    {{- include "airbyte.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "airbyte.selectorLabels" . | nindent 6 }}
+      {{- if .Values.extraSelectorLabels }}
+      {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }}
+      {{- end }}
+  strategy:
+    type: {{ .Values.deploymentStrategyType }}
+  template:
+    metadata:
+      labels:
+        {{- include "airbyte.selectorLabels" . | nindent 8 }}
+        {{- if .Values.extraSelectorLabels }}
+        {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }}
+        {{- end }}
+        {{- if .Values.podLabels }}
+          {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }}
+        {{- end }}
+      {{- if .Values.podAnnotations }}
+      annotations:
+        {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }}
+      {{- end }}
+    spec:
+      serviceAccountName: {{ .Values.global.serviceAccountName }}
+      {{- if .Values.global.imagePullSecrets }}
+      imagePullSecrets:
+        {{- range .Values.global.imagePullSecrets }}
+          {{- printf "- name: %s" .name | nindent 8 }}
+        {{- end }}
+      {{- end }}
+      {{- if .Values.nodeSelector }}
+      nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.tolerations }}
+      tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.affinity }}
+      affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.extraInitContainers }}
+      initContainers:
+      {{- toYaml .Values.extraInitContainers | nindent 6 }}
+      {{- end }}
+      containers:
+      - name: airbyte-api-server
+        image: {{ printf "%s:%s" .Values.image.repository (include "airbyte-api-server.imageTag" .) }}
+        imagePullPolicy: "{{ .Values.image.pullPolicy }}"
+        env:
+        {{- if .Values.debug.enabled }}
+        - name: JAVA_TOOL_OPTIONS
+          value: "-Xdebug -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:{{ .Values.debug.remoteDebugPort }},server=y,suspend=n"
+        {{- end}}
+        {{- if eq .Values.global.deploymentMode "oss"  }}
+        - name: AIRBYTE_VERSION
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-airbyte-env
+              key: AIRBYTE_VERSION
+        {{- end }}
+
+        # Values from secret
+        {{- if .Values.secrets }}
+        {{- range $k, $v := .Values.secrets }}
+        - name: {{ $k }}
+          valueFrom:
+            secretKeyRef:
+              name: server-secrets
+              key: {{ $k }}
+        {{- end }}
+        {{- end }}
+
+        # Values from env
+        {{- if or .Values.env_vars .Values.global.env_vars }}
+        {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }}
+        - name: {{ $k }}
+          value: {{ $v | quote }}
+        {{- end }}
+        {{- end }}
+
+        # Values from extraEnv for more compability(if you want to use external secret source or other stuff)
+        {{- if .Values.extraEnv }}
+        {{- toYaml .Values.extraEnv | nindent 8 }}
+        {{- end }}
+
+        {{- if .Values.livenessProbe.enabled }}
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: http
+          initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
+          periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
+          timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
+          successThreshold: {{ .Values.livenessProbe.successThreshold }}
+          failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
+        {{- end }}
+        {{- if .Values.readinessProbe.enabled }}
+        readinessProbe:
+          httpGet:
+            path: /health
+            port: http
+          initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
+          periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
+          timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
+          successThreshold: {{ .Values.readinessProbe.successThreshold }}
+          failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
+        {{- end }}
+
+        ports:
+        - name: http
+          containerPort: 8006
+          protocol: TCP
+        {{- if .Values.debug.enabled }}
+        - name: debug 
+          containerPort: {{ .Values.debug.remoteDebugPort }}
+          protocol: TCP
+        {{- end}}
+        {{- if .Values.resources }}
+        resources: {{- toYaml .Values.resources | nindent 10 }}
+        {{- end }}
+        {{- if .Values.containerSecurityContext }}
+        securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }}
+        {{- end }}
+        volumeMounts:
+        {{- if .Values.extraVolumeMounts }}
+          {{ toYaml .Values.extraVolumeMounts | nindent 8 }}
+        {{- end }}
+        {{- if .Values.global.extraVolumeMounts }}
+          {{ toYaml .Values.global.extraVolumeMounts | nindent 8 }}
+        {{- end }}
+      {{- if .Values.extraContainers }}
+        {{ toYaml .Values.extraContainers | nindent 6 }}
+      {{- end }}
+      {{- if .Values.global.extraContainers }}
+        {{ toYaml .Values.global.extraContainers | nindent 6 }}
+      {{- end }}
+      volumes:
+      {{- if .Values.extraVolumes }}
+{{ toYaml .Values.extraVolumes | nindent 6 }}
+      {{- end }}

charts/airbyte-api-server/templates/secrets.yaml

@@ -0,0 +1,17 @@
+# Create secrets only for the local deployment
+{{- if .Values.secrets }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: server-secrets
+  labels:
+    {{- include "airbyte.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-weight: "-1"
+type: Opaque
+data:
+  {{- range $k, $v := mergeOverwrite .Values.secrets .Values.global.secrets }}
+  {{ $k }}: {{ if $v }}{{ $v | b64enc  }} {{else}}""{{end}}
+  {{- end }}
+{{- end }}

charts/airbyte-api-server/templates/service.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{.Release.Name }}-airbyte-api-server-svc
+  {{- with .Values.service.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  labels:
+    {{- include "airbyte.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "airbyte.selectorLabels" . | nindent 4 }}