Improved OSS development environment (#6997)

Tiltfile

@@ -0,0 +1,5 @@
+if k8s_context() != 'colima-ab-control-plane':
+  fail('!!! You are currently configured to use a non-local context. Please run `make k8s.up`')
+
+load_dynamic('./airbyte-workers/Tiltfile')
+load_dynamic('./airbyte-server/Tiltfile')

airbyte-connector-builder-server/Dockerfile

@@ -10,7 +10,7 @@ RUN pip install --no-cache-dir -r requirements.txt
 
 ENTRYPOINT ["uvicorn", "connector_builder.entrypoint:app", "--host", "0.0.0.0", "--port", "80"]
 
-EXPOSE 5003
+EXPOSE 5005
 
 LABEL io.airbyte.version=0.44.12
 LABEL io.airbyte.name=airbyte/connector-builder-server

airbyte-server/Dockerfile

@@ -1,7 +1,7 @@
 ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.0.3
 FROM ${JDK_IMAGE} AS server
 
-EXPOSE 8000 5001
+EXPOSE 8000 5005
 
 ARG VERSION=0.44.12
 

airbyte-server/Tiltfile

@@ -0,0 +1,44 @@
+update_settings(suppress_unused_image_warnings=["server"])
+
+DEBUG = os.environ.get("DEBUG", "false")
+IMAGE_TAG = os.environ.get("IMAGE_TAG", "dev")
+
+watch_file('.')
+
+custom_build(
+  'airbyte/server',
+  '../gradlew :airbyte-server:assemble',
+  tag=IMAGE_TAG,
+  deps=['src'],
+  disable_push=True,
+)
+
+template = helm(
+  '../charts/airbyte', 
+  name='ab', 
+  namespace='ab',
+  values=['../charts/airbyte/values.yaml'],
+  set=[
+    'airbyte-bootloader.debug.enabled={}'.format(DEBUG),
+    'airbyte-bootloader.image.tag={}'.format(IMAGE_TAG),
+    'billing.debug.enabled={}'.format(DEBUG),
+    'billing.image.tag={}'.format(IMAGE_TAG),
+    'connector-builder-server.debug.enabled={}'.format(DEBUG),
+    'connector-builder-server.image.tag={}'.format(IMAGE_TAG),
+    'server.debug.enabled={}'.format(DEBUG),
+    'server.image.tag={}'.format(IMAGE_TAG),
+    'webapp.image.tag={}'.format(IMAGE_TAG),
+    'worker.debug.enabled={}'.format(DEBUG),
+    'worker.env_vars.CONTAINER_ORCHESTRATOR_IMAGE=airbyte/container-orchestrator:{}'.format(IMAGE_TAG),
+    'worker.image.tag={}'.format(IMAGE_TAG)
+  ]
+)
+
+deployment = filter_yaml(
+  template, 
+  kind='deployment', 
+  name="ab-server",
+)
+
+k8s_yaml(deployment[0])
+k8s_resource('ab-server', new_name='server', labels='backend')

airbyte-workers/Dockerfile

@@ -25,7 +25,7 @@ ADD bin/${APPLICATION}-${VERSION}.tar /app
 # Grab well-known types file
 COPY WellKnownTypes.json /app
 
-EXPOSE 5007
+EXPOSE 5005
 
 # wait for upstream dependencies to become available before starting server
 ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"]

airbyte-workers/Tiltfile

@@ -0,0 +1,43 @@
+update_settings(suppress_unused_image_warnings=["worker"])
+
+DEBUG = os.environ.get("DEBUG", "false")
+IMAGE_TAG = os.environ.get("IMAGE_TAG", "dev")
+
+watch_file('.')
+
+custom_build(
+  'airbyte/worker',
+  '../gradlew :airbyte-worker:assemble',
+  tag=IMAGE_TAG,
+  deps=['src'],
+  disable_push=True,
+)
+template = helm(
+  '../charts/airbyte', 
+  name='ab', 
+  namespace='ab',
+  values=['../charts/airbyte/values.yaml'],
+  set=[
+    'airbyte-bootloader.debug.enabled={}'.format(DEBUG),
+    'airbyte-bootloader.image.tag={}'.format(IMAGE_TAG),
+    'billing.debug.enabled={}'.format(DEBUG),
+    'billing.image.tag={}'.format(IMAGE_TAG),
+    'connector-builder-server.debug.enabled={}'.format(DEBUG),
+    'connector-builder-server.image.tag={}'.format(IMAGE_TAG),
+    'server.debug.enabled={}'.format(DEBUG),
+    'server.image.tag={}'.format(IMAGE_TAG),
+    'webapp.image.tag={}'.format(IMAGE_TAG),
+    'worker.debug.enabled={}'.format(DEBUG),
+    'worker.env_vars.CONTAINER_ORCHESTRATOR_IMAGE=airbyte/container-orchestrator:{}'.format(IMAGE_TAG),
+    'worker.image.tag={}'.format(IMAGE_TAG)
+  ]
+)
+
+deployment = filter_yaml(
+  template, 
+  kind='deployment', 
+  name="ab-worker",
+)
+
+k8s_yaml(deployment[0])
+k8s_resource('ab-worker', new_name='worker', labels='backend')

charts/airbyte-connector-builder-server/values.yaml

@@ -183,4 +183,4 @@ env_vars: {}
 
 debug:
   enabled: false
-  remoteDebugPort: 5003
+  remoteDebugPort: 5005

charts/airbyte-server/values.yaml

@@ -421,4 +421,4 @@ deploymentStrategyType: Recreate
 
 debug:
   enabled: false
-  remoteDebugPort: 5001
+  remoteDebugPort: 5005

charts/airbyte-worker/values.yaml

@@ -392,4 +392,4 @@ extraLabels: {}
 
 debug:
   enabled: false
-  remoteDebugPort: 5007
+  remoteDebugPort: 5005

charts/airbyte/values.yaml

@@ -9,6 +9,7 @@ global:
   ##  deploymentMode -- Deployment mode, whether or not render the default env vars and volumes in deployment spec
   deploymentMode: "oss"
 
+  env_vars: {}
   ##  database [object] -- Object used to overrite database configuration(to use external DB)
   ##  database.secretName -- secret name where DB creds are stored
   ##  database.secretValue -- secret value for database password