feat: Support configuration-based provisioning of Cloud Ops artifacts…

… (part 1) (#1036)

Add schemas describing configuration for alerts, monitoring dashboards,
log-based metrics, notification channels, SLOs and uptime checks.
Follow JSON schema standard (draft 2020-12).
Add validation checks including linting and schema validation.
Setup `/configurations/test` configuration to validate schema. This
configuration will be further used to validate Terraform execution plan.
Constraint app-id field in all schemas to prevent path injection.

.github/workflows/README.md

@@ -70,6 +70,15 @@ the use of the workflow as [required status check].
 It is configured to run on any "non-terraform" changes, so the required workflow will always
 guaranteed to terminate.
 
+### Configurations workflow ([configurations.yaml])
+
+The `configurations` checks correctness of the Sandbox configurations. It includes:
+
+* yaml linting
+* validation of the configuration's yaml files vs. schema that is defined following JSON schema [draft2020]
+* json linting
+* testing configuration vs. expected terraform plan to make sure that all components are built using "right" provider and resource definitions
+
 ### Required workflows
 
 The workflows triggered by pull request modifications (excluding a closure of the request)
@@ -114,3 +123,4 @@ For information about the customized workflow, see [workfows/README]
 [convention]: https://www.conventionalcommits.org/en/v1.0.0/
 [snippets]: https://github.com/googleapis/repo-automation-bots/tree/main/packages/snippet-bot
 [trusted contributors]: https://github.com/googleapis/repo-automation-bots/tree/main/packages/trusted-contribution
+[draft2020]: https://json-schema.org/draft/2020-12/release-notes.html

.github/workflows/cli.yaml

@@ -13,6 +13,10 @@
 # limitations under the License.
 name: CLI
 
+concurrency:
+  group: "cli-${{ github.workflow }}-${{ github.ref }}"
+  cancel-in-progress: true
+
 on:
   pull_request:
     types: [opened,synchronize,reopened]
@@ -23,9 +27,6 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: read
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}
-      cancel-in-progress: true
 
     steps:
     - name: Checkout source code

.github/workflows/configurations.yaml

@@ -0,0 +1,66 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Configurations
+
+concurrency:
+  group: "configurations-${{ github.workflow }}-${{ github.ref }}"
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    types: [opened,synchronize,reopened]
+    paths:
+    - 'configurations/**/*.yaml'
+    - 'provisioning/schemas/configuration/**/*.json'
+
+
+jobs:
+
+  validation:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v3
+
+    - name: install ajv
+      run: npm install -g ajv-cli
+
+    - name: Lint configurations
+      run: find "${{ github.workspace }}/configurations" -mindepth 2 -maxdepth 2 -type f \( -name "*.yaml" -o -name "*.yml" \) -exec echo {} +
+
+    - name: Lint schemas
+      run: echo "JSON liniting will be here"
+
+    - name: Schema validation
+      run: |-
+        cd "${{ github.workspace }}"
+        for CONFIG_FILE in $(find configurations -mindepth 2 -maxdepth 2 -type f -name "*.yaml"); do
+          FILENAME=$(basename -s ".yaml" "${CONFIG_FILE@L}")
+          case "${FILENAME}" in
+            alerts | dashboards | healthchecks | metrics | services | slos)
+              ;;
+            *)
+              echo "Unknown file ${CONFIG_FILE}" 1>&2
+              continue
+              ;;
+          esac
+          ajv --spec=draft2020 -s "provisioning/schemas/configuration/${FILENAME}.json" -d "${CONFIG_FILE}"
+        done
+  
+    - name: Dry run
+      run: echo "Validation of Terraform plan for test configuration will be here"

.github/workflows/non-terraform.yaml

@@ -13,6 +13,10 @@
 # limitations under the License.
 name: Terraform
 
+concurrency:
+  group: "terraform-${{ github.workflow }}-${{ github.ref }}"
+  cancel-in-progress: true
+
 on:
   pull_request:
     types: [opened,synchronize,reopened]
@@ -21,13 +25,10 @@ on:
 
 
 jobs:
-  tflint:
+  validation:
     runs-on: ubuntu-latest
     permissions:
       contents: read
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}
-      cancel-in-progress: true
 
     steps:
     - run: 'echo "No work to do for non-terraform changes"'
@@ -37,9 +38,6 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: read
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}
-      cancel-in-progress: true
 
     steps:
     - run: 'echo "No work to do for non-terraform changes"'

.github/workflows/terraform.yaml

@@ -13,6 +13,10 @@
 # limitations under the License.
 name: Terraform
 
+concurrency:
+  group: "terraform-${{ github.workflow }}-${{ github.ref }}"
+  cancel-in-progress: true
+
 on:
   pull_request:
     types: [opened,synchronize,reopened]
@@ -22,13 +26,10 @@ on:
 
 jobs:
 
-  tflint:
+  validation:
     runs-on: ubuntu-latest
     permissions:
       contents: read
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}
-      cancel-in-progress: true
 
     steps:
     - name: Checkout source code
@@ -69,9 +70,6 @@ jobs:
     permissions:
       contents: 'read'
       id-token: 'write'
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}
-      cancel-in-progress: true
 
     steps:
     - name: Checkout source code

configurations/test/alerts.yaml

@@ -0,0 +1,47 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+app-id: "test"
+version: "0.0.1"
+alerts:
+- name: test-alert-with-notifications
+  display-name: "Test alert with notification channel"
+  documentation: "Lorem ipsum Lorem ipsum Lorem ipsum"
+  notification-channels: [sms-test-channel, pubsub-test-channel]
+  conditions:
+  - display-name: "test condition 1"
+    condition:
+      filter: "metric.type=\"monitoring.googleapis.com/uptime_check/check_passed\""
+      threshold: 3
+      duration: "300s"
+      comparison: "COMPARISON_GT"
+channels:
+- name: sms-test-channel
+  display-name: "SMS test channel"
+  channel:
+    type: sms
+    labels:
+      number: "1234567890"
+- name: email-test-channel
+  display-name: "E-mail test channel"
+  channel:
+    type: email
+    labels:
+      email_address: "[email protected]"
+- name: pubsub-test-channel
+  display-name: "PubSub test channel"
+  channel:
+    type: pubsub
+    labels:
+      topic: "projects/unknown/topics/unknown"

configurations/test/dashboards.yaml

@@ -0,0 +1,31 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+app-id: "test"
+version: "0.0.1"
+dashboards:
+- name: test-dashboard-1
+  display-name: "Test Dashboard 1"
+  widgets:
+  - title: "CPU Usage"
+    x-axis-label: "Time"
+    y-axis-label: "%"
+    datasets:
+    - time-series:
+        filter:
+          query: "metric.type=\"compute.googleapis.com/instance/cpu/usage_time\" resource.type=\"gce_instance\""
+          alignment: ALIGN_PERCENTILE_99
+          unit-override: "cpu"
+
+      min-alignment-period: "60s"

configurations/test/healthchecks.yaml

@@ -0,0 +1,32 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+app-id: "test"
+version: "0.0.1"
+checks:
+- name: test-uptime-alert-1
+  display-name: "HTTP uptime check"
+  timeout: 10
+  resource:
+    type: k8s_service
+    labels:
+      cluster_name: "test-cluster-1"
+  content:
+    content: "ok"
+    matcher: CONTAINS_STRING
+  type:
+    request-method: "GET"
+    path: "/healtz"
+    port: 8008
+

configurations/test/metrics.yaml

@@ -0,0 +1,25 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+app-id: "test"
+version: "0.0.1"
+metrics:
+- name: test-log-metric-1
+  description: "Count audit resources"
+  filter: "resource.type=\"audited_resource\""
+  labels:
+  - key: resource
+    extractor: "EXTRACT(resource.type)"
+  - key: project
+    extractor: "EXTRACT(resource.labels.project_id)"

configurations/test/services.yaml

@@ -0,0 +1,32 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+app-id: "test"
+version: "0.0.1"
+services:
+- name: custom-test-service-1
+  display-name: "Custom service 1"
+  labels:
+    abc1: value1
+    abc2: value2
+- name: custom-test-service-2
+  display-name: "Custom service 2"
+  basic-service:
+    service-type: CLOUD_ENDPOINTS
+    labels:
+      endpoint1: value1
+      endpoint2: value2
+  labels:
+    bce1: value1
+    bce2: value2

configurations/test/slos.yaml

@@ -0,0 +1,33 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+app-id: "test"
+version: "0.0.1"
+slos:
+- name: test-service-slo-1
+  display-name: "Test availability SLO for basic service"
+  goal: 0.5
+  period: WEEK
+  service: "gae:test-service"
+  sli:
+    availability: true
+
+- name: test-service-slo-2
+  display-name: "Test latency SLO for basic service"
+  goal: 0.5
+  period: WEEK
+  service: "gae:test-service"
+  sli:
+    latency:
+      threshold: 10s