feat: add chaos experiments

_dev-env/.gitignore

@@ -1,3 +1,6 @@
 k8s-pki
 oidc-provider/tls
 webhook/tls
+experiments_renders
+chaostoolkit.log
+journal.json

_dev-env/Makefile

@@ -2,23 +2,23 @@ export AWS_ACCESS_KEY_ID = test
 export AWS_SECRET_ACCESS_KEY = test
 export AWS_REGION = us-east-1
 export AWS = aws --endpoint-url=http://localhost:4566 --no-cli-pager
-export DOCKER_USER = USER_ID=$(shell id -u) GROUP_ID=$(shell id -g) 
+export DOCKER_USER = USER_ID=$(shell id -u) GROUP_ID=$(shell id -g)
 
-.PHONY: all start tear_down 
+.PHONY: all start tear_down
 .PHONY: start_kind start_docker_compose register_oidc check wait_for_localstack deploy_webhook
 .PHONY: docker_build docker_push docker_build_push local_deploy update_operator restart_operator
 
-all: clean start docker_build_push local_deploy test_irsa 
+all: clean start docker_build_push local_deploy create_irsa
 start: start_kind gen_oidc_certs start_docker_compose wait_for_localstack register_oidc deploy_webhook
 update_operator: docker_build_push local_deploy restart_operator
 
 CERT_FOLDER=./oidc-provider/tls
 
-restart_operator: 
+restart_operator:
 	kubectl delete po -n irsa-operator-system --all
 
-test_irsa:
-	kubectl apply -f ./test-irsa.yml
+create_irsa:
+	kubectl apply -f ./irsa.yml
 
 start_kind:
 	$(info == STARTING KIND CLUSTER ==)
@@ -36,13 +36,13 @@ gen_oidc_certs:
 
 start_docker_compose:
 	$(info == STARTING DOCKER-COMPOSE ==)
-	$(DOCKER_USER) docker-compose up -d 
+	$(DOCKER_USER) docker-compose up -d
 
 register_oidc:
 	$(info == REGISTERING OPENID CONNECT PROVIDER ==)
 	$(AWS) iam \
 		create-open-id-connect-provider --url https://hydra.local:4444 --client-id-list sts.amazonaws.com --thumbprint-list \
-		$(shell openssl s_client -connect localhost:4444 < /dev/null 2>/dev/null | openssl x509 -fingerprint -noout -in /dev/stdin | sed 's/.*=\|://g') 
+		$(shell openssl s_client -connect localhost:4444 < /dev/null 2>/dev/null | openssl x509 -fingerprint -noout -in /dev/stdin | sed 's/.*=\|://g')
 
 check:
 	$(info == CHECKING OPENID CONNECT PROVIDERS ==)
@@ -81,3 +81,10 @@ clean:
 	kind delete clusters irsa-operator
 	$(DOCKER_USER) docker-compose down
 	sudo rm -rf ./k8s-pki
+
+chaos: create_irsa
+	rm -rf ./experiments_renders/
+	mkdir ./experiments_renders/
+	jsonnet ./experiments/_main.jsonnet -m ./experiments_renders/
+	chaos run ./experiments_renders/*.json
+

_dev-env/experiments/_main.jsonnet

@@ -0,0 +1,8 @@
+local experiments = {
+  policyRecreates: import './policy-recreation.libsonnet',
+};
+
+{
+  [e + '.json']: experiments[e]
+  for e in std.objectFields(experiments)
+}

_dev-env/experiments/policy-recreation.libsonnet

@@ -0,0 +1,29 @@
+local s = import './shared.libsonnet';
+
+{
+  version: '1.0.0',
+  title: 'Policy Deletion',
+  description: "We expect that when a policy is deleted, it's automatically recreated",
+  tags: [],
+  'steady-state-hypothesis': {
+    title: 'the policiy is present',
+    probes: [
+      s.policyIsPresent,
+      s.operatorHealthy,
+    ],
+  },
+  method: [
+    {
+      name: 'delete-custom-object',
+      type: 'action',
+      provider: {
+        type: 'python',
+        module: 'chaosk8s.crd.actions',
+        func: 'delete_custom_object',
+        arguments: s.policy,
+      },
+    },
+    s.policyIsPresent,
+    s.operatorHealthy,
+  ],
+}

_dev-env/experiments/shared.libsonnet

@@ -0,0 +1,40 @@
+{
+  policy:: {
+    group: 'irsa.voodoo.io',
+    version: 'v1alpha1',
+    plural: 'policies',
+    name: 's3put',
+    ns: 'default',
+  },
+
+  policyIsPresent:: {
+    name: 'get-custom-object',
+    type: 'probe',
+    tolerance: {
+      type: 'jsonpath',
+      path: '$.apiVersion',
+    },
+    provider: {
+      type: 'python',
+      module: 'chaosk8s.crd.probes',
+      func: 'get_custom_object',
+      arguments: $.policy,
+    },
+  },
+
+  operatorHealthy:: {
+    name: 'pods-in-conditions',
+    type: 'probe',
+    tolerance: true,
+    provider: {
+      type: 'python',
+      module: 'chaosk8s.pod.probes',
+      func: 'pods_in_conditions',
+      arguments: {
+        label_selector: 'app=irsa-operator',
+        conditions: [{ type: 'Ready', status: 'True' }],
+        ns: 'irsa-operator-system',
+      },
+    },
+  },
+}

config/manager/manager.yaml

@@ -10,7 +10,7 @@ kind: ServiceAccount
 metadata:
   name: oidc-sa
   annotations:
-    eks.amazonaws.com/role-arn: $ROLE_ARN 
+    eks.amazonaws.com/role-arn: $ROLE_ARN
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -19,6 +19,7 @@ metadata:
   namespace: system
   labels:
     control-plane: controller-manager
+    app: irsa-operator
 spec:
   selector:
     matchLabels:
@@ -28,6 +29,7 @@ spec:
     metadata:
       labels:
         control-plane: controller-manager
+        app: irsa-operator
     spec:
       serviceAccountName: irsa-operator-oidc-sa
       securityContext:

shell.nix

@@ -13,6 +13,24 @@ let
       ref = "v0.1.0";
     })
     { inherit pkgs; system = builtins.currentSystem; };
+
+  mach-nix = import
+    (builtins.fetchGit {
+      url = "https://github.com/DavHau/mach-nix/";
+      ref = "refs/tags/3.1.1";
+    })
+    {
+      python = "python39";
+      inherit pkgs;
+    };
+
+  pythonPkgs = mach-nix.mkPython {
+    requirements = ''
+      chaostoolkit
+      chaostoolkit-kubernetes
+      jsonpath2
+    '';
+  };
 in
 pkgs.mkShell {
   buildInputs =
@@ -36,5 +54,7 @@ pkgs.mkShell {
       pkgs.jq
       pkgs.gnumake
       pkgs.envsubst
+
+      pythonPkgs
     ];
 }