Skip to content

Commit 1f39bd7

Browse files
authored
Merge pull request #16 from snapp-incubator/cleanup
implemented cleanup especially for sidecars
2 parents b8a4207 + f69090f commit 1f39bd7

File tree

13 files changed

+443
-60
lines changed

13 files changed

+443
-60
lines changed

internal/controller/basic_authenticator/basicauthenticator_controller.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,12 @@ package basic_authenticator
1919
import (
2020
"context"
2121
"github.com/go-logr/logr"
22+
"github.com/opdev/subreconciler"
2223
authenticatorv1alpha1 "github.com/snapp-incubator/simple-authenticator/api/v1alpha1"
2324
"github.com/snapp-incubator/simple-authenticator/internal/config"
2425
appv1 "k8s.io/api/apps/v1"
2526
corev1 "k8s.io/api/core/v1"
27+
"k8s.io/apimachinery/pkg/api/errors"
2628
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2729
"k8s.io/apimachinery/pkg/runtime"
2830
"k8s.io/apimachinery/pkg/types"
@@ -59,6 +61,19 @@ func (r *BasicAuthenticatorReconciler) Reconcile(ctx context.Context, req ctrl.R
5961
r.logger.Info("reconcile triggered")
6062
r.logger.Info(req.String())
6163
r.initVars(req)
64+
65+
basicAuthenticator := &authenticatorv1alpha1.BasicAuthenticator{}
66+
switch err := r.Get(ctx, req.NamespacedName, basicAuthenticator); {
67+
case errors.IsNotFound(err):
68+
return r.Cleanup(ctx, req)
69+
case err != nil:
70+
r.logger.Error(err, "failed to fetch object")
71+
return subreconciler.Evaluate(subreconciler.Requeue())
72+
default:
73+
if basicAuthenticator.ObjectMeta.DeletionTimestamp != nil {
74+
return r.Cleanup(ctx, req)
75+
}
76+
}
6277
return r.Provision(ctx, req)
6378
}
6479

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
package basic_authenticator
2+
3+
import (
4+
"context"
5+
"errors"
6+
"github.com/opdev/subreconciler"
7+
"github.com/snapp-incubator/simple-authenticator/api/v1alpha1"
8+
appsv1 "k8s.io/api/apps/v1"
9+
v1 "k8s.io/api/core/v1"
10+
"k8s.io/apimachinery/pkg/labels"
11+
ctrl "sigs.k8s.io/controller-runtime"
12+
"sigs.k8s.io/controller-runtime/pkg/client"
13+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
14+
)
15+
16+
func (r *BasicAuthenticatorReconciler) Cleanup(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
17+
// Do the actual reconcile work
18+
subRecs := []subreconciler.FnWithRequest{
19+
r.setDeletionStatus,
20+
r.removeInjectedContainers,
21+
r.removeCleanupFinalizer,
22+
}
23+
for _, rec := range subRecs {
24+
result, err := rec(ctx, req)
25+
if subreconciler.ShouldHaltOrRequeue(result, err) {
26+
return subreconciler.Evaluate(result, err)
27+
}
28+
}
29+
30+
return subreconciler.Evaluate(subreconciler.DoNotRequeue())
31+
}
32+
func (r *BasicAuthenticatorReconciler) setDeletionStatus(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
33+
basicAuthenticator := &v1alpha1.BasicAuthenticator{}
34+
if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
35+
return subreconciler.RequeueWithError(err)
36+
}
37+
basicAuthenticator.Status.State = StatusDeleting
38+
39+
if err := r.Update(ctx, basicAuthenticator); err != nil {
40+
r.logger.Error(err, "Failed to update status while cleaning")
41+
return subreconciler.RequeueWithError(err)
42+
}
43+
return subreconciler.ContinueReconciling()
44+
}
45+
func (r *BasicAuthenticatorReconciler) removeInjectedContainers(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
46+
basicAuthenticator := &v1alpha1.BasicAuthenticator{}
47+
48+
if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
49+
return subreconciler.RequeueWithError(err)
50+
}
51+
52+
if basicAuthenticator.Spec.Type != "sidecar" {
53+
return subreconciler.ContinueReconciling()
54+
}
55+
basicAuthLabel := map[string]string{
56+
basicAuthenticatorNameLabel: basicAuthenticator.Name,
57+
}
58+
deployments, err := getTargetDeployment(ctx, basicAuthenticator, r.Client, basicAuthLabel)
59+
if err != nil {
60+
r.logger.Error(err, "failed to get target deployments to clean up")
61+
return subreconciler.RequeueWithError(err)
62+
}
63+
configmaps, err := getTargetConfigmapNames(ctx, basicAuthenticator, r.Client, basicAuthLabel)
64+
if err != nil {
65+
r.logger.Error(err, "failed to get target configmap to clean up")
66+
return subreconciler.RequeueWithError(err)
67+
}
68+
secrets, err := getTargetSecretName(ctx, basicAuthenticator, r.Client, basicAuthLabel)
69+
if err != nil {
70+
r.logger.Error(err, "failed to get target secret to clean up")
71+
return subreconciler.RequeueWithError(err)
72+
}
73+
r.logger.Info("debug", "configmap", configmaps, "secret", secrets)
74+
75+
cleanupDeployments := removeInjectedResources(deployments, secrets, configmaps)
76+
for _, deploy := range cleanupDeployments {
77+
if err := r.Update(ctx, deploy); err != nil {
78+
r.logger.Error(err, "failed to add update cleaned up deployments")
79+
return subreconciler.RequeueWithError(err)
80+
}
81+
}
82+
return subreconciler.ContinueReconciling()
83+
}
84+
func (r *BasicAuthenticatorReconciler) removeCleanupFinalizer(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
85+
basicAuthenticator := &v1alpha1.BasicAuthenticator{}
86+
if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
87+
return subreconciler.RequeueWithError(err)
88+
}
89+
if controllerutil.ContainsFinalizer(basicAuthenticator, basicAuthenticatorFinalizer) {
90+
if ok := controllerutil.RemoveFinalizer(basicAuthenticator, basicAuthenticatorFinalizer); !ok {
91+
r.logger.Error(errors.New("finalizer not updated"), "Failed to remove finalizer for BasicAuthenticator")
92+
return subreconciler.Requeue()
93+
}
94+
}
95+
96+
if err := r.Update(ctx, basicAuthenticator); err != nil {
97+
r.logger.Error(err, "Failed to remove finalizer for BasicAuthenticator")
98+
return subreconciler.RequeueWithError(err)
99+
}
100+
return subreconciler.ContinueReconciling()
101+
}
102+
103+
func getTargetDeployment(ctx context.Context, basicAuthenticator *v1alpha1.BasicAuthenticator, k8Client client.Client, basicAuthLabels map[string]string) ([]*appsv1.Deployment, error) {
104+
var deploymentList appsv1.DeploymentList
105+
if err := k8Client.List(
106+
ctx,
107+
&deploymentList,
108+
client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(basicAuthLabels)},
109+
client.InNamespace(basicAuthenticator.Namespace)); err != nil {
110+
return nil, err
111+
}
112+
113+
resultDeployments := make([]*appsv1.Deployment, 0)
114+
for _, deploy := range deploymentList.Items {
115+
resultDeployments = append(resultDeployments, &deploy)
116+
}
117+
return resultDeployments, nil
118+
}
119+
func getTargetConfigmapNames(ctx context.Context, basicAuthenticator *v1alpha1.BasicAuthenticator, k8Client client.Client, basicAuthLabels map[string]string) ([]string, error) {
120+
var configMapList v1.ConfigMapList
121+
if err := k8Client.List(
122+
ctx,
123+
&configMapList,
124+
client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(basicAuthLabels)},
125+
client.InNamespace(basicAuthenticator.Namespace)); err != nil {
126+
return nil, err
127+
}
128+
resultConfigMaps := make([]string, 0)
129+
for _, cm := range configMapList.Items {
130+
resultConfigMaps = append(resultConfigMaps, cm.Name)
131+
}
132+
return resultConfigMaps, nil
133+
}
134+
func getTargetSecretName(ctx context.Context, basicAuthenticator *v1alpha1.BasicAuthenticator, k8Client client.Client, basicAuthLabels map[string]string) ([]string, error) {
135+
var secretList v1.SecretList
136+
if err := k8Client.List(
137+
ctx,
138+
&secretList,
139+
client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(basicAuthLabels)},
140+
client.InNamespace(basicAuthenticator.Namespace)); err != nil {
141+
return nil, err
142+
}
143+
144+
resultSecrets := make([]string, 0)
145+
for _, sec := range secretList.Items {
146+
resultSecrets = append(resultSecrets, sec.Name)
147+
}
148+
return resultSecrets, nil
149+
}
150+
func removeInjectedResources(deployments []*appsv1.Deployment, secrets []string, configmap []string) []*appsv1.Deployment {
151+
for _, deploy := range deployments {
152+
containers := make([]v1.Container, 0)
153+
for _, container := range deploy.Spec.Template.Spec.Containers {
154+
if container.Name != nginxDefaultContainerName {
155+
containers = append(containers, container)
156+
}
157+
}
158+
deploy.Spec.Template.Spec.Containers = containers
159+
volumes := make([]v1.Volume, 0)
160+
for _, vol := range deploy.Spec.Template.Spec.Volumes {
161+
if !existsInList(secrets, vol.Name) && !existsInList(configmap, vol.Name) {
162+
volumes = append(volumes, vol)
163+
}
164+
}
165+
deploy.Spec.Template.Spec.Volumes = volumes
166+
if deploy.Labels != nil {
167+
delete(deploy.Labels, basicAuthenticatorNameLabel)
168+
}
169+
}
170+
return deployments
171+
}
172+
173+
func existsInList(strList []string, targetStr string) bool {
174+
for _, val := range strList {
175+
if val == targetStr {
176+
return true
177+
}
178+
}
179+
return false
180+
}

internal/controller/basic_authenticator/const.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,7 @@ package basic_authenticator
33
const (
44
nginxDefaultImageAddress = "nginx:1.25.3"
55
nginxDefaultContainerName = "nginx"
6-
SecretAnnotation = "authenticator.snappcloud.io/secret.name"
7-
ConfigmapAnnotation = "authenticator.snappcloud.io/configmap.name"
6+
basicAuthenticatorNameLabel = "basicauthenticator.snappcloud.io/name"
87
basicAuthenticatorFinalizer = "basicauthenticator.snappcloud.io/finalizer"
98
ExternallyManaged = "basicauthenticator.snappcloud.io/externally.managed"
109
ConfigMountPath = "/etc/nginx/conf.d"
@@ -15,7 +14,6 @@ const (
1514
template = `server {
1615
listen AUTHENTICATOR_PORT;
1716
location / {
18-
resolver 8.8.8.8;
1917
auth_basic "basic authentication area";
2018
auth_basic_user_file "FILE_PATH";
2119
proxy_pass http://APP_SERVICE:APP_PORT;
@@ -25,4 +23,7 @@ const (
2523
proxy_set_header X-Forwarded-Proto $scheme;
2624
}
2725
}`
26+
StatusAvailable = "Available"
27+
StatusReconciling = "Reconciling"
28+
StatusDeleting = "Deleting"
2829
)

internal/controller/basic_authenticator/provision.go

Lines changed: 58 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,20 @@ import (
1313
"reflect"
1414
ctrl "sigs.k8s.io/controller-runtime"
1515
"sigs.k8s.io/controller-runtime/pkg/client"
16+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
1617
)
1718

1819
// Provision provisions the required resources for the basicAuthenticator object
1920
func (r *BasicAuthenticatorReconciler) Provision(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
2021
// Do the actual reconcile work
2122
subProvisioner := []subreconciler.FnWithRequest{
23+
r.setReconcilingStatus,
24+
r.addCleanupFinalizer,
2225
r.ensureSecret,
2326
r.ensureConfigmap,
2427
r.ensureDeployment,
2528
r.ensureService,
29+
r.setAvailableStatus,
2630
}
2731
for _, provisioner := range subProvisioner {
2832
result, err := provisioner(ctx, req)
@@ -33,6 +37,37 @@ func (r *BasicAuthenticatorReconciler) Provision(ctx context.Context, req ctrl.R
3337

3438
return subreconciler.Evaluate(subreconciler.DoNotRequeue())
3539
}
40+
func (r *BasicAuthenticatorReconciler) setReconcilingStatus(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
41+
basicAuthenticator := &v1alpha1.BasicAuthenticator{}
42+
43+
if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
44+
return subreconciler.RequeueWithError(err)
45+
}
46+
47+
basicAuthenticator.Status.State = StatusReconciling
48+
if err := r.Update(ctx, basicAuthenticator); err != nil {
49+
r.logger.Error(err, "failed to update status")
50+
return subreconciler.Requeue()
51+
}
52+
return subreconciler.ContinueReconciling()
53+
}
54+
55+
func (r *BasicAuthenticatorReconciler) addCleanupFinalizer(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
56+
basicAuthenticator := &v1alpha1.BasicAuthenticator{}
57+
58+
if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
59+
return subreconciler.RequeueWithError(err)
60+
}
61+
if !controllerutil.ContainsFinalizer(basicAuthenticator, basicAuthenticatorFinalizer) {
62+
if objUpdated := controllerutil.AddFinalizer(basicAuthenticator, basicAuthenticatorFinalizer); objUpdated {
63+
if err := r.Update(ctx, basicAuthenticator); err != nil {
64+
r.logger.Error(err, "failed to add basicAuthenticator finalizer")
65+
return subreconciler.Requeue()
66+
}
67+
}
68+
}
69+
return subreconciler.ContinueReconciling()
70+
}
3671

3772
func (r *BasicAuthenticatorReconciler) getLatestBasicAuthenticator(ctx context.Context, req ctrl.Request, basicAuthenticator *v1alpha1.BasicAuthenticator) (*ctrl.Result, error) {
3873
err := r.Get(ctx, req.NamespacedName, basicAuthenticator)
@@ -51,7 +86,7 @@ func (r *BasicAuthenticatorReconciler) ensureSecret(ctx context.Context, req ctr
5186
basicAuthenticator := &v1alpha1.BasicAuthenticator{}
5287

5388
if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
54-
return r, err
89+
return subreconciler.RequeueWithError(err)
5590
}
5691
r.credentialName = basicAuthenticator.Spec.CredentialsSecretRef
5792
var credentialSecret corev1.Secret
@@ -118,7 +153,7 @@ func (r *BasicAuthenticatorReconciler) ensureConfigmap(ctx context.Context, req
118153
basicAuthenticator := &v1alpha1.BasicAuthenticator{}
119154

120155
if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
121-
return r, err
156+
return subreconciler.RequeueWithError(err)
122157
}
123158

124159
authenticatorConfig := createNginxConfigmap(basicAuthenticator)
@@ -160,7 +195,7 @@ func (r *BasicAuthenticatorReconciler) ensureDeployment(ctx context.Context, req
160195
basicAuthenticator := &v1alpha1.BasicAuthenticator{}
161196

162197
if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
163-
return r, err
198+
return subreconciler.RequeueWithError(err)
164199
}
165200

166201
if r.configMapName == "" {
@@ -182,7 +217,7 @@ func (r *BasicAuthenticatorReconciler) ensureService(ctx context.Context, req ct
182217
basicAuthenticator := &v1alpha1.BasicAuthenticator{}
183218

184219
if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
185-
return r, err
220+
return subreconciler.RequeueWithError(err)
186221
}
187222
if r.deploymentLabel == nil {
188223
return subreconciler.ContinueReconciling()
@@ -217,6 +252,22 @@ func (r *BasicAuthenticatorReconciler) ensureService(ctx context.Context, req ct
217252
}
218253
return subreconciler.ContinueReconciling()
219254
}
255+
256+
func (r *BasicAuthenticatorReconciler) setAvailableStatus(ctx context.Context, req ctrl.Request) (*ctrl.Result, error) {
257+
basicAuthenticator := &v1alpha1.BasicAuthenticator{}
258+
259+
if r, err := r.getLatestBasicAuthenticator(ctx, req, basicAuthenticator); subreconciler.ShouldHaltOrRequeue(r, err) {
260+
return subreconciler.RequeueWithError(err)
261+
}
262+
263+
basicAuthenticator.Status.State = StatusAvailable
264+
if err := r.Update(ctx, basicAuthenticator); err != nil {
265+
r.logger.Error(err, "failed to update status")
266+
return subreconciler.Requeue()
267+
}
268+
return subreconciler.ContinueReconciling()
269+
}
270+
220271
func (r *BasicAuthenticatorReconciler) createDeploymentAuthenticator(ctx context.Context, req ctrl.Request, basicAuthenticator *v1alpha1.BasicAuthenticator, authenticatorConfigName, secretName string) (*ctrl.Result, error) {
221272

222273
newDeployment := createNginxDeployment(basicAuthenticator, authenticatorConfigName, secretName, r.CustomConfig)
@@ -253,6 +304,7 @@ func (r *BasicAuthenticatorReconciler) createDeploymentAuthenticator(ctx context
253304
replica, err := r.acquireTargetReplica(ctx, basicAuthenticator)
254305
if err != nil {
255306
r.logger.Error(err, "failed to acquire target replica using adaptiveScale")
307+
return subreconciler.RequeueWithError(err)
256308
}
257309
targetReplica = &replica
258310
}
@@ -262,16 +314,15 @@ func (r *BasicAuthenticatorReconciler) createDeploymentAuthenticator(ctx context
262314

263315
foundDeployment.Spec = newDeployment.Spec
264316
foundDeployment.Spec.Replicas = targetReplica
265-
266-
err := r.Update(ctx, foundDeployment)
317+
err = r.Update(ctx, foundDeployment)
267318
if err != nil {
268319
r.logger.Error(err, "failed to update deployment")
269320
return subreconciler.RequeueWithError(err)
270321
}
271322
}
272323
r.logger.Info("updating ready replicas")
273324
basicAuthenticator.Status.ReadyReplicas = int(foundDeployment.Status.ReadyReplicas)
274-
err := r.Status().Update(ctx, basicAuthenticator)
325+
err = r.Status().Update(ctx, basicAuthenticator)
275326
if err != nil {
276327
r.logger.Error(err, "failed to update basic authenticator status")
277328
return subreconciler.RequeueWithError(err)
@@ -287,10 +338,6 @@ func (r *BasicAuthenticatorReconciler) createSidecarAuthenticator(ctx context.Co
287338
return subreconciler.RequeueWithError(err)
288339
}
289340
for _, deploy := range deploymentsToUpdate {
290-
if err := ctrl.SetControllerReference(basicAuthenticator, deploy, r.Scheme); err != nil {
291-
r.logger.Error(err, "failed to set injected deployment owner")
292-
return subreconciler.RequeueWithError(err)
293-
}
294341
err := r.Update(ctx, deploy)
295342
if err != nil {
296343
r.logger.Error(err, "failed to update injected deployments")

0 commit comments

Comments
 (0)