Skip to content

Commit

Permalink
feat(ec): migration from v1 to v2 - fixes (#1725)
Browse files Browse the repository at this point in the history
emosbaugh authored Jan 21, 2025
1 parent f386880 commit ccc7fcd
Showing 11 changed files with 299 additions and 235 deletions.
4 changes: 3 additions & 1 deletion kinds/apis/v1beta1/installation_types.go
Original file line number Diff line number Diff line change
@@ -35,6 +35,7 @@ const (
InstallationStateInstalled string = "Installed"
InstallationStateKubernetesInstalled string = "KubernetesInstalled"
InstallationStateAddonsInstalling string = "AddonsInstalling"
InstallationStateAddonsInstalled string = "AddonsInstalled"
InstallationStateHelmChartUpdateFailure string = "HelmChartUpdateFailure"
InstallationStateObsolete string = "Obsolete"
InstallationStateFailed string = "Failed"
@@ -49,7 +50,8 @@ const (
)

const (
DisableReconcileConditionType = "DisableReconcile"
ConditionTypeV2MigrationInProgress = "V2MigrationInProgress"
ConditionTypeDisableReconcile = "DisableReconcile"
)

// ConfigSecretEntryName holds the entry name we are looking for in the secret
2 changes: 1 addition & 1 deletion operator/controllers/installation_controller.go
Original file line number Diff line number Diff line change
@@ -590,7 +590,7 @@ func (r *InstallationReconciler) Reconcile(ctx context.Context, req ctrl.Request
return ctrl.Result{}, nil
}

if k8sutil.CheckConditionStatus(in.Status, v1beta1.DisableReconcileConditionType) == metav1.ConditionTrue {
if k8sutil.CheckConditionStatus(in.Status, v1beta1.ConditionTypeDisableReconcile) == metav1.ConditionTrue {
log.Info("Installation reconciliation is disabled, reconciliation ended")
return ctrl.Result{}, nil
}
8 changes: 7 additions & 1 deletion operator/pkg/charts/reconcile.go
Original file line number Diff line number Diff line change
@@ -9,8 +9,10 @@ import (
k0shelmv1beta1 "github.com/k0sproject/k0s/pkg/apis/helm/v1beta1"
k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
"github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
"github.com/replicatedhq/embedded-cluster/operator/pkg/k8sutil"
"github.com/replicatedhq/embedded-cluster/operator/pkg/release"
"github.com/replicatedhq/embedded-cluster/pkg/helpers"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
controllerruntime "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -120,7 +122,11 @@ func ReconcileHelmCharts(ctx context.Context, cli client.Client, in *v1beta1.Ins
if in.Status.State != v1beta1.InstallationStateInstalled {
ev = &RecordedEvent{Reason: "AddonsUpgraded", Message: "Addons upgraded"}
}
in.Status.SetState(v1beta1.InstallationStateInstalled, "Addons upgraded", nil)
if k8sutil.CheckConditionStatus(in.Status, v1beta1.ConditionTypeV2MigrationInProgress) == metav1.ConditionTrue {
in.Status.SetState(v1beta1.InstallationStateAddonsInstalled, "Addons upgraded", nil)
} else {
in.Status.SetState(v1beta1.InstallationStateInstalled, "Addons upgraded", nil)
}
return ev, nil
}

3 changes: 3 additions & 0 deletions operator/pkg/cli/logging.go
Original file line number Diff line number Diff line change
@@ -13,3 +13,6 @@ func NewLogger(level logrus.Level) (logr.Logger, error) {
log := logrusr.New(logrusLog)
return log, nil
}

// LogFunc can be used as an argument to functions that log messages.
type LogFunc func(string, ...any)
3 changes: 0 additions & 3 deletions operator/pkg/cli/migrate_v2_pod.go
Original file line number Diff line number Diff line change
@@ -74,9 +74,6 @@ var _migrateV2PodSpec = corev1.Pod{
},
}

// LogFunc can be used as an argument to Run to log messages.
type LogFunc func(string, ...any)

// runMigrateV2PodAndWait runs the v2 migration pod and waits for the pod to finish.
func runMigrateV2PodAndWait(
ctx context.Context, logf LogFunc, cli client.Client,
102 changes: 101 additions & 1 deletion operator/pkg/cli/migratev2/installation.go
Original file line number Diff line number Diff line change
@@ -3,14 +3,72 @@ package migratev2
import (
"context"
"fmt"
"time"

"github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
"github.com/replicatedhq/embedded-cluster/pkg/kubeutils"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
"sigs.k8s.io/controller-runtime/pkg/client"
)

// setV2MigrationInProgress sets the Installation condition to indicate that the v2 migration is in
// progress.
func setV2MigrationInProgress(ctx context.Context, logf LogFunc, cli client.Client, in *ecv1beta1.Installation) error {
logf("Setting v2 migration in progress")

err := setInstallationCondition(ctx, cli, in, metav1.Condition{
Type: ecv1beta1.ConditionTypeV2MigrationInProgress,
Status: metav1.ConditionTrue,
Reason: "V2MigrationInProgress",
})
if err != nil {
return fmt.Errorf("set v2 migration in progress condition: %w", err)
}

logf("Successfully set v2 migration in progress")
return nil
}

// waitForInstallationStateInstalled waits for the installation to be in a successful state and
// ready for the migration.
func waitForInstallationStateInstalled(ctx context.Context, logf LogFunc, cli client.Client, installation *ecv1beta1.Installation) error {
logf("Waiting for installation to reconcile")

err := wait.PollUntilContextCancel(ctx, 2*time.Second, true, func(ctx context.Context) (bool, error) {
in, err := kubeutils.GetCRDInstallation(ctx, cli, installation.Name)
if err != nil {
return false, fmt.Errorf("get installation: %w", err)
}

switch in.Status.State {
// Success states
case ecv1beta1.InstallationStateInstalled, ecv1beta1.InstallationStateAddonsInstalled:
return true, nil

// Failure states
case ecv1beta1.InstallationStateFailed, ecv1beta1.InstallationStateHelmChartUpdateFailure:
return false, fmt.Errorf("installation failed: %s", in.Status.Reason)
case ecv1beta1.InstallationStateObsolete:
return false, fmt.Errorf("installation is obsolete")

// In progress states
default:
return false, nil
}
})
if err != nil {
return err
}

logf("Installation reconciled")
return nil
}

// copyInstallationsToConfigMaps copies the Installation CRs to ConfigMaps.
func copyInstallationsToConfigMaps(ctx context.Context, logf LogFunc, cli client.Client) error {
var installationList ecv1beta1.InstallationList
@@ -37,7 +95,6 @@ func copyInstallationsToConfigMaps(ctx context.Context, logf LogFunc, cli client

func ensureInstallationConfigMap(ctx context.Context, cli client.Client, in *ecv1beta1.Installation) error {
copy := in.DeepCopy()
copy.Spec.SourceType = ecv1beta1.InstallationSourceTypeConfigMap
err := kubeutils.CreateInstallation(ctx, cli, copy)
if k8serrors.IsAlreadyExists(err) {
err := kubeutils.UpdateInstallation(ctx, cli, copy)
@@ -49,3 +106,46 @@ func ensureInstallationConfigMap(ctx context.Context, cli client.Client, in *ecv
}
return nil
}

// ensureInstallationStateInstalled sets the ConfigMap installation state to installed and updates
// the status to mark the upgrade as complete.
func ensureInstallationStateInstalled(ctx context.Context, logf LogFunc, cli client.Client, in *ecv1beta1.Installation) error {
logf("Setting installation state to installed")

// the installation will be in a ConfigMap at this point
copy, err := kubeutils.GetInstallation(ctx, cli, in.Name)
if err != nil {
return fmt.Errorf("get installation: %w", err)
}

copy.Status.SetState(v1beta1.InstallationStateInstalled, "V2MigrationComplete", nil)
meta.RemoveStatusCondition(&copy.Status.Conditions, ecv1beta1.ConditionTypeV2MigrationInProgress)
meta.RemoveStatusCondition(&copy.Status.Conditions, ecv1beta1.ConditionTypeDisableReconcile)

err = kubeutils.UpdateInstallationStatus(ctx, cli, copy)
if err != nil {
return fmt.Errorf("update installation status: %w", err)
}

logf("Successfully set installation state to installed")
return nil
}

func setInstallationCondition(ctx context.Context, cli client.Client, in *ecv1beta1.Installation, condition metav1.Condition) error {
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
var copy ecv1beta1.Installation
err := cli.Get(ctx, client.ObjectKey{Name: in.Name}, &copy)
if err != nil {
return fmt.Errorf("get installation: %w", err)
}

copy.Status.SetCondition(condition)

err = cli.Status().Update(ctx, &copy)
if err != nil {
return fmt.Errorf("update installation status: %w", err)
}

return nil
})
}
354 changes: 153 additions & 201 deletions operator/pkg/cli/migratev2/installation_test.go

Large diffs are not rendered by default.

17 changes: 16 additions & 1 deletion operator/pkg/cli/migratev2/migrate.go
Original file line number Diff line number Diff line change
@@ -20,7 +20,17 @@ func Run(
in *ecv1beta1.Installation,
migrationSecret string, appSlug string, appVersionLabel string,
) error {
err := runManagerInstallPodsAndWait(ctx, logf, cli, in, migrationSecret, appSlug, appVersionLabel)
err := setV2MigrationInProgress(ctx, logf, cli, in)
if err != nil {
return fmt.Errorf("set v2 migration in progress: %w", err)
}

err = waitForInstallationStateInstalled(ctx, logf, cli, in)
if err != nil {
return fmt.Errorf("failed to wait for addon installation: %w", err)
}

err = runManagerInstallPodsAndWait(ctx, logf, cli, in, migrationSecret, appSlug, appVersionLabel)
if err != nil {
return fmt.Errorf("run manager install pods: %w", err)
}
@@ -46,6 +56,11 @@ func Run(
return fmt.Errorf("enable v2 admin console: %w", err)
}

err = ensureInstallationStateInstalled(ctx, logf, cli, in)
if err != nil {
return fmt.Errorf("set installation state to installed: %w", err)
}

err = cleanupV1(ctx, logf, cli)
if err != nil {
return fmt.Errorf("cleanup v1: %w", err)
37 changes: 12 additions & 25 deletions operator/pkg/cli/migratev2/operator.go
Original file line number Diff line number Diff line change
@@ -10,40 +10,27 @@ import (
ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
"github.com/replicatedhq/embedded-cluster/pkg/helm"
"github.com/replicatedhq/embedded-cluster/pkg/helpers"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
apitypes "k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)

// disableOperator sets the DisablingReconcile condition to true on the installation object which
// will prevent the operator from reconciling the installation.
func disableOperator(ctx context.Context, logf LogFunc, cli client.Client, in *ecv1beta1.Installation) error {
logf("Disabling operator")

for timer := time.NewTimer(0); ; timer = time.NewTimer(2 * time.Second) {
<-timer.C

var copy ecv1beta1.Installation
err := cli.Get(ctx, client.ObjectKey{Name: in.Name}, &copy)
if err != nil {
return fmt.Errorf("get installation: %w", err)
}

copy.Status.SetCondition(metav1.Condition{
Type: ecv1beta1.DisableReconcileConditionType,
Status: metav1.ConditionTrue,
Reason: "V2MigrationInProgress",
})

err = cli.Status().Update(ctx, &copy)
if k8serrors.IsConflict(err) {
continue
} else if err != nil {
return fmt.Errorf("update installation status: %w", err)
}

logf("Successfully disabled operator")
return nil
err := setInstallationCondition(ctx, cli, in, metav1.Condition{
Type: ecv1beta1.ConditionTypeDisableReconcile,
Status: metav1.ConditionTrue,
Reason: "V2MigrationInProgress",
})
if err != nil {
return fmt.Errorf("set disable reconcile condition: %w", err)
}

logf("Successfully disabled operator")
return nil
}

// cleanupV1 removes control of the Helm Charts from the k0s controller and uninstalls the Embedded
2 changes: 1 addition & 1 deletion operator/pkg/cli/migratev2/operator_test.go
Original file line number Diff line number Diff line change
@@ -62,7 +62,7 @@ func Test_disableOperator(t *testing.T) {
require.NoError(t, err)

// Check that the DisableReconcile condition was set correctly
condition := meta.FindStatusCondition(updatedInstallation.Status.Conditions, ecv1beta1.DisableReconcileConditionType)
condition := meta.FindStatusCondition(updatedInstallation.Status.Conditions, ecv1beta1.ConditionTypeDisableReconcile)
require.NotNil(t, condition)
assert.Equal(t, metav1.ConditionTrue, condition.Status)
assert.Equal(t, "V2MigrationInProgress", condition.Reason)
2 changes: 2 additions & 0 deletions pkg/kubeutils/kubeutils.go
Original file line number Diff line number Diff line change
@@ -228,6 +228,8 @@ func (k *KubeUtils) WaitForInstallation(ctx context.Context, cli client.Client,
}

func CreateInstallation(ctx context.Context, cli client.Client, in *ecv1beta1.Installation) error {
in.Spec.SourceType = ecv1beta1.InstallationSourceTypeConfigMap

data, err := json.Marshal(in)
if err != nil {
return fmt.Errorf("marshal installation: %w", err)

0 comments on commit ccc7fcd

Please sign in to comment.