diff --git a/.changeset/calm-wombats-sin.md b/.changeset/calm-wombats-sin.md new file mode 100644 index 000000000..6a8e75421 --- /dev/null +++ b/.changeset/calm-wombats-sin.md @@ -0,0 +1,5 @@ +--- +"chainlink-deployments-framework": minor +--- + +feat: run post-proposal hooks even when the timelock execution fails diff --git a/engine/cld/changeset/hooks.go b/engine/cld/changeset/hooks.go index 762a2055c..31c0a4507 100644 --- a/engine/cld/changeset/hooks.go +++ b/engine/cld/changeset/hooks.go @@ -85,6 +85,7 @@ type PostProposalHookParams struct { Proposal *mcms.TimelockProposal Config any Reports []MCMSTimelockExecuteReport + Err string // Deprecated: use `Config` instead. Will be removed in a future version. Input any diff --git a/engine/cld/changeset/mcms.go b/engine/cld/changeset/mcms.go index 9e450b326..734bfd1aa 100644 --- a/engine/cld/changeset/mcms.go +++ b/engine/cld/changeset/mcms.go @@ -83,7 +83,7 @@ func (*EVMForkContext) ChainFamily() string { // 2. Global post-proposal-hooks func (r *ChangesetsRegistry) RunProposalHooks( key string, e fdeployment.Environment, proposal *mcms.TimelockProposal, input, config any, - reports []MCMSTimelockExecuteReport, forkCtx ForkContext, + reports []MCMSTimelockExecuteReport, execError string, forkCtx ForkContext, ) error { applySnapshot, err := r.getApplySnapshot(key) if err != nil { @@ -112,6 +112,7 @@ func (r *ChangesetsRegistry) RunProposalHooks( Input: input, Config: config, Reports: reports, + Err: execError, } for _, h := range applySnapshot.registryEntry.postProposalHooks { diff --git a/engine/cld/changeset/mcms_test.go b/engine/cld/changeset/mcms_test.go index b9ff7e43a..d9cabec35 100644 --- a/engine/cld/changeset/mcms_test.go +++ b/engine/cld/changeset/mcms_test.go @@ -148,7 +148,7 @@ func Test_RunProposalHooks(t *testing.T) { execLogs := []string{} registry := tt.setup(&execLogs) - err := registry.RunProposalHooks(tt.key, hookTestEnv(t), &mcms.TimelockProposal{}, nil, nil, nil, nil) + err := registry.RunProposalHooks(tt.key, hookTestEnv(t), &mcms.TimelockProposal{}, nil, nil, nil, "", nil) if tt.wantErr == "" { require.NoError(t, err) @@ -182,7 +182,7 @@ func Test_RunProposalHooks_HookReceivesCorrectParams(t *testing.T) { }}, } - err := r.RunProposalHooks("test-cs", hookTestEnv(t), proposal, input, config, reports, nil) + err := r.RunProposalHooks("test-cs", hookTestEnv(t), proposal, input, config, reports, "", nil) require.NoError(t, err) expectedParams := PostProposalHookParams{ diff --git a/engine/cld/changeset/registry_test.go b/engine/cld/changeset/registry_test.go index 24ccfca5f..376912bf4 100644 --- a/engine/cld/changeset/registry_test.go +++ b/engine/cld/changeset/registry_test.go @@ -964,7 +964,7 @@ func Test_FluentAPI_HooksExtractedByAdd(t *testing.T) { require.Len(t, entry.postProposalHooks, 1, "Add should extract post-proposal-hooks via hookCarrier") require.Equal(t, "proposal-hook", entry.postProposalHooks[0].Name) - err := r.RunProposalHooks("test-cs", hookTestEnv(t), nil, "input", "config", nil, nil) + err := r.RunProposalHooks("test-cs", hookTestEnv(t), nil, "input", "config", nil, "", nil) require.NoError(t, err) require.Equal(t, []string{"proposal"}, hookExecutions) }) @@ -985,7 +985,7 @@ func Test_FluentAPI_HooksExtractedByAdd(t *testing.T) { require.Len(t, entry.postProposalHooks, 1) require.Equal(t, "proposal-hook", entry.postProposalHooks[0].Name) - err := r.RunProposalHooks("test-cs", hookTestEnv(t), nil, "input", "config", nil, nil) + err := r.RunProposalHooks("test-cs", hookTestEnv(t), nil, "input", "config", nil, "", nil) require.NoError(t, err) require.Equal(t, []string{"proposal"}, hookExecutions) }) diff --git a/engine/cld/commands/mcms/cmd_execute_fork.go b/engine/cld/commands/mcms/cmd_execute_fork.go index 8f729d731..299dc23fc 100644 --- a/engine/cld/commands/mcms/cmd_execute_fork.go +++ b/engine/cld/commands/mcms/cmd_execute_fork.go @@ -177,10 +177,6 @@ func executeFork( if len(chainConfig.HTTPRPCs) == 0 { return fmt.Errorf("no rpcs loaded in forked environment for chain %d (fork tests require public RPCs)", cfg.chainSelector) } - forkClient, ok := cfg.forkedEnv.ForkClients[cfg.chainSelector] - if !ok { - return fmt.Errorf("failed to get fork client for chain %d", cfg.chainSelector) - } // zkSync VM chains (zkSync Era, Lens, Cronos zkEVM, etc.) require anvil-zksync, // not standard Anvil. Derive this from the loaded chain which is set by the chain @@ -260,8 +256,14 @@ func executeFork( } lggr.Info("Executing timelock chain command") - reports, err := timelockExecuteChainCommand(ctx, lggr, cfg) - if err != nil { + reports, execErr := timelockExecuteChainCommand(ctx, lggr, cfg) + + herr := runPostProposalHooks(lggr, cfg, mcmsCfg, reports, errorString(execErr)) + if herr != nil { + lggr.Warnw("failed to run proposal hooks", "err", herr) + } + + if execErr != nil { lggr.Warnw("Timelock.execute() - failure; starting calling individual ops for debugging", "err", err) if derr := diagnoseTimelockRevert(ctx, lggr, anvilClient.URL, cfg.chainSelector, cfg.timelockProposal.Operations, timelockAddress, cfg.env.ExistingAddresses, cfg.proposalCtx); derr != nil { //nolint:staticcheck @@ -272,16 +274,33 @@ func executeFork( return fmt.Errorf("failed to timelock execute chain: %w", err) } + lggr.Info("Timelock.execute() - success") + return nil +} + +func runPostProposalHooks( + lggr logger.Logger, forkCfg *forkConfig, mcmsCfg Config, reports []cldfchangeset.MCMSTimelockExecuteReport, + execError string, +) error { if mcmsCfg.LoadChangesets == nil { lggr.Debug("LoadChangesets function not set in mcms config; skipping proposal hooks") return nil } + forkClient, ok := forkCfg.forkedEnv.ForkClients[forkCfg.chainSelector] + if !ok { + return fmt.Errorf("failed to get fork client for chain %d", forkCfg.chainSelector) + } + chainConfig, ok := forkCfg.forkedEnv.ChainConfigs[forkCfg.chainSelector] + if !ok { + return fmt.Errorf("failed to get forked env's chain config for chain %d", forkCfg.chainSelector) + } forkContext := &cldfchangeset.EVMForkContext{ChainConfig: chainConfig, Client: forkClient} - cfg.env.Name = cfg.envStr // ensure hooks load the correct env config for the fork - err = runHooksInternal(mcmsCfg, cfg.env, cfg.timelockProposal, reports, forkContext) + forkCfg.env.Name = forkCfg.envStr // ensure hooks load the correct env config for the fork + + err := runHooksInternal(mcmsCfg, forkCfg.env, forkCfg.timelockProposal, reports, execError, forkContext) if err != nil { return fmt.Errorf("failed to run post-proposal hooks: %w", err) } @@ -486,3 +505,11 @@ func (c *loggingRPCClient) SendTransaction(ctx context.Context, tx *gethtypes.Tr return c.OnchainClient.SendTransaction(ctx, tx) } + +func errorString(err error) string { + if err == nil { + return "" + } + + return err.Error() +} diff --git a/engine/cld/commands/mcms/cmd_run_hooks.go b/engine/cld/commands/mcms/cmd_run_hooks.go index 4cb7f3780..645576af9 100644 --- a/engine/cld/commands/mcms/cmd_run_hooks.go +++ b/engine/cld/commands/mcms/cmd_run_hooks.go @@ -42,6 +42,7 @@ type runHooksFlags struct { proposalKind string chainSelector uint64 reports []cldfchangeset.MCMSTimelockExecuteReport + error string } type proposalMetadata struct { @@ -78,6 +79,7 @@ func newRunProposalHooksCmd(cfg Config) *cobra.Command { proposalKind: flags.MustString(cmd.Flags().GetString("proposalKind")), chainSelector: flags.MustUint64(cmd.Flags().GetUint64("selector")), reports: reports, + error: flags.MustString(cmd.Flags().GetString("error")), } return runHooks(cmd.Context(), cfg, f) @@ -88,7 +90,8 @@ func newRunProposalHooksCmd(cfg Config) *cobra.Command { flags.Proposal(cmd) flags.ProposalKind(cmd, string(mcmstypes.KindTimelockProposal)) flags.ChainSelector(cmd, true) - cmd.Flags().String("report", "", "File with timelock execution report.") + cmd.Flags().String("report", "", "File with timelock execution report (required).") + cmd.Flags().String("error", "", "The error message, in case the timelock execution failed.") _ = cmd.MarkFlagRequired("report") return cmd @@ -114,7 +117,7 @@ func runHooks(ctx context.Context, cfg Config, hFlags runHooksFlags) error { return errors.New("expected proposal to be a TimelockProposal") } - return runHooksInternal(cfg, proposalCfg.Env, proposalCfg.TimelockProposal, hFlags.reports, nil) + return runHooksInternal(cfg, proposalCfg.Env, proposalCfg.TimelockProposal, hFlags.reports, hFlags.error, nil) } func runHooksInternal( @@ -122,6 +125,7 @@ func runHooksInternal( env cldf.Environment, timelockProposal *mcms.TimelockProposal, reports []cldfchangeset.MCMSTimelockExecuteReport, + execError string, forkCtx cldfchangeset.ForkContext, ) error { if cfg.LoadChangesets == nil { @@ -145,7 +149,7 @@ func runHooksInternal( }) herr := changesetRegistry.RunProposalHooks(changeset.Name, env, timelockProposal, - changeset.Input, changeset.Config, changesetReports, forkCtx) + changeset.Input, changeset.Config, changesetReports, execError, forkCtx) if herr != nil { return fmt.Errorf("proposal hook for changeset %q failed: %w", changeset.Name, herr) } diff --git a/engine/cld/commands/mcms/operations.go b/engine/cld/commands/mcms/operations.go index f6981dc03..d077a6ea6 100644 --- a/engine/cld/commands/mcms/operations.go +++ b/engine/cld/commands/mcms/operations.go @@ -196,51 +196,53 @@ func timelockExecuteChainCommand( reports := []cldfchangeset.MCMSTimelockExecuteReport{} for i := range cfg.timelockProposal.Operations { - if uint64(cfg.timelockProposal.Operations[i].ChainSelector) == cfg.chainSelector { - // Check if operation is done, if so, skip it - if err := executable.IsOperationDone(ctx, i); err == nil { - lggr.Warnf("Operation %d is already done, skipping...\n", i) - - continue - } - - if err := executable.IsOperationReady(ctx, i); err != nil { - return nil, fmt.Errorf("operation %d is not ready to be executed: %w", i, err) - } + if uint64(cfg.timelockProposal.Operations[i].ChainSelector) != cfg.chainSelector { + continue + } - timestamp := options.clockFn() + // Check if operation is done, if so, skip it + if err = executable.IsOperationDone(ctx, i); err == nil { + lggr.Warnf("Operation %d is already done, skipping...\n", i) + continue + } - result, err := executable.Execute(ctx, i, executeOptions...) - if err != nil { - return nil, fmt.Errorf("failed to execute operation %d: %w", i, err) - } + if err = executable.IsOperationReady(ctx, i); err != nil { + return nil, fmt.Errorf("operation %d is not ready to be executed: %w", i, err) + } - reports = append(reports, cldfchangeset.MCMSTimelockExecuteReport{ - ID: options.idFn(), - Type: cldfchangeset.MCMSTimelockExecuteReportType, - Status: "SUCCESS", - Timestamp: timestamp, - Input: cldfchangeset.MCMSTimelockExecuteReportInput{ - Index: i, - ChainSelector: cfg.chainSelector, - OperationID: operationIDs[i], - TimelockAddress: timelockAddress, - MCMAddress: chainMetadata.MCMAddress, - AdditionalFields: chainMetadata.AdditionalFields, - Changeset: findChangeset(cfg.timelockProposal, i), - }, - Output: cldfchangeset.MCMSTimelockExecuteReportOutput{ - TransactionResult: result, - }, - }) - - err = confirmTransaction(ctx, lggr, result, cfg) - if err != nil { - return nil, fmt.Errorf("failed to confirm execute transaction: %w", err) - } + timestamp := options.clockFn() + + reports = append(reports, cldfchangeset.MCMSTimelockExecuteReport{ + ID: options.idFn(), + Status: "SUCCESS", + Type: cldfchangeset.MCMSTimelockExecuteReportType, + Timestamp: timestamp, + Input: cldfchangeset.MCMSTimelockExecuteReportInput{ + Index: i, + ChainSelector: cfg.chainSelector, + OperationID: operationIDs[i], + TimelockAddress: timelockAddress, + MCMAddress: chainMetadata.MCMAddress, + AdditionalFields: chainMetadata.AdditionalFields, + Changeset: findChangeset(cfg.timelockProposal, i), + }, + Output: cldfchangeset.MCMSTimelockExecuteReportOutput{}, + }) + report := &reports[len(reports)-1] + + report.Output.TransactionResult, err = executable.Execute(ctx, i, executeOptions...) + if err != nil { + report.Status, report.Error = "FAILURE", err.Error() + return reports, fmt.Errorf("failed to execute operation %d: %w", i, err) + } - lggr.Infof("Operation %d executed successfully: %s\n", i, result) + err = confirmTransaction(ctx, lggr, report.Output.TransactionResult, cfg) + if err != nil { + report.Status, report.Error = "FAILURE", err.Error() + return reports, fmt.Errorf("failed to confirm execute transaction: %w", err) } + + lggr.Infof("Operation %d executed successfully: %s\n", i, report.Output.TransactionResult) } lggr.Infof("All operations executed successfully")