Skip to content

Commit 5640f50

Browse files
Fix mage integration:clean to actually delete instances and stacks (#3201) (#3219)
(cherry picked from commit decadc9) Co-authored-by: Blake Rouse <[email protected]>
1 parent 859bbc8 commit 5640f50

File tree

7 files changed

+195
-26
lines changed

7 files changed

+195
-26
lines changed

.buildkite/scripts/steps/integration_tests.sh

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,8 @@ DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64 PACKAGES=
88

99
# Run integration tests
1010
set +e
11-
SNAPSHOT=true mage integration:test
11+
TEST_INTEG_CLEAN_ON_EXIT=true SNAPSHOT=true mage integration:test
1212
TESTS_EXIT_STATUS=$?
13-
mage integration:clean
1413
set -e
1514

1615
# HTML report

magefile.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1468,9 +1468,15 @@ func integRunner(ctx context.Context, matrix bool, singleTest string) error {
14681468
return err
14691469
}
14701470
if failedCount > 0 {
1471+
if hasCleanOnExit() {
1472+
mg.Deps(Integration.Clean)
1473+
}
14711474
os.Exit(1)
14721475
}
14731476
if !hasRunUntilFailure() {
1477+
if hasCleanOnExit() {
1478+
mg.Deps(Integration.Clean)
1479+
}
14741480
return nil
14751481
}
14761482
}
@@ -1588,6 +1594,7 @@ func createTestRunner(matrix bool, singleTest string, goTestFlags string, batche
15881594
BuildDir: agentBuildDir,
15891595
GOVersion: goVersion,
15901596
RepoDir: ".",
1597+
StateDir: ".integration-cache",
15911598
DiagnosticsDir: diagDir,
15921599
Platforms: testPlatforms(),
15931600
Matrix: matrix,
@@ -1974,3 +1981,9 @@ func hasRunUntilFailure() bool {
19741981
b, _ := strconv.ParseBool(runUntil)
19751982
return b
19761983
}
1984+
1985+
func hasCleanOnExit() bool {
1986+
clean := os.Getenv("TEST_INTEG_CLEAN_ON_EXIT")
1987+
b, _ := strconv.ParseBool(clean)
1988+
return b
1989+
}

pkg/testing/ogc/provisioner.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ func (p *provisioner) Supported(os define.OS) bool {
5252
return ok
5353
}
5454

55-
func (p *provisioner) Provision(ctx context.Context, batches []runner.OSBatch) ([]runner.Instance, error) {
55+
func (p *provisioner) Provision(ctx context.Context, cfg runner.Config, batches []runner.OSBatch) ([]runner.Instance, error) {
5656
// ensure the latest version
5757
pullCtx, pullCancel := context.WithTimeout(ctx, 5*time.Minute)
5858
defer pullCancel()
@@ -64,7 +64,7 @@ func (p *provisioner) Provision(ctx context.Context, batches []runner.OSBatch) (
6464
// import the calculated layouts
6565
importCtx, importCancel := context.WithTimeout(ctx, 30*time.Second)
6666
defer importCancel()
67-
err = p.ogcImport(importCtx, batches)
67+
err = p.ogcImport(importCtx, cfg, batches)
6868
if err != nil {
6969
return nil, err
7070
}
@@ -116,7 +116,7 @@ func (p *provisioner) Provision(ctx context.Context, batches []runner.OSBatch) (
116116
}
117117

118118
// Clean cleans up all provisioned resources.
119-
func (p *provisioner) Clean(ctx context.Context, _ []runner.Instance) error {
119+
func (p *provisioner) Clean(ctx context.Context, cfg runner.Config, _ []runner.Instance) error {
120120
return p.ogcDown(ctx)
121121
}
122122

@@ -142,10 +142,10 @@ func (p *provisioner) ogcPull(ctx context.Context) error {
142142
}
143143

144144
// ogcImport imports all the required batches into OGC.
145-
func (p *provisioner) ogcImport(ctx context.Context, batches []runner.OSBatch) error {
145+
func (p *provisioner) ogcImport(ctx context.Context, cfg runner.Config, batches []runner.OSBatch) error {
146146
var layouts []Layout
147147
for _, ob := range batches {
148-
layouts = append(layouts, osBatchToOGC(ob))
148+
layouts = append(layouts, osBatchToOGC(cfg.StateDir, ob))
149149
}
150150
layoutData, err := yaml.Marshal(struct {
151151
Layouts []Layout `yaml:"layouts"`
@@ -279,7 +279,7 @@ func (p *provisioner) ogcRun(ctx context.Context, args []string, interactive boo
279279
return process.Start("docker", opts...)
280280
}
281281

282-
func osBatchToOGC(batch runner.OSBatch) Layout {
282+
func osBatchToOGC(cacheDir string, batch runner.OSBatch) Layout {
283283
tags := []string{
284284
LayoutIntegrationTag,
285285
batch.OS.Type,
@@ -305,8 +305,8 @@ func osBatchToOGC(batch runner.OSBatch) Layout {
305305
RemotePath: los.RemotePath,
306306
Scale: 1,
307307
Username: los.Username,
308-
SSHPrivateKey: ".integration-cache/id_rsa",
309-
SSHPublicKey: ".integration-cache/id_rsa.pub",
308+
SSHPrivateKey: cacheDir + "/id_rsa",
309+
SSHPublicKey: cacheDir + "/id_rsa.pub",
310310
Ports: []string{"22:22"},
311311
Tags: tags,
312312
Labels: map[string]string{

pkg/testing/runner/config.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ type Config struct {
1919
BuildDir string
2020
GOVersion string
2121
RepoDir string
22+
StateDir string
2223
DiagnosticsDir string
2324

2425
// Platforms filters the tests to only run on the provided list
@@ -63,6 +64,9 @@ func (c *Config) Validate() error {
6364
if c.RepoDir == "" {
6465
return errors.New("field RepoDir must be set")
6566
}
67+
if c.StateDir == "" {
68+
return errors.New("field StateDir must be set")
69+
}
6670
_, err := c.GetPlatforms()
6771
if err != nil {
6872
return err

pkg/testing/runner/provisioner.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,10 @@ type InstanceProvisioner interface {
4141
// Provision brings up the machines.
4242
//
4343
// The provision should re-use already prepared instances when possible.
44-
Provision(ctx context.Context, batches []OSBatch) ([]Instance, error)
44+
Provision(ctx context.Context, cfg Config, batches []OSBatch) ([]Instance, error)
4545

4646
// Clean cleans up all provisioned resources.
47-
Clean(ctx context.Context, instances []Instance) error
47+
Clean(ctx context.Context, cfg Config, instances []Instance) error
4848
}
4949

5050
// Stack is a created stack.

pkg/testing/runner/runner.go

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -171,14 +171,20 @@ func NewRunner(cfg Config, ip InstanceProvisioner, sp StackProvisioner, batches
171171
}
172172
osBatches = filterSupportedOS(osBatches, ip)
173173

174-
return &Runner{
174+
r := &Runner{
175175
cfg: cfg,
176176
logger: logger,
177177
ip: ip,
178178
sp: sp,
179179
batches: osBatches,
180180
batchToStack: make(map[string]Stack),
181-
}, nil
181+
}
182+
183+
err = r.loadState()
184+
if err != nil {
185+
return nil, err
186+
}
187+
return r, nil
182188
}
183189

184190
// Logger returns the logger used by the runner.
@@ -188,14 +194,8 @@ func (r *Runner) Logger() Logger {
188194

189195
// Run runs all the tests.
190196
func (r *Runner) Run(ctx context.Context) (Result, error) {
191-
// load the state
192-
err := r.loadState()
193-
if err != nil {
194-
return Result{}, err
195-
}
196-
197197
// validate tests can even be performed
198-
err = r.validate()
198+
err := r.validate()
199199
if err != nil {
200200
return Result{}, err
201201
}
@@ -228,7 +228,7 @@ func (r *Runner) Run(ctx context.Context) (Result, error) {
228228
}
229229
}
230230
if len(batches) > 0 {
231-
provisionedInstances, err := r.ip.Provision(ctx, batches)
231+
provisionedInstances, err := r.ip.Provision(ctx, r.cfg, batches)
232232
if err != nil {
233233
return Result{}, err
234234
}
@@ -272,7 +272,7 @@ func (r *Runner) Clean() error {
272272
g.Go(func() error {
273273
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
274274
defer cancel()
275-
return r.ip.Clean(ctx, instances)
275+
return r.ip.Clean(ctx, r.cfg, instances)
276276
})
277277
g.Go(func() error {
278278
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
@@ -316,7 +316,7 @@ func (r *Runner) runInstances(ctx context.Context, sshAuth ssh.AuthMethod, repoA
316316

317317
// runInstance runs the batch on the machine.
318318
func (r *Runner) runInstance(ctx context.Context, sshAuth ssh.AuthMethod, logger Logger, repoArchive string, batch OSBatch, instance StateInstance) (OSRunnerResult, error) {
319-
sshPrivateKeyPath, err := filepath.Abs(filepath.Join(".integration-cache", "id_rsa"))
319+
sshPrivateKeyPath, err := filepath.Abs(filepath.Join(r.cfg.StateDir, "id_rsa"))
320320
if err != nil {
321321
return OSRunnerResult{}, fmt.Errorf("failed to determine OGC SSH private key path: %w", err)
322322
}
@@ -464,7 +464,7 @@ func (r *Runner) prepare(ctx context.Context) (ssh.AuthMethod, string, error) {
464464
if err != nil {
465465
return nil, "", err
466466
}
467-
cacheDir := filepath.Join(wd, ".integration-cache")
467+
cacheDir := filepath.Join(wd, r.cfg.StateDir)
468468
_, err = os.Stat(cacheDir)
469469
if errors.Is(err, os.ErrNotExist) {
470470
err = os.Mkdir(cacheDir, 0755)
@@ -730,7 +730,7 @@ func (r *Runner) writeState() error {
730730
}
731731

732732
func (r *Runner) getStatePath() string {
733-
return filepath.Join(".integration-cache", "state.yml")
733+
return filepath.Join(r.cfg.StateDir, "state.yml")
734734
}
735735

736736
func (r *Runner) mergeResults(results map[string]OSRunnerResult) (Result, error) {

pkg/testing/runner/runner_test.go

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
2+
// or more contributor license agreements. Licensed under the Elastic License;
3+
// you may not use this file except in compliance with the Elastic License.
4+
5+
package runner
6+
7+
import (
8+
"context"
9+
"os"
10+
"path/filepath"
11+
"testing"
12+
13+
"github.com/stretchr/testify/assert"
14+
"github.com/stretchr/testify/require"
15+
16+
"github.com/elastic/elastic-agent/pkg/testing/define"
17+
)
18+
19+
func TestNewRunner_Clean(t *testing.T) {
20+
tmpdir := t.TempDir()
21+
stateDir := filepath.Join(tmpdir, "state")
22+
err := os.MkdirAll(stateDir, 0755)
23+
require.NoError(t, err)
24+
25+
cfg := Config{
26+
AgentVersion: "8.10.0",
27+
AgentStackVersion: "8.10.0-SNAPSHOT",
28+
BuildDir: filepath.Join(tmpdir, "build"),
29+
GOVersion: "1.20.7",
30+
RepoDir: filepath.Join(tmpdir, "repo"),
31+
StateDir: stateDir,
32+
ExtraEnv: nil,
33+
}
34+
ip := &fakeInstanceProvisioner{}
35+
sp := &fakeStackProvisioner{}
36+
r, err := NewRunner(cfg, ip, sp)
37+
require.NoError(t, err)
38+
39+
i1 := Instance{
40+
ID: "id-1",
41+
Name: "name-1",
42+
IP: "127.0.0.1",
43+
Username: "ubuntu",
44+
RemotePath: "/home/ubuntu/agent",
45+
Internal: map[string]interface{}{}, // ElementsMatch fails without this set
46+
}
47+
err = r.addOrUpdateInstance(StateInstance{
48+
Instance: i1,
49+
Prepared: true,
50+
})
51+
require.NoError(t, err)
52+
i2 := Instance{
53+
ID: "id-2",
54+
Name: "name-2",
55+
IP: "127.0.0.2",
56+
Username: "ubuntu",
57+
RemotePath: "/home/ubuntu/agent",
58+
Internal: map[string]interface{}{}, // ElementsMatch fails without this set
59+
}
60+
err = r.addOrUpdateInstance(StateInstance{
61+
Instance: i2,
62+
Prepared: true,
63+
})
64+
require.NoError(t, err)
65+
s1 := Stack{
66+
ID: "id-1",
67+
Version: "8.10.0",
68+
Internal: map[string]interface{}{}, // ElementsMatch fails without this set
69+
}
70+
err = r.addOrUpdateStack(s1)
71+
require.NoError(t, err)
72+
s2 := Stack{
73+
ID: "id-2",
74+
Version: "8.9.0",
75+
Internal: map[string]interface{}{}, // ElementsMatch fails without this set
76+
}
77+
err = r.addOrUpdateStack(s2)
78+
require.NoError(t, err)
79+
80+
// create the runner again ensuring that it loads the saved state
81+
r, err = NewRunner(cfg, ip, sp)
82+
require.NoError(t, err)
83+
84+
// clean should use the stored state
85+
err = r.Clean()
86+
require.NoError(t, err)
87+
88+
assert.ElementsMatch(t, ip.instances, []Instance{i1, i2})
89+
assert.ElementsMatch(t, sp.stacks, []Stack{s1, s2})
90+
}
91+
92+
type fakeInstanceProvisioner struct {
93+
batches []OSBatch
94+
instances []Instance
95+
}
96+
97+
func (f *fakeInstanceProvisioner) SetLogger(_ Logger) {
98+
}
99+
100+
func (f *fakeInstanceProvisioner) Supported(_ define.OS) bool {
101+
return true
102+
}
103+
104+
func (f *fakeInstanceProvisioner) Provision(_ context.Context, _ Config, batches []OSBatch) ([]Instance, error) {
105+
f.batches = batches
106+
var instances []Instance
107+
for _, batch := range batches {
108+
instances = append(instances, Instance{
109+
ID: batch.ID,
110+
Name: batch.ID,
111+
IP: "127.0.0.1",
112+
Username: "ubuntu",
113+
RemotePath: "/home/ubuntu/agent",
114+
Internal: nil,
115+
})
116+
}
117+
return instances, nil
118+
}
119+
120+
func (f *fakeInstanceProvisioner) Clean(_ context.Context, _ Config, instances []Instance) error {
121+
f.instances = instances
122+
return nil
123+
}
124+
125+
type fakeStackProvisioner struct {
126+
requests []StackRequest
127+
stacks []Stack
128+
}
129+
130+
func (f *fakeStackProvisioner) SetLogger(_ Logger) {
131+
}
132+
133+
func (f *fakeStackProvisioner) Provision(_ context.Context, requests []StackRequest) ([]Stack, error) {
134+
f.requests = requests
135+
var stacks []Stack
136+
for _, req := range requests {
137+
stacks = append(stacks, Stack{
138+
ID: req.ID,
139+
Version: req.Version,
140+
Elasticsearch: "http://localhost:9200",
141+
Kibana: "http://localhost:5601",
142+
Username: "elastic",
143+
Password: "changeme",
144+
Internal: nil,
145+
})
146+
}
147+
return stacks, nil
148+
}
149+
150+
func (f *fakeStackProvisioner) Clean(_ context.Context, stacks []Stack) error {
151+
f.stacks = stacks
152+
return nil
153+
}

0 commit comments

Comments
 (0)