Skip to content

Commit 2f8f9f9

Browse files
authored
Add KIND provider to infra (#390)
This PR adds support of running test-infra components locally using the KIND provider. KIND specific commands are added to the infra CLI. When running locally, we don't have lot of CPU and memory resources. LOADGEN_SCALE_UP_REPLICAS deployment variable was added which makes sure that when run locally we're only scale to 2 fake-webservers. Signed-off-by: Raj Babu Das <[email protected]>
1 parent d76339f commit 2f8f9f9

File tree

10 files changed

+425
-2
lines changed

10 files changed

+425
-2
lines changed

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,5 @@ require (
2424
k8s.io/apiextensions-apiserver v0.18.4
2525
k8s.io/apimachinery v0.18.4
2626
k8s.io/client-go v0.18.4
27+
sigs.k8s.io/kind v0.8.1
2728
)

go.sum

Lines changed: 36 additions & 0 deletions
Large diffs are not rendered by default.

infra/README.md

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,25 @@ Commands:
6060
GKE_PROJECT_ID:test -v ZONE:europe-west1-b -v CLUSTER_NAME:test -v
6161
hashStable:COMMIT1 -v hashTesting:COMMIT2
6262
63+
kind info
64+
kind info -v hashStable:COMMIT1 -v hashTesting:COMMIT2
65+
66+
kind cluster create
67+
kind cluster create -f File -v PR_NUMBER:$PR_NUMBER -v
68+
CLUSTER_NAME:$CLUSTER_NAME
69+
70+
kind cluster delete
71+
kind cluster delete -f File -v PR_NUMBER:$PR_NUMBER -v
72+
CLUSTER_NAME:$CLUSTER_NAME
73+
74+
kind resource apply
75+
kind resource apply -f manifestsFileOrFolder -v hashStable:COMMIT1 -v
76+
hashTesting:COMMIT2
77+
78+
kind resource delete
79+
kind resource delete -f manifestsFileOrFolder -v hashStable:COMMIT1 -v
80+
hashTesting:COMMIT2
81+
6382
6483
```
6584

infra/infra.go

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"github.com/pkg/errors"
2323
"github.com/prometheus/test-infra/pkg/provider"
2424
"github.com/prometheus/test-infra/pkg/provider/gke"
25+
kind "github.com/prometheus/test-infra/pkg/provider/kind"
2526
"gopkg.in/alecthomas/kingpin.v2"
2627
)
2728

@@ -82,6 +83,30 @@ func main() {
8283
k8sGKEResource.Command("delete", "gke resource delete -a service-account.json -f manifestsFileOrFolder -v GKE_PROJECT_ID:test -v ZONE:europe-west1-b -v CLUSTER_NAME:test -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
8384
Action(g.ResourceDelete)
8485

86+
k := kind.New(dr)
87+
k8sKIND := app.Command("kind", `Kubernetes In Docker (KIND) provider - https://kind.sigs.k8s.io/docs/user/quick-start/`).
88+
Action(k.SetupDeploymentResources)
89+
90+
k8sKIND.Command("info", "kind info -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
91+
Action(k.GetDeploymentVars)
92+
93+
//Cluster operations.
94+
k8sKINDCluster := k8sKIND.Command("cluster", "manage KIND clusters").
95+
Action(k.KINDDeploymentsParse)
96+
k8sKINDCluster.Command("create", "kind cluster create -f File -v PR_NUMBER:$PR_NUMBER -v CLUSTER_NAME:$CLUSTER_NAME").
97+
Action(k.ClusterCreate)
98+
k8sKINDCluster.Command("delete", "kind cluster delete -f File -v PR_NUMBER:$PR_NUMBER -v CLUSTER_NAME:$CLUSTER_NAME").
99+
Action(k.ClusterDelete)
100+
101+
// K8s resource operations.
102+
k8sKINDResource := k8sKIND.Command("resource", `Apply and delete different k8s resources - deployments, services, config maps etc.`).
103+
Action(k.NewK8sProvider).
104+
Action(k.K8SDeploymentsParse)
105+
k8sKINDResource.Command("apply", "kind resource apply -f manifestsFileOrFolder -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
106+
Action(k.ResourceApply)
107+
k8sKINDResource.Command("delete", "kind resource delete -f manifestsFileOrFolder -v hashStable:COMMIT1 -v hashTesting:COMMIT2").
108+
Action(k.ResourceDelete)
109+
85110
if _, err := app.Parse(os.Args[1:]); err != nil {
86111
fmt.Fprintln(os.Stderr, errors.Wrapf(err, "Error parsing commandline arguments"))
87112
app.Usage(os.Args[1:])

pkg/provider/kind/kind.go

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
// Copyright 2020 The Prometheus Authors
2+
// Licensed under the Apache License, Version 2.0 (the "License");
3+
// you may not use this file except in compliance with the License.
4+
// You may obtain a copy of the License at
5+
//
6+
// http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
14+
package kind
15+
16+
import (
17+
"context"
18+
"fmt"
19+
"strings"
20+
21+
"github.com/pkg/errors"
22+
"github.com/prometheus/test-infra/pkg/provider"
23+
k8sProvider "github.com/prometheus/test-infra/pkg/provider/k8s"
24+
"gopkg.in/alecthomas/kingpin.v2"
25+
"k8s.io/apimachinery/pkg/runtime"
26+
"k8s.io/client-go/kubernetes/scheme"
27+
"k8s.io/client-go/tools/clientcmd"
28+
"k8s.io/client-go/util/homedir"
29+
"sigs.k8s.io/kind/pkg/cluster"
30+
"sigs.k8s.io/kind/pkg/cmd"
31+
)
32+
33+
type Resource = provider.Resource
34+
35+
// KIND holds the fields used to generate an API request.
36+
type KIND struct {
37+
38+
// The k8s provider used when we work with the manifest files.
39+
k8sProvider *k8sProvider.K8s
40+
// The kind provider used to instantiate a new provider.
41+
kindProvider *cluster.Provider
42+
// Final DeploymentFiles files.
43+
DeploymentFiles []string
44+
// Final DeploymentVars.
45+
DeploymentVars map[string]string
46+
// DeployResource to construct DeploymentVars and DeploymentFiles
47+
DeploymentResource *provider.DeploymentResource
48+
// Content bytes after parsing the template variables, grouped by filename.
49+
kindResources []Resource
50+
// K8s resource.runtime objects after parsing the template variables, grouped by filename.
51+
k8sResources []k8sProvider.Resource
52+
53+
ctx context.Context
54+
// KIND kuberconfig file
55+
kubeconfig string
56+
}
57+
58+
// New is the KIND constructor.
59+
func New(dr *provider.DeploymentResource) *KIND {
60+
return &KIND{
61+
DeploymentResource: dr,
62+
kindProvider: cluster.NewProvider(
63+
cluster.ProviderWithLogger(cmd.NewLogger()),
64+
),
65+
ctx: context.Background(),
66+
kubeconfig: homedir.HomeDir() + "/.kube/config",
67+
}
68+
}
69+
70+
// SetupDeploymentResources Sets up DeploymentVars and DeploymentFiles
71+
func (c *KIND) SetupDeploymentResources(*kingpin.ParseContext) error {
72+
customDeploymentVars := map[string]string{
73+
"NGINX_SERVICE_TYPE": "NodePort",
74+
"LOADGEN_SCALE_UP_REPLICAS": "2",
75+
}
76+
77+
c.DeploymentFiles = c.DeploymentResource.DeploymentFiles
78+
c.DeploymentVars = provider.MergeDeploymentVars(
79+
c.DeploymentResource.DefaultDeploymentVars,
80+
customDeploymentVars,
81+
c.DeploymentResource.FlagDeploymentVars,
82+
)
83+
return nil
84+
}
85+
86+
// KINDDeploymentsParse parses the environment/kind deployment files and saves the result as bytes grouped by the filename.
87+
// Any DeploymentVar will be replaced in the resources files following the golang text template format.
88+
func (c *KIND) KINDDeploymentsParse(*kingpin.ParseContext) error {
89+
if err := c.checkDeploymentVarsAndFiles(); err != nil {
90+
return err
91+
}
92+
93+
deploymentResource, err := provider.DeploymentsParse(c.DeploymentFiles, c.DeploymentVars)
94+
if err != nil {
95+
return err
96+
}
97+
c.kindResources = deploymentResource
98+
return nil
99+
}
100+
101+
func (c *KIND) K8SDeploymentsParse(*kingpin.ParseContext) error {
102+
if err := c.checkDeploymentVarsAndFiles(); err != nil {
103+
return err
104+
}
105+
106+
deploymentResource, err := provider.DeploymentsParse(c.DeploymentFiles, c.DeploymentVars)
107+
if err != nil {
108+
return err
109+
}
110+
for _, deployment := range deploymentResource {
111+
112+
decode := scheme.Codecs.UniversalDeserializer().Decode
113+
k8sObjects := make([]runtime.Object, 0)
114+
115+
for _, text := range strings.Split(string(deployment.Content), provider.Separator) {
116+
text = strings.TrimSpace(text)
117+
if len(text) == 0 {
118+
continue
119+
}
120+
121+
resource, _, err := decode([]byte(text), nil, nil)
122+
if err != nil {
123+
return errors.Wrapf(err, "decoding the resource file:%v, section:%v...", deployment.FileName, text[:100])
124+
}
125+
if resource == nil {
126+
continue
127+
}
128+
k8sObjects = append(k8sObjects, resource)
129+
}
130+
if len(k8sObjects) > 0 {
131+
c.k8sResources = append(c.k8sResources, k8sProvider.Resource{FileName: deployment.FileName, Objects: k8sObjects})
132+
}
133+
}
134+
return nil
135+
}
136+
137+
// checkDeploymentVarsAndFiles checks whether the requied deployment vars are passed.
138+
func (c *KIND) checkDeploymentVarsAndFiles() error {
139+
reqDepVars := []string{"CLUSTER_NAME"}
140+
for _, k := range reqDepVars {
141+
if v, ok := c.DeploymentVars[k]; !ok || v == "" {
142+
return fmt.Errorf("missing required %v variable", k)
143+
}
144+
}
145+
if len(c.DeploymentFiles) == 0 {
146+
return fmt.Errorf("missing deployment file(s)")
147+
}
148+
return nil
149+
}
150+
151+
// ClusterCreate create a new cluster or applies changes to an existing cluster.
152+
func (c *KIND) ClusterCreate(*kingpin.ParseContext) error {
153+
for _, deployment := range c.kindResources {
154+
CreateWithConfigFile := cluster.CreateWithRawConfig(deployment.Content)
155+
156+
err := c.kindProvider.Create(c.DeploymentVars["CLUSTER_NAME"], CreateWithConfigFile)
157+
if err != nil {
158+
return err
159+
}
160+
}
161+
return nil
162+
}
163+
164+
// ClusterDelete deletes a k8s cluster.
165+
func (c *KIND) ClusterDelete(*kingpin.ParseContext) error {
166+
err := c.kindProvider.Delete(c.DeploymentVars["CLUSTER_NAME"], c.kubeconfig)
167+
if err != nil {
168+
return err
169+
}
170+
return nil
171+
}
172+
173+
// NewK8sProvider sets the k8s provider used for deploying k8s manifests.
174+
func (c *KIND) NewK8sProvider(*kingpin.ParseContext) error {
175+
var err error
176+
apiConfig, err := clientcmd.LoadFromFile(c.kubeconfig)
177+
if err != nil {
178+
return err
179+
}
180+
181+
c.k8sProvider, err = k8sProvider.New(c.ctx, apiConfig)
182+
if err != nil {
183+
return err
184+
}
185+
return nil
186+
}
187+
188+
// ResourceApply calls k8s.ResourceApply to apply the k8s objects in the manifest files.
189+
func (c *KIND) ResourceApply(*kingpin.ParseContext) error {
190+
if err := c.k8sProvider.ResourceApply(c.k8sResources); err != nil {
191+
return err
192+
}
193+
return nil
194+
}
195+
196+
// ResourceDelete calls k8s.ResourceDelete to apply the k8s objects in the manifest files.
197+
func (c *KIND) ResourceDelete(*kingpin.ParseContext) error {
198+
if err := c.k8sProvider.ResourceDelete(c.k8sResources); err != nil {
199+
return err
200+
}
201+
return nil
202+
}
203+
204+
// GetDeploymentVars shows deployment variables.
205+
func (c *KIND) GetDeploymentVars(parseContext *kingpin.ParseContext) error {
206+
fmt.Print("-------------------\n DeploymentVars \n------------------- \n")
207+
for key, value := range c.DeploymentVars {
208+
fmt.Println(key, ": ", value)
209+
}
210+
return nil
211+
}

pkg/provider/provider.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,8 @@ func NewDeploymentResource() *DeploymentResource {
4747
DeploymentFiles: []string{},
4848
FlagDeploymentVars: map[string]string{},
4949
DefaultDeploymentVars: map[string]string{
50-
"NGINX_SERVICE_TYPE": "LoadBalancer",
50+
"NGINX_SERVICE_TYPE": "LoadBalancer",
51+
"LOADGEN_SCALE_UP_REPLICAS": "10",
5152
},
5253
}
5354
}

prombench/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ The `/manifest` directory contains all the kubernetes manifest files.
1818
Prombench can be run on various providers, following are the provider specific instructions:
1919

2020
- Instructions for [Google Kubernetes Engine](docs/gke.md)
21+
- Instructions for [Kubernetes In Docker](docs/kind.md)
2122

2223
## Setup GitHub Actions
2324

0 commit comments

Comments
 (0)