-
Notifications
You must be signed in to change notification settings - Fork 46
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Feat: prometheus check steps provider (#149)
* check-metrics Signed-off-by: 楚岳 <[email protected]> small fix Signed-off-by: 楚岳 <[email protected]> small fix Signed-off-by: 楚岳 <[email protected]> refactor some code Signed-off-by: 楚岳 <[email protected]> add tests Signed-off-by: 楚岳 <[email protected]> * try to fix go lint Signed-off-by: 楚岳 <[email protected]> small fix Signed-off-by: 楚岳 <[email protected]> small fix Signed-off-by: 楚岳 <[email protected]> * fix test Signed-off-by: 楚岳 <[email protected]> * fix comments Signed-off-by: 楚岳 <[email protected]> small fix Signed-off-by: 楚岳 <[email protected]> delete useless code Signed-off-by: 楚岳 <[email protected]> add nolint Signed-off-by: 楚岳 <[email protected]> small fix Signed-off-by: 楚岳 <[email protected]> small fix Signed-off-by: 楚岳 <[email protected]> fix comments Signed-off-by: 楚岳 <[email protected]> fix comments Signed-off-by: 楚岳 <[email protected]> --------- Signed-off-by: 楚岳 <[email protected]>
- Loading branch information
1 parent
5b55dbd
commit c730c05
Showing
6 changed files
with
321 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,210 @@ | ||
| /* | ||
| Copyright 2022 The KubeVela Authors. | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package metrics | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "strconv" | ||
| "time" | ||
|
|
||
| monitorContext "github.com/kubevela/pkg/monitor/context" | ||
| wfContext "github.com/kubevela/workflow/pkg/context" | ||
| "github.com/kubevela/workflow/pkg/cue/model/value" | ||
| "github.com/kubevela/workflow/pkg/types" | ||
| "github.com/prometheus/client_golang/api" | ||
| v1 "github.com/prometheus/client_golang/api/prometheus/v1" | ||
| "github.com/prometheus/common/model" | ||
| ) | ||
|
|
||
| const ( | ||
| // ProviderName is provider name for install. | ||
| ProviderName = "metrics" | ||
| ) | ||
|
|
||
| type provider struct{} | ||
|
|
||
| // PromCheck do health check from metrics from prometheus | ||
| func (h *provider) PromCheck(ctx monitorContext.Context, wfCtx wfContext.Context, v *value.Value, act types.Action) error { | ||
| stepID, err := v.GetString("stepID") | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| valueStr, err := getQueryResult(ctx, v) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| conditionStr, err := v.GetString("condition") | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| res, err := compareValueWithCondition(valueStr, conditionStr, v) | ||
|
|
||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if res { | ||
| // meet the condition | ||
| return handleSuccessCompare(wfCtx, stepID, v, conditionStr, valueStr) | ||
| } | ||
| return handleFailCompare(wfCtx, stepID, v, conditionStr, valueStr) | ||
| } | ||
|
|
||
| func handleSuccessCompare(wfCtx wfContext.Context, stepID string, v *value.Value, conditionStr, valueStr string) error { | ||
| // clean up fail timeStamp | ||
| setMetricsStatusTime(wfCtx, stepID, "fail", 0) | ||
| d, err := v.GetString("duration") | ||
| if err != nil { | ||
| return err | ||
| } | ||
| duration, err := time.ParseDuration(d) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| st := getMetricsStatusTime(wfCtx, stepID, "success") | ||
| if st == 0 { | ||
| // first success | ||
| if err := v.FillObject(fmt.Sprintf("The healthy condition should be %s, and the query result is %s, indicating success.", conditionStr, valueStr), "message"); err != nil { | ||
| return err | ||
| } | ||
| setMetricsStatusTime(wfCtx, stepID, "success", time.Now().Unix()) | ||
| return v.FillObject(false, "result") | ||
| } | ||
| successTime := time.Unix(st, 0) | ||
| if successTime.Add(duration).Before(time.Now()) { | ||
| if err = v.FillObject("The metric check has passed successfully.", "message"); err != nil { | ||
| return err | ||
| } | ||
| return v.FillObject(true, "result") | ||
| } | ||
| if err := v.FillObject(fmt.Sprintf("The healthy condition should be %s, and the query result is %s, indicating success. The success has persisted for %s, with success duration being %s.", conditionStr, valueStr, time.Since(successTime).String(), duration), "message"); err != nil { | ||
| return err | ||
| } | ||
| return v.FillObject(false, "result") | ||
| } | ||
|
|
||
| func handleFailCompare(wfCtx wfContext.Context, stepID string, v *value.Value, conditionStr, valueStr string) error { | ||
| // clean up success timeStamp | ||
| setMetricsStatusTime(wfCtx, stepID, "success", 0) | ||
| ft := getMetricsStatusTime(wfCtx, stepID, "") | ||
| d, err := v.GetString("failDuration") | ||
| if err != nil { | ||
| return err | ||
| } | ||
| failDuration, err := time.ParseDuration(d) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if ft == 0 { | ||
| // first failed | ||
| setMetricsStatusTime(wfCtx, stepID, "fail", time.Now().Unix()) | ||
| if err := v.FillObject(fmt.Sprintf("The healthy condition should be %s, but the query result is %s, indicating failure, with the failure duration being %s. This is first failed checking.", conditionStr, valueStr, failDuration), "message"); err != nil { | ||
| return err | ||
| } | ||
| return v.FillObject(false, "result") | ||
| } | ||
|
|
||
| failTime := time.Unix(ft, 0) | ||
| if failTime.Add(failDuration).Before(time.Now()) { | ||
| if err = v.FillObject(true, "failed"); err != nil { | ||
| return err | ||
| } | ||
| if err := v.FillObject(fmt.Sprintf("The healthy condition should be %s, but the query result is %s, indicating failure. The failure has persisted for %s, with the failure duration being %s. The check has terminated.", conditionStr, valueStr, time.Since(failTime).String(), failDuration), "message"); err != nil { | ||
| return err | ||
| } | ||
| return v.FillObject(false, "result") | ||
| } | ||
| if err := v.FillObject(fmt.Sprintf("The healthy condition should be %s, but the query result is %s, indicating failure. The failure has persisted for %s, with the failure duration being %s.", conditionStr, valueStr, time.Since(failTime).String(), failDuration), "message"); err != nil { | ||
| return err | ||
| } | ||
| return v.FillObject(false, "result") | ||
| } | ||
|
|
||
| func getQueryResult(ctx monitorContext.Context, v *value.Value) (string, error) { | ||
| addr, err := v.GetString("metricEndpoint") | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| c, err := api.NewClient(api.Config{ | ||
| Address: addr, | ||
| }) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| promCli := v1.NewAPI(c) | ||
| query, err := v.GetString("query") | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| resp, _, err := promCli.Query(ctx, query, time.Now()) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
|
|
||
| var valueStr string | ||
| switch v := resp.(type) { | ||
| case *model.Scalar: | ||
| valueStr = v.Value.String() | ||
| case model.Vector: | ||
| if len(v) != 1 { | ||
| return "", fmt.Errorf(fmt.Sprintf("ehe query is returning %d results when it should only return one. Please review the query to identify and fix the issue", len(v))) | ||
| } | ||
| valueStr = v[0].Value.String() | ||
| default: | ||
| return "", fmt.Errorf("cannot handle the not query value") | ||
| } | ||
| return valueStr, nil | ||
| } | ||
|
|
||
| func compareValueWithCondition(valueStr string, conditionStr string, v *value.Value) (bool, error) { | ||
| template := fmt.Sprintf("if: %s %s", valueStr, conditionStr) | ||
| cueValue, err := value.NewValue(template, nil, "") | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| res, err := cueValue.GetBool("if") | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| return res, nil | ||
| } | ||
|
|
||
| func setMetricsStatusTime(wfCtx wfContext.Context, stepID string, status string, time int64) { | ||
| wfCtx.SetMutableValue(strconv.FormatInt(time, 10), stepID, "metrics", status, "time") | ||
| } | ||
|
|
||
| func getMetricsStatusTime(wfCtx wfContext.Context, stepID string, status string) int64 { | ||
| str := wfCtx.GetMutableValue(stepID, "metrics", status, "time") | ||
| if len(str) == 0 { | ||
| return 0 | ||
| } | ||
| t, _ := strconv.ParseInt(str, 10, 64) | ||
| return t | ||
| } | ||
|
|
||
| // Install register handlers to provider discover. | ||
| func Install(p types.Providers) { | ||
| prd := &provider{} | ||
| p.Register(ProviderName, map[string]types.Handler{ | ||
| "promCheck": prd.PromCheck, | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| /* | ||
| Copyright 2022 The KubeVela Authors. | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
| package metrics | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "net/http" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/crossplane/crossplane-runtime/pkg/test" | ||
|
|
||
| monitorContext "github.com/kubevela/pkg/monitor/context" | ||
| context2 "github.com/kubevela/workflow/pkg/context" | ||
| "github.com/kubevela/workflow/pkg/cue/model/value" | ||
| "github.com/stretchr/testify/assert" | ||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
| ) | ||
|
|
||
| func TestMetricCheck(t *testing.T) { | ||
| srv := runMockPrometheusServer() // no lint | ||
|
|
||
| v, err := value.NewValue(` | ||
| metricEndpoint: "http://127.0.0.1:18089" | ||
| query: "sum(nginx_ingress_controller_requests{host=\"canary-demo.com\",status=\"200\"})" | ||
| duration: "4s" | ||
| failDuration: "2s" | ||
| condition: ">=3" | ||
| stepID: "123456"`, nil, "") | ||
| assert.NoError(t, err) | ||
| prd := &provider{} | ||
| ctx := monitorContext.NewTraceContext(context.Background(), "") | ||
| cli := &test.MockClient{ | ||
| MockCreate: func(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { | ||
| return nil | ||
| }, | ||
| MockPatch: func(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { | ||
| return nil | ||
| }, | ||
| MockGet: func(ctx context.Context, key client.ObjectKey, obj client.Object) error { | ||
| return nil | ||
| }, | ||
| } | ||
| wfCtx, err := context2.NewContext(context.Background(), cli, "default", "v1", nil) | ||
| assert.NoError(t, err) | ||
| err = prd.PromCheck(ctx, wfCtx, v, nil) | ||
| assert.NoError(t, err) | ||
| res, err := v.GetBool("result") | ||
| assert.NoError(t, err) | ||
| assert.Equal(t, res, false) | ||
| message, err := v.GetString("message") | ||
| assert.NoError(t, err) | ||
| assert.Equal(t, message, "The healthy condition should be >=3, and the query result is 10, indicating success.") | ||
| if err := srv.Close(); err != nil { | ||
| fmt.Printf("Server shutdown error: %v\n", err) | ||
| } | ||
| } | ||
|
|
||
| func runMockPrometheusServer() *http.Server { | ||
| srv := http.Server{Addr: ":18089", Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
| w.Header().Set("Content-Type", "application/json") | ||
| w.Write([]byte(`{ | ||
| "status": "success", | ||
| "data": { | ||
| "resultType": "vector", | ||
| "result": [ | ||
| { | ||
| "metric": {}, | ||
| "value": [ | ||
| 1678701380.73, | ||
| "10" | ||
| ] | ||
| } | ||
| ] | ||
| } | ||
| }`)) | ||
| })} | ||
| go srv.ListenAndServe() // no lint | ||
| time.Sleep(3 * time.Second) | ||
| return &srv | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| #PromCheck: { | ||
| #do: "promCheck" | ||
| #provider: "metrics" | ||
|
|
||
| query: string | ||
| promAddress: string | ||
| condition: string | ||
| duration: string | ||
| ... | ||
| } |