feat: better prompt

flanksource · Dec 25, 2024 · 642912a · 642912a
1 parent 0d9435f
commit 642912a
Show file tree

Hide file tree

Showing 16 changed files with 370 additions and 73 deletions.
diff --git a/api/v1/playbook_actions.go b/api/v1/playbook_actions.go
@@ -279,21 +279,47 @@ type AIActionClient struct {
 	// Supported: anthropic (default), ollama, openai.
 	Backend api.LLMBackend `json:"backend,omitempty"`
 
-	// Example: gpt-4o for openai, claude-3-haiku-20240307 for Anthropic, llama3.1:8b for Ollama ...
+	// Model name based on the backend chosen.
+	// Example: gpt-4o for openai, claude-3-5-sonnet-latest for Anthropic, llama3.1:8b for Ollama
 	Model string `json:"model,omitempty"`
 
 	// BaseURL or API url.
-	// Example: server URL for ollama or custom url for Anthropic if using a proxy, ...
+	// Example: server URL for ollama or custom url for Anthropic if using a proxy
 	APIURL string `json:"apiURL,omitempty"`
 }
 
 type AIActionContext struct {
-	Config        string                 `json:"config" yaml:"config" template:"true"`
-	Changes       TimeMetadata           `json:"changes,omitempty" yaml:"changes,omitempty"`
-	Analysis      TimeMetadata           `json:"analysis,omitempty" yaml:"analysis,omitempty"`
+	// The config id to operate on.
+	// If not provided, the playbook's config is used.
+	Config string `json:"config,omitempty" yaml:"config,omitempty" template:"true"`
+
+	// Select changes for the config to provide as an additional context to the AI model.
+	Changes TimeMetadata `json:"changes,omitempty" yaml:"changes,omitempty"`
+
+	// Select analysis for the config to provide as an additional context to the AI model.
+	Analysis TimeMetadata `json:"analysis,omitempty" yaml:"analysis,omitempty"`
+
+	// Select related configs to provide as an additional context to the AI model.
 	Relationships []AIActionRelationship `json:"relationships,omitempty" yaml:"relationships,omitempty"`
 }
 
+func (t AIActionContext) ShouldFetchConfigChanges() bool {
+	// if changes are being fetched from relationships, we don't have to query
+	// the changes for just the config alone.
+
+	if t.Changes.Since == "" {
+		return false
+	}
+
+	for _, r := range t.Relationships {
+		if r.Changes.Since != "" {
+			return false
+		}
+	}
+
+	return true
+}
+
 type AIAction struct {
 	AIActionClient  `json:",inline" yaml:",inline"`
 	AIActionContext `json:",inline" yaml:",inline" template:"true"`

diff --git a/config/crds/mission-control.flanksource.com_incidentrules.yaml b/config/crds/mission-control.flanksource.com_incidentrules.yaml
@@ -74,6 +74,11 @@ spec:
                       type: string
                     fieldSelector:
                       type: string
+                    healths:
+                      description: Healths filter resources by the health
+                      items:
+                        type: string
+                      type: array
                     id:
                       type: string
                     includeDeleted:

diff --git a/config/crds/mission-control.flanksource.com_notificationsilences.yaml b/config/crds/mission-control.flanksource.com_notificationsilences.yaml
@@ -68,6 +68,11 @@ spec:
                       type: string
                     fieldSelector:
                       type: string
+                    healths:
+                      description: Healths filter resources by the health
+                      items:
+                        type: string
+                      type: array
                     id:
                       type: string
                     includeDeleted:

diff --git a/config/crds/mission-control.flanksource.com_playbooks.yaml b/config/crds/mission-control.flanksource.com_playbooks.yaml
@@ -45,6 +45,8 @@ spec:
                     ai:
                       properties:
                         analysis:
+                          description: Select analysis for the config to provide as
+                            an additional context to the AI model.
                           properties:
                             since:
                               type: string
@@ -97,29 +99,37 @@ spec:
                         apiURL:
                           description: |-
                             BaseURL or API url.
-                            Example: server URL for ollama or custom url for Anthropic if using a proxy, ...
+                            Example: server URL for ollama or custom url for Anthropic if using a proxy
                           type: string
                         backend:
                           description: |-
                             Optionally specify the LLM backend.
                             Supported: anthropic (default), ollama, openai.
                           type: string
                         changes:
+                          description: Select changes for the config to provide as
+                            an additional context to the AI model.
                           properties:
                             since:
                               type: string
                           required:
                           - since
                           type: object
                         config:
+                          description: |-
+                            The config id to operate on.
+                            If not provided, the playbook's config is used.
                           type: string
                         model:
-                          description: 'Example: gpt-4o for openai, claude-3-haiku-20240307
-                            for Anthropic, llama3.1:8b for Ollama ...'
+                          description: |-
+                            Model name based on the backend chosen.
+                            Example: gpt-4o for openai, claude-3-5-sonnet-latest for Anthropic, llama3.1:8b for Ollama
                           type: string
                         prompt:
                           type: string
                         relationships:
+                          description: Select related configs to provide as an additional
+                            context to the AI model.
                           items:
                             properties:
                               analysis:
@@ -137,13 +147,15 @@ spec:
                                 - since
                                 type: object
                               depth:
+                                description: max depth to traverse the relationship.
+                                  Defaults to 3
                                 type: integer
                               direction:
+                                description: use incoming/outgoing/all relationships.
                                 type: string
                             type: object
                           type: array
                       required:
-                      - config
                       - prompt
                       type: object
                     azureDevopsPipeline:
@@ -1383,6 +1395,11 @@ spec:
                       type: string
                     fieldSelector:
                       type: string
+                    healths:
+                      description: Healths filter resources by the health
+                      items:
+                        type: string
+                      type: array
                     id:
                       type: string
                     includeDeleted:
@@ -1439,6 +1456,11 @@ spec:
                       type: string
                     fieldSelector:
                       type: string
+                    healths:
+                      description: Healths filter resources by the health
+                      items:
+                        type: string
+                      type: array
                     id:
                       type: string
                     includeDeleted:
@@ -1495,6 +1517,11 @@ spec:
                       type: string
                     fieldSelector:
                       type: string
+                    healths:
+                      description: Healths filter resources by the health
+                      items:
+                        type: string
+                      type: array
                     id:
                       type: string
                     includeDeleted:

diff --git a/config/schemas/incident-rules.schema.json b/config/schemas/incident-rules.schema.json
@@ -484,6 +484,9 @@
         },
         "statuses": {
           "$ref": "#/$defs/Items"
+        },
+        "healths": {
+          "$ref": "#/$defs/Items"
         }
       },
       "additionalProperties": false,

diff --git a/config/schemas/notificationsilence.schema.json b/config/schemas/notificationsilence.schema.json
@@ -233,6 +233,9 @@
         },
         "statuses": {
           "$ref": "#/$defs/Items"
+        },
+        "healths": {
+          "$ref": "#/$defs/Items"
         }
       },
       "additionalProperties": false,

diff --git a/config/schemas/openapi.go b/config/schemas/openapi.go
@@ -0,0 +1,31 @@
+package schemas
+
+import (
+	"embed"
+	"net/http"
+
+	"github.com/samber/oops"
+	"github.com/xeipuuv/gojsonschema"
+)
+
+//go:embed *
+var Schemas embed.FS
+
+func ValidatePlaybookSpec(schema []byte) (error, error) {
+	return ValidateSpec("playbook-spec.schema.json", schema)
+}
+
+func ValidateSpec(path string, schema []byte) (error, error) {
+	var playbookSchemaLoader = gojsonschema.NewReferenceLoaderFileSystem("file:///"+path, http.FS(Schemas))
+	documentLoader := gojsonschema.NewBytesLoader(schema)
+	result, err := gojsonschema.Validate(playbookSchemaLoader, documentLoader)
+	if err != nil {
+		return nil, oops.Wrap(err)
+	}
+
+	if len(result.Errors()) != 0 {
+		return oops.Errorf("spec is invalid: %v", result.Errors()), nil
+	}
+
+	return nil, nil
+}
diff --git a/config/schemas/playbook-spec.schema.json b/config/schemas/playbook-spec.schema.json
@@ -1017,6 +1017,9 @@
         },
         "statuses": {
           "$ref": "#/$defs/Items"
+        },
+        "healths": {
+          "$ref": "#/$defs/Items"
         }
       },
       "additionalProperties": false,

diff --git a/config/schemas/playbook.schema.json b/config/schemas/playbook.schema.json
@@ -1171,6 +1171,9 @@
         },
         "statuses": {
           "$ref": "#/$defs/Items"
+        },
+        "healths": {
+          "$ref": "#/$defs/Items"
         }
       },
       "additionalProperties": false,

diff --git a/...res/playbooks/ai-diagnose-aws-openai.yaml → ...res/playbooks/ai-diagnose-aws-ollama.yaml b/...res/playbooks/ai-diagnose-aws-openai.yaml → ...res/playbooks/ai-diagnose-aws-ollama.yaml
@@ -14,16 +14,14 @@ spec:
     - name: prompt
       label: Prompt
       default: Using the context provided, as an AWS engineer find out why {{.config.name}} is not healthy
+      properties:
+        multiline: "true"
   actions:
     - name: query
       ai:
-        backend: openai
-        model: gpt-4o
-        apiKey:
-          valueFrom:
-            secretKeyRef:
-              name: openai
-              key: API_KEY
+        backend: ollama
+        model: llama3.1:8b
+        apiURL: http://localhost:11434
         prompt: '{{.params.prompt}}'
         changes:
           since: 2h

diff --git a/fixtures/playbooks/ai-diagnose-kubernetes-resource-ollama.yaml b/fixtures/playbooks/ai-diagnose-kubernetes-resource-ollama.yaml
diff --git a/fixtures/playbooks/ai-diagnose-kubernetes-resource.yaml b/fixtures/playbooks/ai-diagnose-kubernetes-resource.yaml
@@ -0,0 +1,45 @@
+---
+# yaml-language-server: $schema=../../config/schemas/playbook.schema.json
+apiVersion: mission-control.flanksource.com/v1
+kind: Playbook
+metadata:
+  name: diagnose-kubernetes-resource
+spec:
+  description: Use AI to diagnose unhealthy kubernetes resources
+  configs:
+    - healths:
+        - unhealthy
+        - warning
+  parameters:
+    - name: prompt
+      label: Prompt
+      type: text
+      default: |
+        You are a Kubernetes expert. Use the context provided to find out why {{.config.name}} is unhealthy.
+        Explain your investigation in steps and then give a concise expert-level summary of your findings.
+      properties:
+        multiline: 'true'
+  actions:
+    - name: query
+      ai:
+        backend: anthropic
+        model: claude-3-5-sonnet-latest
+        apiKey:
+          valueFrom:
+            secretKeyRef:
+              name: anthropic
+              key: API_KEY
+        prompt: '{{.params.prompt}}'
+        changes:
+          since: 2d
+        analysis:
+          since: 2d
+        relationships:
+          - depth: 3
+            direction: outgoing
+            changes:
+              since: 12h
+          - depth: 5
+            direction: incoming
+            changes:
+              since: 12h
diff --git a/go.mod b/go.mod
@@ -366,7 +366,7 @@ require (
 	sigs.k8s.io/yaml v1.4.0
 )
 
-replace github.com/flanksource/duty => ../duty
+// replace github.com/flanksource/duty => ../duty
 
 // replace github.com/flanksource/gomplate/v3 => ../gomplate
 

diff --git a/go.sum b/go.sum
@@ -462,6 +462,8 @@ github.com/flanksource/artifacts v1.0.14 h1:Vv70bccsae0MwGaf/uSPp34J5V1/PyKfct9z
 github.com/flanksource/artifacts v1.0.14/go.mod h1:qHVCnQu5k50aWNJ5UhpcAKEl7pAzqUrFFKGSm147G70=
 github.com/flanksource/commons v1.35.3 h1:EG46iWodmSQQbXywjvEAgK56ZH26jYtMv0RiPM3eKDE=
 github.com/flanksource/commons v1.35.3/go.mod h1:cLZURmvF0dw3wR5VyPuibRl/7h+NEiyMdo70WhJFB9Y=
+github.com/flanksource/duty v1.0.782 h1:LfVwzDh/DEX2CEpaRfSBiybmBWjqBw0yVQjAHQL+8gU=
+github.com/flanksource/duty v1.0.782/go.mod h1:zoqvYtkrZDeAWAp/DmFZEpRZ+YsoFN3SDsDawk36vDo=
 github.com/flanksource/gomplate/v3 v3.24.49 h1:j28zhFoyL4wOC4uIRk6RMV1LzW/UtcG2sjR+P8zE7BA=
 github.com/flanksource/gomplate/v3 v3.24.49/go.mod h1:2qqMPYn/4SJUpeqTP/LChqvEPT5hd/7bQhDVBET4ZKY=
 github.com/flanksource/is-healthy v1.0.56 h1:FV57lH2CRB3ox8ifM9MJPN/XTmonlEiS1L3f8JcwhY4=