From a43e7f0fbc5c81401a6a93dce1e344c0ee3ae659 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 5 Sep 2025 14:19:02 -0500 Subject: [PATCH 1/9] DRAFT tool: defra-kv -- key-value store via defradb embedded in CLI tool This is a first-draft of a tool that satisfies the following design priorities: - written in go - acts as CLI tool - embeds defradb in-process - persists to disk (via Badger) - creates a basic key-value type store (via an embedded `type` DDL) - accepts parameters to set config, like keyring-secret and data-root - primary input is a graphql-form query against the defra, such as `query { .. }` or `mutation { .. }` - accepts query input either via stdin or `-query` CLI parameter - includes a few helpful affordances such as home-dir expansion, default root dir, pretty JSON printing, etc. --- defra-kv.go | 175 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 defra-kv.go diff --git a/defra-kv.go b/defra-kv.go new file mode 100644 index 0000000..6aebf2d --- /dev/null +++ b/defra-kv.go @@ -0,0 +1,175 @@ +// main.go +package main + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "os" + "os/signal" + "path/filepath" + "strings" + "syscall" + "time" + + dclient "github.com/sourcenetwork/defradb/client" + dnode "github.com/sourcenetwork/defradb/node" +) + +func defaultRootdir() string { + if cwd, err := os.Getwd(); err == nil { + return filepath.Join(cwd, ".defra-kv") + } + return ".defra-kv" +} + +func expandHome(p string) string { + if strings.HasPrefix(p, "~/") { + if h, err := os.UserHomeDir(); err == nil { + return filepath.Join(h, p[2:]) + } + } + return p +} + +func resolveRootdir(p string) string { + p = expandHome(p) + if !filepath.IsAbs(p) { + if abs, err := filepath.Abs(p); err == nil { + p = abs + } + } + if err := os.MkdirAll(p, 0o755); err != nil { + log.Fatalf("create rootdir: %v", err) + } + return p +} + +// Single JSON-based KV schema with indexes where useful. +const kvSchema = ` +type KV { + key: String @index + value: JSON + updatedAt: DateTime @index +} +` + +func kvExists(ctx context.Context, n *dnode.Node) bool { + res := n.DB.ExecRequest(ctx, `query { __type(name: "KV") { name } }`) + if len(res.GQL.Errors) > 0 { + return false + } + b, err := json.Marshal(res.GQL.Data) + if err != nil { + return false + } + return bytes.Contains(b, []byte(`"name":"KV"`)) +} + +func ensureKV(ctx context.Context, n *dnode.Node) error { + if kvExists(ctx, n) { + return nil + } + if _, err := n.DB.AddSchema(ctx, kvSchema); err != nil { + return fmt.Errorf("KV schema add failed: %v", err) + } + return nil +} + +func main() { + // ---- Custom FlagSet (avoid random flags from transitive deps) ---- + fs := flag.NewFlagSet("defra-kv", flag.ExitOnError) + rootdir := fs.String("rootdir", defaultRootdir(), "data/config directory (default: ./.defra-kv)") + secret := fs.String("keyring-secret", "", "keyring secret (or set DEFRA_KEYRING_SECRET)") + query := fs.String("query", "", "GraphQL query/mutation; if empty, read from stdin") + varsStr := fs.String("vars", "", "JSON variables (optional)") + pretty := fs.Bool("pretty", true, "pretty-print JSON") + reqTO := fs.Duration("timeout", 10*time.Second, "per-request timeout") + _ = fs.Parse(os.Args[1:]) + + // Keyring secret (first run convenience). + if *secret != "" { + _ = os.Setenv("DEFRA_KEYRING_SECRET", *secret) + } + if os.Getenv("DEFRA_KEYRING_SECRET") == "" { + _ = os.Setenv("DEFRA_KEYRING_SECRET", "dev-dev-dev") + } + + // Read query (flag or stdin). + q := strings.TrimSpace(*query) + if q == "" { + b, err := io.ReadAll(os.Stdin) + if err != nil { + log.Fatalf("read stdin: %v", err) + } + q = strings.TrimSpace(string(b)) + } + if q == "" { + fmt.Fprintln(os.Stderr, "no query provided; pass --query or pipe to stdin") + os.Exit(2) + } + + // Variables (optional, parsed later). + var rawVars json.RawMessage + if v := strings.TrimSpace(*varsStr); v != "" { + rawVars = json.RawMessage(v) + } + + // Context + signals. + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + // --- 1) Create the node (embedded, persistent Badger) --- + n, err := dnode.New( + ctx, + dnode.WithDisableAPI(true), // no HTTP server + dnode.WithDisableP2P(true), // local only + dnode.WithBadgerInMemory(false), // persistent + dnode.WithStoreType(dnode.BadgerStore), + dnode.WithStorePath(resolveRootdir(*rootdir)), // data dir + dnode.WithLensRuntime(dnode.Wazero), // pure-Go WASM runtime + ) + if err != nil { + log.Fatalf("node.New: %v", err) + } + defer func() { _ = n.Close(ctx) }() + if err := n.Start(ctx); err != nil { + log.Fatalf("node.Start: %v", err) + } + + // --- 2) Ensure KV schema (idempotent) --- + if err := ensureKV(ctx, n); err != nil { + log.Fatalf("ensure KV schema: %v", err) + } + + // --- 3) Execute the user’s GraphQL directly in-process --- + var vars map[string]any + if len(rawVars) > 0 { + if err := json.Unmarshal(rawVars, &vars); err != nil { + log.Fatalf("parse -vars: %v", err) + } + } + + reqCtx, cancel := context.WithTimeout(ctx, *reqTO) + defer cancel() + + res := n.DB.ExecRequest(reqCtx, q, dclient.WithVariables(vars)) + if len(res.GQL.Errors) > 0 { + enc, _ := json.MarshalIndent(res.GQL.Errors, "", " ") + fmt.Fprintln(os.Stderr, string(enc)) + os.Exit(1) + } + + // Output JSON (pretty or compact). + if *pretty { + out, _ := json.MarshalIndent(map[string]any{"data": res.GQL.Data}, "", " ") + fmt.Println(string(out)) + } else { + out, _ := json.Marshal(map[string]any{"data": res.GQL.Data}) + fmt.Println(string(out)) + } +} From 54038721fe77dfcd2fb371c7c561a87f1a18750a Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Sat, 6 Sep 2025 11:50:53 -0500 Subject: [PATCH 2/9] tweaks/improvements --- defra-kv.go | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/defra-kv.go b/defra-kv.go index 6aebf2d..a72766d 100644 --- a/defra-kv.go +++ b/defra-kv.go @@ -1,4 +1,3 @@ -// main.go package main import ( @@ -81,17 +80,17 @@ func ensureKV(ctx context.Context, n *dnode.Node) error { } func main() { - // ---- Custom FlagSet (avoid random flags from transitive deps) ---- + // Define Custom FlagSet fs := flag.NewFlagSet("defra-kv", flag.ExitOnError) - rootdir := fs.String("rootdir", defaultRootdir(), "data/config directory (default: ./.defra-kv)") - secret := fs.String("keyring-secret", "", "keyring secret (or set DEFRA_KEYRING_SECRET)") - query := fs.String("query", "", "GraphQL query/mutation; if empty, read from stdin") - varsStr := fs.String("vars", "", "JSON variables (optional)") + rootdir := fs.String("rootdir", defaultRootdir(), "data/config directory") + secret := fs.String("keyring-secret", "", "keyring secret (sets DEFRA_KEYRING_SECRET)") + query := fs.String("query", "", "GraphQL query/mutation") + varsStr := fs.String("vars", "", "JSON variables") pretty := fs.Bool("pretty", true, "pretty-print JSON") reqTO := fs.Duration("timeout", 10*time.Second, "per-request timeout") _ = fs.Parse(os.Args[1:]) - // Keyring secret (first run convenience). + // Process keyring secret if *secret != "" { _ = os.Setenv("DEFRA_KEYRING_SECRET", *secret) } @@ -109,21 +108,27 @@ func main() { q = strings.TrimSpace(string(b)) } if q == "" { - fmt.Fprintln(os.Stderr, "no query provided; pass --query or pipe to stdin") + fmt.Fprintln(os.Stderr, "no query provided; pass -query or pipe to stdin") os.Exit(2) } - // Variables (optional, parsed later). - var rawVars json.RawMessage + // Parse user-provided variables (if any) + var vars map[string]any if v := strings.TrimSpace(*varsStr); v != "" { - rawVars = json.RawMessage(v) + var rawVars json.RawMessage = json.RawMessage(v) + + if len(rawVars) > 0 { + if err := json.Unmarshal(rawVars, &vars); err != nil { + log.Fatalf("parse -vars: %v", err) + } + } } - // Context + signals. + // Initialize context and signals ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() - // --- 1) Create the node (embedded, persistent Badger) --- + // Create and start the node (embedded, persistent Badger) n, err := dnode.New( ctx, dnode.WithDisableAPI(true), // no HTTP server @@ -134,29 +139,24 @@ func main() { dnode.WithLensRuntime(dnode.Wazero), // pure-Go WASM runtime ) if err != nil { - log.Fatalf("node.New: %v", err) + log.Fatalf("dnode.New: %v", err) } + defer func() { _ = n.Close(ctx) }() if err := n.Start(ctx); err != nil { log.Fatalf("node.Start: %v", err) } - // --- 2) Ensure KV schema (idempotent) --- + // Ensure KV schema (idempotent) if err := ensureKV(ctx, n); err != nil { log.Fatalf("ensure KV schema: %v", err) } - // --- 3) Execute the user’s GraphQL directly in-process --- - var vars map[string]any - if len(rawVars) > 0 { - if err := json.Unmarshal(rawVars, &vars); err != nil { - log.Fatalf("parse -vars: %v", err) - } - } - + // Setup timeout handler reqCtx, cancel := context.WithTimeout(ctx, *reqTO) defer cancel() + // Execute GraphQL query directly in-process res := n.DB.ExecRequest(reqCtx, q, dclient.WithVariables(vars)) if len(res.GQL.Errors) > 0 { enc, _ := json.MarshalIndent(res.GQL.Errors, "", " ") @@ -164,7 +164,7 @@ func main() { os.Exit(1) } - // Output JSON (pretty or compact). + // Output JSON (pretty or compact) if *pretty { out, _ := json.MarshalIndent(map[string]any{"data": res.GQL.Data}, "", " ") fmt.Println(string(out)) From 544347ac557f23e73279b19fdfaca2542deddae9 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Sat, 6 Sep 2025 13:23:47 -0500 Subject: [PATCH 3/9] trying to suppress all the info/debug output (from defra) --- defra-kv.go | 144 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 115 insertions(+), 29 deletions(-) diff --git a/defra-kv.go b/defra-kv.go index a72766d..eb4ec02 100644 --- a/defra-kv.go +++ b/defra-kv.go @@ -17,6 +17,7 @@ import ( dclient "github.com/sourcenetwork/defradb/client" dnode "github.com/sourcenetwork/defradb/node" + "github.com/rs/zerolog" ) func defaultRootdir() string { @@ -79,18 +80,76 @@ func ensureKV(ctx context.Context, n *dnode.Node) error { return nil } +type fdSilencer struct { + muted bool + devnull *os.File + origStdout *os.File + origStderr *os.File + origLogWriter io.Writer +} + +func (s *fdSilencer) Mute() { + if s.muted { + return + } + dn, err := os.OpenFile(os.DevNull, os.O_WRONLY, 0) + if err != nil { + return + } + s.devnull = dn + s.origStdout = os.Stdout + s.origStderr = os.Stderr + s.origLogWriter = log.Writer() + + // redirect global stdio and stdlib logger + os.Stdout = dn + os.Stderr = dn + log.SetOutput(dn) + + s.muted = true +} + +func (s *fdSilencer) PrintlnOut(line string) { + if s != nil && s.origStdout != nil { + _, _ = s.origStdout.Write([]byte(line)) + _, _ = s.origStdout.Write([]byte("\n")) + return + } + _, _ = os.Stdout.Write([]byte(line + "\n")) +} + +func (s *fdSilencer) PrintlnErr(line string) { + if s != nil && s.origStderr != nil { + _, _ = s.origStderr.Write([]byte(line)) + _, _ = s.origStderr.Write([]byte("\n")) + return + } + _, _ = os.Stderr.Write([]byte(line + "\n")) +} + +func die(s *fdSilencer, format string, a ...any) { + msg := fmt.Sprintf(format, a...) + if s != nil { + s.PrintlnErr(msg) + } else { + fmt.Fprintln(os.Stderr, msg) + } + os.Exit(1) +} + func main() { - // Define Custom FlagSet + // Flags fs := flag.NewFlagSet("defra-kv", flag.ExitOnError) - rootdir := fs.String("rootdir", defaultRootdir(), "data/config directory") - secret := fs.String("keyring-secret", "", "keyring secret (sets DEFRA_KEYRING_SECRET)") + rootdir := fs.String("rootdir", defaultRootdir(), "Data/config directory") + secret := fs.String("keyring-secret", "", "Keyring secret (sets DEFRA_KEYRING_SECRET)") query := fs.String("query", "", "GraphQL query/mutation") varsStr := fs.String("vars", "", "JSON variables") - pretty := fs.Bool("pretty", true, "pretty-print JSON") - reqTO := fs.Duration("timeout", 10*time.Second, "per-request timeout") + pretty := fs.Bool("pretty", true, "Pretty-print JSON output") + reqTO := fs.Duration("timeout", 10*time.Second, "Request timeout") + devMode := fs.Bool("dev", false, "enable development mode and verbose logging") _ = fs.Parse(os.Args[1:]) - // Process keyring secret + // Keyring secret (first run convenience) if *secret != "" { _ = os.Setenv("DEFRA_KEYRING_SECRET", *secret) } @@ -98,7 +157,7 @@ func main() { _ = os.Setenv("DEFRA_KEYRING_SECRET", "dev-dev-dev") } - // Read query (flag or stdin). + // Read query (flag or stdin) q := strings.TrimSpace(*query) if q == "" { b, err := io.ReadAll(os.Stdin) @@ -108,26 +167,40 @@ func main() { q = strings.TrimSpace(string(b)) } if q == "" { - fmt.Fprintln(os.Stderr, "no query provided; pass -query or pipe to stdin") + fmt.Fprintln(os.Stderr, "no query provided; pass --query or pipe to stdin") os.Exit(2) } - // Parse user-provided variables (if any) + // Variables (optional) var vars map[string]any if v := strings.TrimSpace(*varsStr); v != "" { - var rawVars json.RawMessage = json.RawMessage(v) - - if len(rawVars) > 0 { - if err := json.Unmarshal(rawVars, &vars); err != nil { - log.Fatalf("parse -vars: %v", err) - } + if err := json.Unmarshal([]byte(v), &vars); err != nil { + log.Fatalf("parse -vars: %v", err) } } - // Initialize context and signals + // Context + signals ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() + // Configure logging based on dev mode + var sil fdSilencer + if !*devMode { + // Environment-driven loggers used by Defra & deps + _ = os.Setenv("DEFRA_LOG_LEVEL", "error") + _ = os.Setenv("CORELOG_LEVEL", "error") // if corelog is present + _ = os.Setenv("GOLOG_LOG_LEVEL", "error") + + // zerolog global level + zerolog.SetGlobalLevel(zerolog.Disabled) + + // mute stdio + sil.Mute() + } else { + // allow all logs through + zerolog.SetGlobalLevel(zerolog.InfoLevel) + } + // Create and start the node (embedded, persistent Badger) n, err := dnode.New( ctx, @@ -137,39 +210,52 @@ func main() { dnode.WithStoreType(dnode.BadgerStore), dnode.WithStorePath(resolveRootdir(*rootdir)), // data dir dnode.WithLensRuntime(dnode.Wazero), // pure-Go WASM runtime + dnode.WithEnableDevelopment(*devMode), // toggle dev features/logging ) if err != nil { - log.Fatalf("dnode.New: %v", err) + die(&sil, "dnode.New: %v", err) } + defer func() { + _ = n.Close(ctx) + }() - defer func() { _ = n.Close(ctx) }() if err := n.Start(ctx); err != nil { - log.Fatalf("node.Start: %v", err) + die(&sil, "n.Start: %v", err) } - // Ensure KV schema (idempotent) if err := ensureKV(ctx, n); err != nil { - log.Fatalf("ensure KV schema: %v", err) + die(&sil, "ensure KV schema: %v", err) } - // Setup timeout handler reqCtx, cancel := context.WithTimeout(ctx, *reqTO) defer cancel() - // Execute GraphQL query directly in-process res := n.DB.ExecRequest(reqCtx, q, dclient.WithVariables(vars)) + + // Close the node explicitly + _ = n.Close(ctx) + + // Output GraphQL errors as reported (if any) if len(res.GQL.Errors) > 0 { enc, _ := json.MarshalIndent(res.GQL.Errors, "", " ") - fmt.Fprintln(os.Stderr, string(enc)) + if !*devMode { + sil.PrintlnErr(string(enc)) + } else { + fmt.Fprintln(os.Stderr, string(enc)) + } os.Exit(1) } - // Output JSON (pretty or compact) + // Output JSON (with pretty-printing if specified) + var outBytes []byte if *pretty { - out, _ := json.MarshalIndent(map[string]any{"data": res.GQL.Data}, "", " ") - fmt.Println(string(out)) + outBytes, _ = json.MarshalIndent(map[string]any{"data": res.GQL.Data}, "", " ") + } else { + outBytes, _ = json.Marshal(map[string]any{"data": res.GQL.Data}) + } + if !*devMode { + sil.PrintlnOut(string(outBytes)) } else { - out, _ := json.Marshal(map[string]any{"data": res.GQL.Data}) - fmt.Println(string(out)) + fmt.Println(string(outBytes)) } } From 12e29d17f19ad4ec606e2d71e2b568d0f014736e Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Sat, 6 Sep 2025 14:55:40 -0500 Subject: [PATCH 4/9] minor tweaks --- defra-kv.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/defra-kv.go b/defra-kv.go index eb4ec02..46a6438 100644 --- a/defra-kv.go +++ b/defra-kv.go @@ -44,7 +44,7 @@ func resolveRootdir(p string) string { } } if err := os.MkdirAll(p, 0o755); err != nil { - log.Fatalf("create rootdir: %v", err) + log.Fatalf("create dataConfigDir: %v", err) } return p } @@ -140,13 +140,13 @@ func die(s *fdSilencer, format string, a ...any) { func main() { // Flags fs := flag.NewFlagSet("defra-kv", flag.ExitOnError) - rootdir := fs.String("rootdir", defaultRootdir(), "Data/config directory") - secret := fs.String("keyring-secret", "", "Keyring secret (sets DEFRA_KEYRING_SECRET)") - query := fs.String("query", "", "GraphQL query/mutation") + dataConfigDir := fs.String("dir", defaultRootdir(), "Data/config directory") + secret := fs.String("keyring-secret", "", "Keyring secret (sets DEFRA_KEYRING_SECRET)") + query := fs.String("query", "", "GraphQL query/mutation") varsStr := fs.String("vars", "", "JSON variables") - pretty := fs.Bool("pretty", true, "Pretty-print JSON output") - reqTO := fs.Duration("timeout", 10*time.Second, "Request timeout") - devMode := fs.Bool("dev", false, "enable development mode and verbose logging") + pretty := fs.Bool("pretty", true, "Pretty-print JSON output") + reqTO := fs.Duration("timeout", 10*time.Second, "Request timeout") + devMode := fs.Bool("dev", false, "Enable DefraDB development mode and verbose logging") _ = fs.Parse(os.Args[1:]) // Keyring secret (first run convenience) @@ -167,7 +167,7 @@ func main() { q = strings.TrimSpace(string(b)) } if q == "" { - fmt.Fprintln(os.Stderr, "no query provided; pass --query or pipe to stdin") + fmt.Fprintln(os.Stderr, "no query provided; pass -query or pipe to stdin") os.Exit(2) } @@ -208,7 +208,7 @@ func main() { dnode.WithDisableP2P(true), // local only dnode.WithBadgerInMemory(false), // persistent dnode.WithStoreType(dnode.BadgerStore), - dnode.WithStorePath(resolveRootdir(*rootdir)), // data dir + dnode.WithStorePath(resolveRootdir(*dataConfigDir)), // data dir dnode.WithLensRuntime(dnode.Wazero), // pure-Go WASM runtime dnode.WithEnableDevelopment(*devMode), // toggle dev features/logging ) From d6489c431fa42a986958ed4b126c1dac337f92d3 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Sat, 6 Sep 2025 18:01:59 -0500 Subject: [PATCH 5/9] minor tweak --- defra-kv.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/defra-kv.go b/defra-kv.go index 46a6438..c6c480b 100644 --- a/defra-kv.go +++ b/defra-kv.go @@ -215,9 +215,6 @@ func main() { if err != nil { die(&sil, "dnode.New: %v", err) } - defer func() { - _ = n.Close(ctx) - }() if err := n.Start(ctx); err != nil { die(&sil, "n.Start: %v", err) From b122ba2c64fd387f3fa75af2b457986b2b751d58 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Sun, 7 Sep 2025 15:47:59 -0500 Subject: [PATCH 6/9] tweaking data schema for KV --- defra-kv.go | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/defra-kv.go b/defra-kv.go index c6c480b..cda640d 100644 --- a/defra-kv.go +++ b/defra-kv.go @@ -52,9 +52,9 @@ func resolveRootdir(p string) string { // Single JSON-based KV schema with indexes where useful. const kvSchema = ` type KV { - key: String @index - value: JSON - updatedAt: DateTime @index + key: String @index(unique: true) + value: JSON + updatedAt: DateTime @index } ` @@ -109,6 +109,25 @@ func (s *fdSilencer) Mute() { s.muted = true } +func (s *fdSilencer) Restore() { + if !s.muted { + return + } + if s.origLogWriter != nil { + log.SetOutput(s.origLogWriter) + } + if s.origStdout != nil { + os.Stdout = s.origStdout + } + if s.origStderr != nil { + os.Stderr = s.origStderr + } + if s.devnull != nil { + _ = s.devnull.Close() + } + s.muted = false +} + func (s *fdSilencer) PrintlnOut(line string) { if s != nil && s.origStdout != nil { _, _ = s.origStdout.Write([]byte(line)) From 38dfc0ded6951dda48ec20543eb98a06c433e51c Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Sun, 7 Sep 2025 20:57:02 -0500 Subject: [PATCH 7/9] adding KV operations to CLI Added `-has`, `-get`, `-set`, and `-remove` CLI commands --- defra-kv.go | 303 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 224 insertions(+), 79 deletions(-) diff --git a/defra-kv.go b/defra-kv.go index cda640d..24e11a4 100644 --- a/defra-kv.go +++ b/defra-kv.go @@ -11,37 +11,37 @@ import ( "os" "os/signal" "path/filepath" + "regexp" "strings" "syscall" "time" dclient "github.com/sourcenetwork/defradb/client" - dnode "github.com/sourcenetwork/defradb/node" + dnode "github.com/sourcenetwork/defradb/node" "github.com/rs/zerolog" ) -func defaultRootdir() string { +// Single JSON-based KV schema with indexes where useful. +const kvSchema = ` +type KV { + key: String @index(unique: true) + value: JSON + updatedAt: DateTime @index +} +` + +var gqlNameRE = regexp.MustCompile(`^[_A-Za-z][_0-9A-Za-z]*$`) + +func defaultDataConfigDir() string { if cwd, err := os.Getwd(); err == nil { return filepath.Join(cwd, ".defra-kv") } return ".defra-kv" } -func expandHome(p string) string { - if strings.HasPrefix(p, "~/") { - if h, err := os.UserHomeDir(); err == nil { - return filepath.Join(h, p[2:]) - } - } - return p -} - func resolveRootdir(p string) string { - p = expandHome(p) - if !filepath.IsAbs(p) { - if abs, err := filepath.Abs(p); err == nil { - p = abs - } + if p == "" || p == "." { + p = defaultDataConfigDir() } if err := os.MkdirAll(p, 0o755); err != nil { log.Fatalf("create dataConfigDir: %v", err) @@ -49,15 +49,6 @@ func resolveRootdir(p string) string { return p } -// Single JSON-based KV schema with indexes where useful. -const kvSchema = ` -type KV { - key: String @index(unique: true) - value: JSON - updatedAt: DateTime @index -} -` - func kvExists(ctx context.Context, n *dnode.Node) bool { res := n.DB.ExecRequest(ctx, `query { __type(name: "KV") { name } }`) if len(res.GQL.Errors) > 0 { @@ -109,32 +100,13 @@ func (s *fdSilencer) Mute() { s.muted = true } -func (s *fdSilencer) Restore() { - if !s.muted { - return - } - if s.origLogWriter != nil { - log.SetOutput(s.origLogWriter) - } - if s.origStdout != nil { - os.Stdout = s.origStdout - } - if s.origStderr != nil { - os.Stderr = s.origStderr - } - if s.devnull != nil { - _ = s.devnull.Close() - } - s.muted = false -} - func (s *fdSilencer) PrintlnOut(line string) { if s != nil && s.origStdout != nil { _, _ = s.origStdout.Write([]byte(line)) _, _ = s.origStdout.Write([]byte("\n")) return } - _, _ = os.Stdout.Write([]byte(line + "\n")) + fmt.Println(line) } func (s *fdSilencer) PrintlnErr(line string) { @@ -143,7 +115,7 @@ func (s *fdSilencer) PrintlnErr(line string) { _, _ = s.origStderr.Write([]byte("\n")) return } - _, _ = os.Stderr.Write([]byte(line + "\n")) + fmt.Fprintln(os.Stderr, line) } func die(s *fdSilencer, format string, a ...any) { @@ -156,18 +128,73 @@ func die(s *fdSilencer, format string, a ...any) { os.Exit(1) } +// Convert a Go value (from JSON) into a GraphQL input literal string. +// Supports: nil, bool, finite numbers, strings, arrays, and objects with GraphQL-Name keys. +func toGraphQLLiteral(v any) (string, error) { + if v == nil { + return "null", nil + } + switch t := v.(type) { + case string: + b, _ := json.Marshal(t) + return string(b), nil + case bool: + if t { + return "true", nil + } + return "false", nil + case float64: + // JSON numbers decode to float64 and are finite by spec. + return fmt.Sprintf("%v", t), nil + case []any: + parts := make([]string, 0, len(t)) + for _, e := range t { + lit, err := toGraphQLLiteral(e) + if err != nil { + return "", err + } + parts = append(parts, lit) + } + return "[" + strings.Join(parts, ", ") + "]", nil + case map[string]any: + parts := make([]string, 0, len(t)) + for k, val := range t { + if !gqlNameRE.MatchString(k) { + return "", fmt.Errorf("invalid GraphQL key: %q", k) + } + lit, err := toGraphQLLiteral(val) + if err != nil { + return "", err + } + parts = append(parts, k+": "+lit) + } + return "{ " + strings.Join(parts, ", ") + " }", nil + default: + return "", fmt.Errorf("unsupported type in literal: %T", v) + } +} + func main() { // Flags fs := flag.NewFlagSet("defra-kv", flag.ExitOnError) - dataConfigDir := fs.String("dir", defaultRootdir(), "Data/config directory") - secret := fs.String("keyring-secret", "", "Keyring secret (sets DEFRA_KEYRING_SECRET)") - query := fs.String("query", "", "GraphQL query/mutation") + hasKey := fs.String("has", "", "Check key existence") + getKey := fs.String("get", "", "Get value by key") + setKey := fs.String("set", "", "Set/update value by key (value via stdin)") + removeKey := fs.String("remove", "", "Remove key/value") varsStr := fs.String("vars", "", "JSON variables") + query := fs.String("query", "", "Raw GraphQL query/mutation") pretty := fs.Bool("pretty", true, "Pretty-print JSON output") reqTO := fs.Duration("timeout", 10*time.Second, "Request timeout") + dataConfigDir := fs.String("dir", defaultDataConfigDir(), "Data/config directory") + secret := fs.String("keyring-secret", "", "Keyring secret (sets DEFRA_KEYRING_SECRET)") devMode := fs.Bool("dev", false, "Enable DefraDB development mode and verbose logging") _ = fs.Parse(os.Args[1:]) + // Determine mode: raw (-query) takes precedence over KV actions + hasAction := ( + strings.TrimSpace(*query) == "" && + (*setKey != "" || *getKey != "" || *hasKey != "" || *removeKey != "")) + // Keyring secret (first run convenience) if *secret != "" { _ = os.Setenv("DEFRA_KEYRING_SECRET", *secret) @@ -176,25 +203,30 @@ func main() { _ = os.Setenv("DEFRA_KEYRING_SECRET", "dev-dev-dev") } - // Read query (flag or stdin) - q := strings.TrimSpace(*query) - if q == "" { - b, err := io.ReadAll(os.Stdin) - if err != nil { - log.Fatalf("read stdin: %v", err) + // Read query (flag or stdin) for raw mode only + var q string + if !hasAction { + q = strings.TrimSpace(*query) + if q == "" { + b, err := io.ReadAll(os.Stdin) + if err != nil { + log.Fatalf("read stdin: %v", err) + } + q = strings.TrimSpace(string(b)) + } + if q == "" { + fmt.Fprintln(os.Stderr, "no query provided; pass -query or pipe to stdin") + os.Exit(2) } - q = strings.TrimSpace(string(b)) - } - if q == "" { - fmt.Fprintln(os.Stderr, "no query provided; pass -query or pipe to stdin") - os.Exit(2) } - // Variables (optional) + // Variables (optional) for raw mode var vars map[string]any - if v := strings.TrimSpace(*varsStr); v != "" { - if err := json.Unmarshal([]byte(v), &vars); err != nil { - log.Fatalf("parse -vars: %v", err) + if !hasAction { + if v := strings.TrimSpace(*varsStr); v != "" { + if err := json.Unmarshal([]byte(v), &vars); err != nil { + log.Fatalf("parse -vars: %v", err) + } } } @@ -223,13 +255,13 @@ func main() { // Create and start the node (embedded, persistent Badger) n, err := dnode.New( ctx, - dnode.WithDisableAPI(true), // no HTTP server - dnode.WithDisableP2P(true), // local only - dnode.WithBadgerInMemory(false), // persistent + dnode.WithDisableAPI(true), // no HTTP server + dnode.WithDisableP2P(true), // local only + dnode.WithBadgerInMemory(false), // persistent dnode.WithStoreType(dnode.BadgerStore), dnode.WithStorePath(resolveRootdir(*dataConfigDir)), // data dir - dnode.WithLensRuntime(dnode.Wazero), // pure-Go WASM runtime - dnode.WithEnableDevelopment(*devMode), // toggle dev features/logging + dnode.WithLensRuntime(dnode.Wazero), // pure-Go WASM runtime + dnode.WithEnableDevelopment(*devMode), // toggle dev features/logging ) if err != nil { die(&sil, "dnode.New: %v", err) @@ -243,6 +275,68 @@ func main() { die(&sil, "ensure KV schema: %v", err) } + // Build canned KV queries if in action mode + if hasAction { + var b []byte + var err error + vars = map[string]any{} + if *setKey != "" { + // Read value JSON from stdin + b, err = io.ReadAll(os.Stdin) + if err != nil { + die(&sil, "read stdin value: %v", err) + } + valStr := strings.TrimSpace(string(b)) + if valStr == "" { + die(&sil, "no value on stdin for -set") + } + var val any + if err := json.Unmarshal([]byte(valStr), &val); err != nil { + die(&sil, "invalid JSON on stdin: %v", err) + } + now := time.Now().UTC().Format(time.RFC3339Nano) + vars["now"] = now + vars["key"] = *setKey + + // Note: this style (passing in `value` as an external variable) + // does not currently work, due to a bug in defra + // + // vars["value"] = val + // q = `mutation setKV($key:String!,$value:JSON!,$now:DateTime!) { + // upsert_KV( + // filter: { key: { _eq: $key } } + // create: { key: $key, value: $value, updatedAt: $now } + // update: { value: $value, updatedAt: $now } + // ) { _docID } + // }` + + lit, err := toGraphQLLiteral(val) + if err != nil { + die(&sil, "value cannot be inlined: %v", err) + } + q = fmt.Sprintf( + `mutation setKV($key:String!,$now:DateTime!) { + upsert_KV( + filter: { key: { _eq: $key } } + create: { key: $key, value: %s, updatedAt: $now } + update: { value: %s, updatedAt: $now } + ) { _docID } + }`, + lit, + lit, + ) + } else if *getKey != "" { + vars["key"] = *getKey + q = "query getKV($key:String!) { KV(filter:{ key:{ _eq:$key } }) { key value } }" + } else if *hasKey != "" { + vars["key"] = *hasKey + q = "query hasKV($key:String!) { KV(filter:{ key:{ _eq:$key } }) { _docID } }" + } else if *removeKey != "" { + vars["key"] = *removeKey + q = "mutation removeKV($key:String!) { delete_KV(filter:{ key:{ _eq:$key } }) { _docID } }" + } + } + reqCtx, cancel := context.WithTimeout(ctx, *reqTO) defer cancel() @@ -262,16 +356,67 @@ func main() { os.Exit(1) } - // Output JSON (with pretty-printing if specified) - var outBytes []byte - if *pretty { - outBytes, _ = json.MarshalIndent(map[string]any{"data": res.GQL.Data}, "", " ") - } else { - outBytes, _ = json.Marshal(map[string]any{"data": res.GQL.Data}) - } - if !*devMode { - sil.PrintlnOut(string(outBytes)) + // If using action mode, handle outputs/exit codes and return early + if hasAction { + m, _ := res.GQL.Data.(map[string]any) + + // set + if *setKey != "" { + rows, _ := m["upsert_KV"].([]map[string]any) + if len(rows) > 0 { + os.Exit(0) + } + os.Exit(3) + } + // has + if *hasKey != "" { + rows, _ := m["KV"].([]map[string]any) + if len(rows) > 0 { + os.Exit(0) + } + os.Exit(3) + } + // del + if *removeKey != "" { + rows, _ := m["delete_KV"].([]map[string]any) + if len(rows) > 0 { + os.Exit(0) + } + os.Exit(3) + } + // get + if *getKey != "" { + rows, _ := m["KV"].([]map[string]any) + + if len(rows) == 0 { + os.Exit(3) + } + doc := rows[0] + var outBytes []byte + if *pretty { + outBytes, _ = json.MarshalIndent(doc, "", " ") + } else { + outBytes, _ = json.Marshal(doc) + } + if !*devMode { + sil.PrintlnOut(string(outBytes)) + } else { + fmt.Println(string(outBytes)) + } + os.Exit(0) + } } else { - fmt.Println(string(outBytes)) + // Output JSON (with pretty-printing if specified) for raw mode + var outBytes []byte + if *pretty { + outBytes, _ = json.MarshalIndent(map[string]any{"data": res.GQL.Data}, "", " ") + } else { + outBytes, _ = json.Marshal(map[string]any{"data": res.GQL.Data}) + } + if !*devMode { + sil.PrintlnOut(string(outBytes)) + } else { + fmt.Println(string(outBytes)) + } } } From 7a7151d028dbea86c2512af76f8ec4f11f5e497c Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Tue, 9 Sep 2025 12:00:51 -0500 Subject: [PATCH 8/9] Passing JSON objects directly in external variables Per fix in #3992, now passing JSON objects as variables works, instead of needing to serialize and inline them into the query. Yay! --- defra-kv.go | 83 ++++++----------------------------------------------- 1 file changed, 9 insertions(+), 74 deletions(-) diff --git a/defra-kv.go b/defra-kv.go index 24e11a4..a0185f0 100644 --- a/defra-kv.go +++ b/defra-kv.go @@ -128,52 +128,6 @@ func die(s *fdSilencer, format string, a ...any) { os.Exit(1) } -// Convert a Go value (from JSON) into a GraphQL input literal string. -// Supports: nil, bool, finite numbers, strings, arrays, and objects with GraphQL-Name keys. -func toGraphQLLiteral(v any) (string, error) { - if v == nil { - return "null", nil - } - switch t := v.(type) { - case string: - b, _ := json.Marshal(t) - return string(b), nil - case bool: - if t { - return "true", nil - } - return "false", nil - case float64: - // JSON numbers decode to float64 and are finite by spec. - return fmt.Sprintf("%v", t), nil - case []any: - parts := make([]string, 0, len(t)) - for _, e := range t { - lit, err := toGraphQLLiteral(e) - if err != nil { - return "", err - } - parts = append(parts, lit) - } - return "[" + strings.Join(parts, ", ") + "]", nil - case map[string]any: - parts := make([]string, 0, len(t)) - for k, val := range t { - if !gqlNameRE.MatchString(k) { - return "", fmt.Errorf("invalid GraphQL key: %q", k) - } - lit, err := toGraphQLLiteral(val) - if err != nil { - return "", err - } - parts = append(parts, k+": "+lit) - } - return "{ " + strings.Join(parts, ", ") + " }", nil - default: - return "", fmt.Errorf("unsupported type in literal: %T", v) - } -} - func main() { // Flags fs := flag.NewFlagSet("defra-kv", flag.ExitOnError) @@ -297,34 +251,15 @@ func main() { now := time.Now().UTC().Format(time.RFC3339Nano) vars["now"] = now vars["key"] = *setKey - - // Note: this style (passing in `value` as an external variable) - // does not currently work, due to a bug in defra - // - // vars["value"] = val - // q = `mutation setKV($key:String!,$value:JSON!,$now:DateTime!) { - // upsert_KV( - // filter: { key: { _eq: $key } } - // create: { key: $key, value: $value, updatedAt: $now } - // update: { value: $value, updatedAt: $now } - // ) { _docID } - // }` - - lit, err := toGraphQLLiteral(val) - if err != nil { - die(&sil, "value cannot be inlined: %v", err) - } - q = fmt.Sprintf( - `mutation setKV($key:String!,$now:DateTime!) { - upsert_KV( - filter: { key: { _eq: $key } } - create: { key: $key, value: %s, updatedAt: $now } - update: { value: %s, updatedAt: $now } - ) { _docID } - }`, - lit, - lit, - ) + vars["value"] = val + + q = `mutation setKV($key:String!,$value:JSON!,$now:DateTime!) { + upsert_KV( + filter: { key: { _eq: $key } } + create: { key: $key, value: $value, updatedAt: $now } + update: { value: $value, updatedAt: $now } + ) { _docID } + }` } else if *getKey != "" { vars["key"] = *getKey q = "query getKV($key:String!) { KV(filter:{ key:{ _eq:$key } }) { key value } }" From 82c8ebbf1d2078cd73eb4dc7a5765a0852fb9bd2 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 10 Sep 2025 12:06:06 -0500 Subject: [PATCH 9/9] updates/rename rename `defra-kv.go` to `defra-kv/main.go` removed unnecessary/ineffective bits of stdio muting/override, fixed log_level env name --- defra-kv.go => defra-kv/main.go | 58 +++++++++------------------------ 1 file changed, 15 insertions(+), 43 deletions(-) rename defra-kv.go => defra-kv/main.go (87%) diff --git a/defra-kv.go b/defra-kv/main.go similarity index 87% rename from defra-kv.go rename to defra-kv/main.go index a0185f0..c4f08a4 100644 --- a/defra-kv.go +++ b/defra-kv/main.go @@ -18,7 +18,6 @@ import ( dclient "github.com/sourcenetwork/defradb/client" dnode "github.com/sourcenetwork/defradb/node" - "github.com/rs/zerolog" ) // Single JSON-based KV schema with indexes where useful. @@ -71,15 +70,13 @@ func ensureKV(ctx context.Context, n *dnode.Node) error { return nil } -type fdSilencer struct { +type errSilencer struct { muted bool devnull *os.File - origStdout *os.File origStderr *os.File - origLogWriter io.Writer } -func (s *fdSilencer) Mute() { +func (s *errSilencer) Mute() { if s.muted { return } @@ -88,28 +85,15 @@ func (s *fdSilencer) Mute() { return } s.devnull = dn - s.origStdout = os.Stdout s.origStderr = os.Stderr - s.origLogWriter = log.Writer() - // redirect global stdio and stdlib logger - os.Stdout = dn + // redirect global stderr os.Stderr = dn - log.SetOutput(dn) s.muted = true } -func (s *fdSilencer) PrintlnOut(line string) { - if s != nil && s.origStdout != nil { - _, _ = s.origStdout.Write([]byte(line)) - _, _ = s.origStdout.Write([]byte("\n")) - return - } - fmt.Println(line) -} - -func (s *fdSilencer) PrintlnErr(line string) { +func (s *errSilencer) PrintlnErr(line string) { if s != nil && s.origStderr != nil { _, _ = s.origStderr.Write([]byte(line)) _, _ = s.origStderr.Write([]byte("\n")) @@ -118,7 +102,7 @@ func (s *fdSilencer) PrintlnErr(line string) { fmt.Fprintln(os.Stderr, line) } -func die(s *fdSilencer, format string, a ...any) { +func die(s *errSilencer, format string, a ...any) { msg := fmt.Sprintf(format, a...) if s != nil { s.PrintlnErr(msg) @@ -189,21 +173,12 @@ func main() { defer stop() // Configure logging based on dev mode - var sil fdSilencer + var sil errSilencer if !*devMode { - // Environment-driven loggers used by Defra & deps - _ = os.Setenv("DEFRA_LOG_LEVEL", "error") - _ = os.Setenv("CORELOG_LEVEL", "error") // if corelog is present - _ = os.Setenv("GOLOG_LOG_LEVEL", "error") - - // zerolog global level - zerolog.SetGlobalLevel(zerolog.Disabled) - - // mute stdio + _ = os.Setenv("LOG_LEVEL", "error") sil.Mute() } else { - // allow all logs through - zerolog.SetGlobalLevel(zerolog.InfoLevel) + _ = os.Setenv("LOG_LEVEL", "info") } // Create and start the node (embedded, persistent Badger) @@ -234,6 +209,7 @@ func main() { var b []byte var err error vars = map[string]any{} + if *setKey != "" { // Read value JSON from stdin b, err = io.ReadAll(os.Stdin) @@ -327,17 +303,15 @@ func main() { os.Exit(3) } doc := rows[0] + var outBytes []byte if *pretty { outBytes, _ = json.MarshalIndent(doc, "", " ") } else { outBytes, _ = json.Marshal(doc) } - if !*devMode { - sil.PrintlnOut(string(outBytes)) - } else { - fmt.Println(string(outBytes)) - } + + fmt.Println(string(outBytes)) os.Exit(0) } } else { @@ -348,10 +322,8 @@ func main() { } else { outBytes, _ = json.Marshal(map[string]any{"data": res.GQL.Data}) } - if !*devMode { - sil.PrintlnOut(string(outBytes)) - } else { - fmt.Println(string(outBytes)) - } + + fmt.Println(string(outBytes)) + os.Exit(0) } }