diff --git a/reconciler/testdata/batching.txtar b/reconciler/testdata/batching.txtar index 10b1fad..4dc5835 100644 --- a/reconciler/testdata/batching.txtar +++ b/reconciler/testdata/batching.txtar @@ -7,10 +7,10 @@ start-reconciler with-batchops # From here this is the same as incremental.txtar. # Step 1: Insert non-faulty objects -db insert test-objects obj1.yaml -db insert test-objects obj2.yaml -db insert test-objects obj3.yaml -db cmp test-objects step1+3.table +db/insert test-objects obj1.yaml +db/insert test-objects obj2.yaml +db/insert test-objects obj3.yaml +db/cmp test-objects step1+3.table expect-ops update(1) update(2) update(3) # Reconciler should be running and reporting health @@ -18,27 +18,27 @@ health 'job-reconcile.*level=OK.*message=OK, 3 object' # Step 2: Update object '1' to be faulty and check that it fails and is being # retried. -db insert test-objects obj1_faulty.yaml +db/insert test-objects obj1_faulty.yaml expect-ops 'update(1) fail' 'update(1) fail' -db cmp test-objects step2.table +db/cmp test-objects step2.table health 'job-reconcile.*level=Degraded.*1 error' # Step 3: Set object '1' back to healthy state -db insert test-objects obj1.yaml +db/insert test-objects obj1.yaml expect-ops 'update(1)' -db cmp test-objects step1+3.table +db/cmp test-objects step1+3.table health 'job-reconcile.*level=OK' # Step 4: Delete '1' and '2' -db delete test-objects obj1.yaml -db delete test-objects obj2.yaml -db cmp test-objects step4.table +db/delete test-objects obj1.yaml +db/delete test-objects obj2.yaml +db/cmp test-objects step4.table expect-ops 'delete(1)' 'delete(2)' # Step 5: Try to delete '3' with faulty target set-faulty true -db delete test-objects obj3.yaml -db cmp test-objects empty.table +db/delete test-objects obj3.yaml +db/cmp test-objects empty.table expect-ops 'delete(3) fail' health 'job-reconcile.*level=Degraded.*1 error' diff --git a/reconciler/testdata/incremental.txtar b/reconciler/testdata/incremental.txtar index bb5e540..f48f4b6 100644 --- a/reconciler/testdata/incremental.txtar +++ b/reconciler/testdata/incremental.txtar @@ -5,10 +5,10 @@ hive start start-reconciler # Step 1: Insert non-faulty objects -db insert test-objects obj1.yaml -db insert test-objects obj2.yaml -db insert test-objects obj3.yaml -db cmp test-objects step1+3.table +db/insert test-objects obj1.yaml +db/insert test-objects obj2.yaml +db/insert test-objects obj3.yaml +db/cmp test-objects step1+3.table expect-ops update(1) update(2) update(3) # Reconciler should be running and reporting health @@ -16,28 +16,28 @@ health 'job-reconcile.*level=OK.*message=OK, 3 object' # Step 2: Update object '1' to be faulty and check that it fails and is being # retried. -db insert test-objects obj1_faulty.yaml -db cmp test-objects step2.table +db/insert test-objects obj1_faulty.yaml +db/cmp test-objects step2.table expect-ops 'update(1) fail' 'update(1) fail' health 'job-reconcile.*level=Degraded.*1 error' # Step 3: Set object '1' back to healthy state -db insert test-objects obj1.yaml -db show test-objects -db cmp test-objects step1+3.table +db/insert test-objects obj1.yaml +db/show test-objects +db/cmp test-objects step1+3.table expect-ops 'update(1)' health 'job-reconcile.*level=OK' # Step 4: Delete '1' and '2' -db delete test-objects obj1.yaml -db delete test-objects obj2.yaml -db cmp test-objects step4.table +db/delete test-objects obj1.yaml +db/delete test-objects obj2.yaml +db/cmp test-objects step4.table expect-ops 'delete(1)' 'delete(2)' # Step 5: Try to delete '3' with faulty target set-faulty true -db delete test-objects obj3.yaml -db cmp test-objects empty.table +db/delete test-objects obj3.yaml +db/cmp test-objects empty.table expect-ops 'delete(3) fail' health 'job-reconcile.*level=Degraded.*1 error' diff --git a/reconciler/testdata/pruning.txtar b/reconciler/testdata/pruning.txtar index 778aed6..431373d 100644 --- a/reconciler/testdata/pruning.txtar +++ b/reconciler/testdata/pruning.txtar @@ -2,10 +2,10 @@ hive start start-reconciler with-prune # Pruning without table being initialized does nothing. -db insert test-objects obj1.yaml +db/insert test-objects obj1.yaml expect-ops update(1) prune -db insert test-objects obj2.yaml +db/insert test-objects obj2.yaml expect-ops update(2) update(1) health 'job-reconcile.*level=OK' @@ -33,12 +33,12 @@ expvar grep 'prune_current_errors.test: 0' # Delete an object and check pruning happens without it -db delete test-objects obj1.yaml +db/delete test-objects obj1.yaml prune expect-ops 'prune(n=1)' delete(1) # Prune without objects -db delete test-objects obj2.yaml +db/delete test-objects obj2.yaml prune expect-ops prune(n=0) delete(2) prune(n=1) diff --git a/reconciler/testdata/refresh.txtar b/reconciler/testdata/refresh.txtar index 57b9041..bb2fc62 100644 --- a/reconciler/testdata/refresh.txtar +++ b/reconciler/testdata/refresh.txtar @@ -2,27 +2,27 @@ hive start start-reconciler with-refresh # Step 1: Add a test object. -db insert test-objects obj1.yaml +db/insert test-objects obj1.yaml expect-ops 'update(1)' -db cmp test-objects step1.table +db/cmp test-objects step1.table # Step 2: Set the object as updated in the past to force refresh -db insert test-objects obj1_old.yaml +db/insert test-objects obj1_old.yaml expect-ops 'update-refresh(1)' # Step 3: Refresh with faulty target, should see fail & retries set-faulty true -db insert test-objects obj1_old.yaml +db/insert test-objects obj1_old.yaml expect-ops 'update-refresh(1) fail' 'update-refresh(1) fail' -db cmp test-objects step3.table +db/cmp test-objects step3.table health health 'job-reconcile.*Degraded' # Step 4: Back to health set-faulty false -db insert test-objects obj1_old.yaml +db/insert test-objects obj1_old.yaml expect-ops 'update-refresh(1)' -db cmp test-objects step4.table +db/cmp test-objects step4.table health 'job-reconcile.*OK, 1 object' # ----- diff --git a/script.go b/script.go index 7915656..6e94b27 100644 --- a/script.go +++ b/script.go @@ -6,12 +6,10 @@ package statedb import ( "bytes" "encoding/json" - "errors" "flag" "fmt" "io" "iter" - "maps" "os" "regexp" "slices" @@ -25,116 +23,30 @@ import ( "gopkg.in/yaml.v3" ) -func underline(s string) string { - return "\033[4m" + s + "\033[0m" +func ScriptCommands(db *DB) hive.ScriptCmdsOut { + return hive.NewScriptCmds(map[string]script.Cmd{ + "db": DBCmd(db), + "db/show": ShowCmd(db), + "db/cmp": CompareCmd(db), + "db/insert": InsertCmd(db), + "db/delete": DeleteCmd(db), + "db/get": GetCmd(db), + "db/prefix": PrefixCmd(db), + "db/list": ListCmd(db), + "db/lowerbound": LowerBoundCmd(db), + "db/watch": WatchCmd(db), + "db/initialized": InitializedCmd(db), + }) } -func usageDetails(cmds map[string]script.Cmd, sortedNames []string) (out []string) { - out = strings.Split(`DESCRIPTION - The 'db' command allows inspecting and manipulating - StateDB tables. Here's an example to get you started: - - > db tables - Name Object count Zombie objects Indexes ... - example 2 0 id, x ... - - > db show example - Name X - one 1 - two 2 - - > db prefix -index=id example o - Name X - one 1 - - > db insert example three.yaml four.yaml - - > db delete example three.yaml - -COMMANDS -`, "\n") - - for _, name := range sortedNames { - cmd := cmds[name] - u := cmd.Usage() - cmdLine := underline(name) + " " + u.Args - // Do manual line wrapping to indent nicely when wrapped. - const wrap = 60 /* wrap mark in script */ - 8 /* indent */ - dots := strings.Repeat(".", len(name)) - if len(cmdLine) > wrap { - idx := strings.LastIndex(cmdLine[:wrap], " ") - out = append(out, cmdLine[:idx]) - rest := cmdLine[idx:] - for { - if len(rest) < wrap { - out = append(out, dots+rest) - break - } - idx := strings.LastIndex(rest[:wrap], " ") - out = append(out, dots+rest[:idx]) - rest = rest[idx:] - } - } else { - out = append(out, cmdLine) - } - out = append(out, "") - out = append(out, u.Detail...) - out = append(out, "") - } - - return out -} - -func ScriptCommands(db *DB) hive.ScriptCmdOut { - subCmds := map[string]script.Cmd{ - "tables": TablesCmd(db), - "show": ShowCmd(db), - "cmp": CompareCmd(db), - "insert": InsertCmd(db), - "delete": DeleteCmd(db), - "get": GetCmd(db), - "prefix": PrefixCmd(db), - "list": ListCmd(db), - "lowerbound": LowerBoundCmd(db), - "watch": WatchCmd(db), - "initialized": InitializedCmd(db), - } - subCmdsNames := slices.Sorted(maps.Keys(subCmds)) - subCmdsList := strings.Join(subCmdsNames, ", ") - return hive.NewScriptCmd( - "db", - script.Command( - script.CmdUsage{ - Summary: "Inspect and manipulate StateDB ('help db' for usage)", - Args: "cmd args...", - Detail: usageDetails(subCmds, subCmdsNames), - }, - func(s *script.State, args ...string) (script.WaitFunc, error) { - if len(args) < 1 { - return nil, fmt.Errorf("expected command (%s), see 'help db'", subCmdsList) - } - cmd, ok := subCmds[args[0]] - if !ok { - return nil, fmt.Errorf("command %q not found, expected one of: %s, see 'help db'", args[0], subCmdsList) - } - wf, err := cmd.Run(s, sortedArgs(args[1:])...) - if errors.Is(err, errUsage) { - s.Logf("usage: db %s %s\n", args[0], cmd.Usage().Args) - } - return wf, err - }, - ), - ) -} - -var errUsage = errors.New("bad arguments") - -func TablesCmd(db *DB) script.Cmd { +func DBCmd(db *DB) script.Cmd { return script.Command( script.CmdUsage{ - Summary: "List StateDB tables", + Summary: "Describe StateDB configuration", Detail: []string{ - "List each registered table.", + "The 'db' command describes the StateDB configuration, showing", + "all registered tables and brief summary of their state.", + "", "The following details are shown:", "- Name: The name of the table as given to 'NewTable'", "- Object count: Objects in the table", @@ -143,6 +55,23 @@ func TablesCmd(db *DB) script.Cmd { "- Initializers: Pending table initializers", "- Go type: The Go type, the T in Table[T]", "- Last WriteTxn: The current/last write against the table", + "", + "The individual tables can be manipulated and inspected with the", + "other commands. See 'help -v db/show' etc. for detailed help.", + "Here is some examples to get you statred:", + "", + "> db/show example", + "Name X", + "one 1", + "two 2", + "", + "> db/prefix -index=id example o", + "Name X", + "one 1", + "", + "> db/insert example three.yaml four.yaml", + "", + "> db/delete example three.yaml", }, }, func(s *script.State, args ...string) (script.WaitFunc, error) { @@ -161,11 +90,10 @@ func TablesCmd(db *DB) script.Cmd { ) } -func newCmdFlagSet() *flag.FlagSet { - return &flag.FlagSet{ - // Disable showing the normal usage. - Usage: func() {}, - } +func newCmdFlagSet(w io.Writer) *flag.FlagSet { + fs := flag.NewFlagSet("", flag.ContinueOnError) + fs.SetOutput(w) + return fs } func InitializedCmd(db *DB) script.Cmd { @@ -183,18 +111,18 @@ func InitializedCmd(db *DB) script.Cmd { }, }, func(s *script.State, args ...string) (script.WaitFunc, error) { - txn := db.ReadTxn() - allTbls := db.GetTables(txn) - tbls := allTbls - - flags := newCmdFlagSet() + flags := newCmdFlagSet(s.LogWriter()) timeout := flags.Duration("timeout", 5*time.Second, "Maximum amount of time to wait for the table contents to match") if err := flags.Parse(args); err != nil { - return nil, fmt.Errorf("%w: %s", errUsage, err) + return nil, fmt.Errorf("%w: %w", script.ErrUsage, err) } - timeoutChan := time.After(*timeout) args = flags.Args() + txn := db.ReadTxn() + timeoutChan := time.After(*timeout) + allTbls := db.GetTables(txn) + tbls := allTbls + if len(args) > 0 { // Specific tables requested, look them up. tbls = make([]TableMeta, 0, len(args)) @@ -254,12 +182,12 @@ func ShowCmd(db *DB) script.Cmd { }, }, func(s *script.State, args ...string) (script.WaitFunc, error) { - flags := newCmdFlagSet() + flags := newCmdFlagSet(s.LogWriter()) file := flags.String("o", "", "File to write to instead of stdout") columns := flags.String("columns", "", "Comma-separated list of columns to write") format := flags.String("format", "table", "Format to write in (table, yaml, json)") if err := flags.Parse(args); err != nil { - return nil, fmt.Errorf("%w: %s", errUsage, err) + return nil, fmt.Errorf("%w: %w", script.ErrUsage, err) } var cols []string @@ -269,7 +197,7 @@ func ShowCmd(db *DB) script.Cmd { args = flags.Args() if len(args) < 1 { - return nil, fmt.Errorf("%w: missing table name", errUsage) + return nil, fmt.Errorf("missing table name") } tableName := args[0] return func(*script.State) (stdout, stderr string, err error) { @@ -305,7 +233,7 @@ func CompareCmd(db *DB) script.Cmd { "The comparison is retried until a timeout (1s default).", "", "The file should be formatted in the same style as", - "the output from 'db show -format=table'. Indentation", + "the output from 'db/show -format=table'. Indentation", "does not matter as long as header is aligned with the data.", "", "Not all columns need to be specified. Remove the columns", @@ -315,13 +243,16 @@ func CompareCmd(db *DB) script.Cmd { }, }, func(s *script.State, args ...string) (script.WaitFunc, error) { - flags := newCmdFlagSet() + flags := newCmdFlagSet(s.LogWriter()) timeout := flags.Duration("timeout", time.Second, "Maximum amount of time to wait for the table contents to match") grep := flags.String("grep", "", "Grep the result rows and only compare matching ones") err := flags.Parse(args) + if err != nil { + return nil, fmt.Errorf("%w: %w", script.ErrUsage, err) + } args = flags.Args() - if err != nil || len(args) != 2 { - return nil, fmt.Errorf("%w: %s", errUsage, err) + if len(args) != 2 { + return nil, fmt.Errorf("expected table and filename") } var grepRe *regexp.Regexp @@ -463,7 +394,7 @@ func getTable(db *DB, tableName string) (*AnyTable, ReadTxn, error) { func insertOrDelete(insert bool, db *DB, s *script.State, args ...string) (script.WaitFunc, error) { if len(args) < 2 { - return nil, fmt.Errorf("%w: expected table and path(s)", errUsage) + return nil, fmt.Errorf("expected table and path(s)") } tbl, _, err := getTable(db, args[0]) @@ -564,14 +495,14 @@ func queryCmd(db *DB, query int, summary string, detail []string) script.Cmd { } func runQueryCmd(query int, db *DB, s *script.State, args []string) (script.WaitFunc, error) { - flags := newCmdFlagSet() + flags := newCmdFlagSet(s.LogWriter()) file := flags.String("o", "", "File to write results to instead of stdout") index := flags.String("index", "", "Index to query") format := flags.String("format", "table", "Format to write in (table, yaml, json)") columns := flags.String("columns", "", "Comma-separated list of columns to write") delete := flags.Bool("delete", false, "Delete all matching objects") if err := flags.Parse(args); err != nil { - return nil, fmt.Errorf("%w: %s", errUsage, err) + return nil, fmt.Errorf("%w: %w", script.ErrUsage, err) } var cols []string @@ -581,7 +512,7 @@ func runQueryCmd(query int, db *DB, s *script.State, args []string) (script.Wait args = flags.Args() if len(args) < 2 { - return nil, fmt.Errorf("%w: expected table and key", errUsage) + return nil, fmt.Errorf("expected table and key") } return func(*script.State) (stdout, stderr string, err error) { @@ -974,7 +905,7 @@ func newTabWriter(out io.Writer) *tabwriter.Writer { // sortArgs sorts the arguments to bring '-arg' first. Allows mixing // the argument order. If e.g. key starts with '-', then it'll just -// need to be quoted: "db get foo '-mykey'" +// need to be quoted: "db/get foo '-mykey'" func sortedArgs(args []string) []string { return slices.SortedStableFunc( slices.Values(args), @@ -992,3 +923,7 @@ func sortedArgs(args []string) []string { }, ) } + +func underline(s string) string { + return "\033[4m" + s + "\033[0m" +} diff --git a/testdata/db.txtar b/testdata/db.txtar index 9c02379..286e29d 100644 --- a/testdata/db.txtar +++ b/testdata/db.txtar @@ -6,78 +6,78 @@ hive start # Show the registered tables -db tables +db # Initialized -db initialized -db initialized test1 -db initialized test1 test2 +db/initialized +db/initialized test1 +db/initialized test1 test2 # Show (empty) -db show test1 -db show test2 +db/show test1 +db/show test2 # Insert -db insert test1 obj1.yaml -db insert test1 obj2.yaml -db insert test2 obj2.yaml +db/insert test1 obj1.yaml +db/insert test1 obj2.yaml +db/insert test2 obj2.yaml # Show (non-empty) -db show test1 +db/show test1 grep ^ID.*Tags grep 1.*bar grep 2.*baz -db show test2 +db/show test2 -db show -format=table test1 +db/show -format=table test1 grep ^ID.*Tags grep 1.*bar grep 2.*baz -db show -format=table -columns=Tags test1 +db/show -format=table -columns=Tags test1 grep ^Tags$ grep '^bar, foo$' grep '^baz, foo$' -db show -format=table -o=test1_show.table test1 +db/show -format=table -o=test1_show.table test1 cmp test1.table test1_show.table -db show -format=yaml -o=test1_show.yaml test1 +db/show -format=yaml -o=test1_show.yaml test1 cmp test1.yaml test1_show.yaml -db show -format=json -o=test1_show.json test1 +db/show -format=json -o=test1_show.json test1 cmp test1.json test1_show.json # Get -db get test2 2 -db get -format=table test2 2 +db/get test2 2 +db/get -format=table test2 2 grep '^ID.*Tags$' grep ^2.*baz -db get -format=table -columns=Tags test2 2 +db/get -format=table -columns=Tags test2 2 grep ^Tags$ grep '^baz, foo$' -db get -format=json test2 2 -db get -format=yaml test2 2 -db get -format=yaml -o=obj2_get.yaml test2 2 +db/get -format=json test2 2 +db/get -format=yaml test2 2 +db/get -format=yaml -o=obj2_get.yaml test2 2 cmp obj2.yaml obj2_get.yaml -db get -index=tags -format=yaml -o=obj1_get.yaml test1 bar +db/get -index=tags -format=yaml -o=obj1_get.yaml test1 bar cmp obj1.yaml obj1_get.yaml # List -db list -o=list.table test1 1 +db/list -o=list.table test1 1 cmp obj1.table list.table -db list -o=list.table test1 2 +db/list -o=list.table test1 2 cmp obj2.table list.table -db list -o=list.table -index=tags test1 bar +db/list -o=list.table -index=tags test1 bar cmp obj1.table list.table -db list -o=list.table -index=tags test1 baz +db/list -o=list.table -index=tags test1 baz cmp obj2.table list.table -db list -o=list.table -index=tags test1 foo +db/list -o=list.table -index=tags test1 foo cmp objs.table list.table -db list -format=table -index=tags -columns=Tags test1 foo +db/list -format=table -index=tags -columns=Tags test1 foo grep ^Tags$ grep '^bar, foo$' grep '^baz, foo$' @@ -85,56 +85,56 @@ grep '^baz, foo$' # Prefix # uint64 so can't really prefix search meaningfully, unless # FromString() accomodates partial keys. -db prefix test1 1 +db/prefix test1 1 -db prefix -o=prefix.table -index=tags test1 ba +db/prefix -o=prefix.table -index=tags test1 ba cmp objs.table prefix.table # LowerBound -db lowerbound -o=lb.table test1 0 +db/lowerbound -o=lb.table test1 0 cmp objs.table lb.table -db lowerbound -o=lb.table test1 1 +db/lowerbound -o=lb.table test1 1 cmp objs.table lb.table -db lowerbound -o=lb.table test1 2 +db/lowerbound -o=lb.table test1 2 cmp obj2.table lb.table -db lowerbound -o=lb.table test1 3 +db/lowerbound -o=lb.table test1 3 cmp empty.table lb.table # Compare -db cmp test1 objs.table -db cmp test1 objs_ids.table -db cmp -grep=bar test1 obj1.table -db cmp -grep=baz test1 obj2.table +db/cmp test1 objs.table +db/cmp test1 objs_ids.table +db/cmp -grep=bar test1 obj1.table +db/cmp -grep=baz test1 obj2.table # Delete -db delete test1 obj1.yaml -db cmp test1 obj2.table +db/delete test1 obj1.yaml +db/cmp test1 obj2.table -db delete test1 obj2.yaml -db cmp test1 empty.table +db/delete test1 obj2.yaml +db/cmp test1 empty.table # Delete with get -db insert test1 obj1.yaml -db cmp test1 obj1.table -db get -delete test1 1 -db cmp test1 empty.table +db/insert test1 obj1.yaml +db/cmp test1 obj1.table +db/get -delete test1 1 +db/cmp test1 empty.table # Delete with prefix -db insert test1 obj1.yaml -db insert test1 obj2.yaml -db cmp test1 objs.table -db prefix -index=tags -delete test1 fo -db cmp test1 empty.table +db/insert test1 obj1.yaml +db/insert test1 obj2.yaml +db/cmp test1 objs.table +db/prefix -index=tags -delete test1 fo +db/cmp test1 empty.table # Delete with lowerbound -db insert test1 obj1.yaml -db insert test1 obj2.yaml -db cmp test1 objs.table -db lowerbound -index=id -delete test1 2 -db cmp test1 obj1.table +db/insert test1 obj1.yaml +db/insert test1 obj2.yaml +db/cmp test1 objs.table +db/lowerbound -index=id -delete test1 2 +db/cmp test1 obj1.table # Tables -db tables +db # ---------------------