Skip to content

Commit

Permalink
feat: benchmarks
Browse files Browse the repository at this point in the history
  • Loading branch information
@adityathebe
adityathebe committed Dec 20, 2024
1 parent 28ab38a commit 40a1b42
Show file tree
Hide file tree
Showing 15 changed files with 549 additions and 74 deletions.
4 changes: 4 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ ginkgo:
test: ginkgo
ginkgo -r -v

.PHONY: bench
bench:
go test -bench=. -benchtime=10s -timeout 30m github.com/flanksource/duty/bench

fmt:
go fmt ./...

Expand Down
156 changes: 156 additions & 0 deletions bench/bench_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
package bench_test

import (
"fmt"
"os"
"testing"

"github.com/flanksource/commons/logger"
"github.com/flanksource/duty/context"
"github.com/flanksource/duty/shutdown"
"github.com/flanksource/duty/tests/setup"
)

type DistinctBenchConfig struct {
viewName string
column string
}

var distinctColumnBenchConfigs = []DistinctBenchConfig{
{"config_changes", "change_type"},
{"config_names", "type"},
{"config_summary", "type"},
}

var aggregatedViews = []string{
"analysis_types",
"analyzer_types",
"change_types",
"config_classes",
"config_types",
}

var (
testCtx context.Context
connUrl string

// number of total configs in the database
testSizes = []int{10_000, 25_000, 50_000, 100_000}
)

func setupTestDB(dbPath string) error {
logger.Infof("using %q as the pg data dir", dbPath)
os.Setenv(setup.DUTY_DB_DATA_DIR, dbPath)

shutdown.AddHookWithPriority("delete data dir", shutdown.PriorityCritical+1, func() {
if err := os.RemoveAll(dbPath); err != nil {
logger.Errorf("failed to delete data dir: %v", err)
}
})

var err error
testCtx, err = setup.SetupDB("test",
setup.WithoutDummyData, // we generate the dummy data
setup.WithoutRLS, // start without RLS
)
if err != nil {
return fmt.Errorf("failed to setup db: %v", err)
}
connUrl = testCtx.Value("db_url").(string)
return nil
}

func TestMain(m *testing.M) {
shutdown.WaitForSignal()

// Create a fixed postgres data directory
dbDataPath, err := os.CreateTemp("", "bench-pg-dir-*")
if err != nil {
shutdown.ShutdownAndExit(1, "failed to create temp dir for db")
}

if err := setupTestDB(dbDataPath.Name()); err != nil {
shutdown.ShutdownAndExit(1, err.Error())
}

result := m.Run()
shutdown.ShutdownAndExit(result, "exiting ...")
}

func BenchmarkMain(b *testing.B) {
for _, size := range testSizes {
resetPG(b, false)
_, err := setupConfigsForSize(testCtx, size)
if err != nil {
b.Fatalf("failed to setup configs for size %d: %v", size, err)
}

b.Run(fmt.Sprintf("Sample-%d", size), func(b *testing.B) {
for _, config := range distinctColumnBenchConfigs {
runDistinctColumnBenchmark(b, config)
}

for _, config := range aggregatedViews {
runViewsBenchmark(b, config)
}
})
}
}

func runDistinctColumnBenchmark(b *testing.B, config DistinctBenchConfig) {
b.Run(config.viewName, func(b *testing.B) {
for _, rls := range []bool{false, true} {
resetPG(b, rls)
name := "Without RLS"
if rls {
name = "With RLS"
}

b.Run(name, func(b *testing.B) {
for i := 0; i < b.N; i++ {
if rls {
b.StopTimer()
tags := sampleTags[i%len(sampleTags)]
if err := setupRLSPayload(testCtx, tags); err != nil {
b.Fatalf("failed to setup rls payload with tag(%v): %v", tags, err)
}
b.StartTimer()
}

if err := fetchConfigSummary(testCtx, config.viewName, config.column); err != nil {
b.Fatalf("%v", err)
}
}
})
}
})
}

func runViewsBenchmark(b *testing.B, view string) {
b.Run(view, func(b *testing.B) {
for _, rls := range []bool{false, true} {
resetPG(b, rls)
name := "Without RLS"
if rls {
name = "With RLS"
}

b.Run(name, func(b *testing.B) {
for i := 0; i < b.N; i++ {
if rls {
b.StopTimer()
tags := sampleTags[i%len(sampleTags)]
if err := setupRLSPayload(testCtx, tags); err != nil {
b.Fatalf("failed to setup rls payload with tag(%v): %v", tags, err)
}
b.StartTimer()
}

if err := fetchView(testCtx, view); err != nil {
b.Fatalf("%v", err)
}
}
})
}
})
}
85 changes: 85 additions & 0 deletions bench/benchmark.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
# PostgreSQL RLS Benchmark

## Running Benchmarks

query duration to fetch 10k, 25k, 50k and 100k config items in random are recorded.

```bash
make bench
```

## Results

```
goos: linux
goarch: amd64
pkg: github.com/flanksource/duty/bench
cpu: Intel(R) Core(TM) i9-14900K
BenchmarkMain/Sample-10000/config_changes/Without_RLS-32 7242 1623646 ns/op
BenchmarkMain/Sample-10000/config_changes/With_RLS-32 644 19038511 ns/op
BenchmarkMain/Sample-10000/config_names/Without_RLS-32 1611 7300407 ns/op
BenchmarkMain/Sample-10000/config_names/With_RLS-32 1335 9254179 ns/op
BenchmarkMain/Sample-10000/config_summary/Without_RLS-32 668 17770548 ns/op
BenchmarkMain/Sample-10000/config_summary/With_RLS-32 162 74343968 ns/op
BenchmarkMain/Sample-10000/analysis_types/Without_RLS-32 9600 1241916 ns/op
BenchmarkMain/Sample-10000/analysis_types/With_RLS-32 9379 1257520 ns/op
BenchmarkMain/Sample-10000/analyzer_types/Without_RLS-32 9411 1196734 ns/op
BenchmarkMain/Sample-10000/analyzer_types/With_RLS-32 9752 1220373 ns/op
BenchmarkMain/Sample-10000/change_types/Without_RLS-32 7281 1613735 ns/op
BenchmarkMain/Sample-10000/change_types/With_RLS-32 7162 1632322 ns/op
BenchmarkMain/Sample-10000/config_classes/Without_RLS-32 10000 1061757 ns/op
BenchmarkMain/Sample-10000/config_classes/With_RLS-32 1414 8434407 ns/op
BenchmarkMain/Sample-10000/config_types/Without_RLS-32 9704 1228238 ns/op
BenchmarkMain/Sample-10000/config_types/With_RLS-32 1432 8347937 ns/op
BenchmarkMain/Sample-25000/config_changes/Without_RLS-32 3128 3847068 ns/op
BenchmarkMain/Sample-25000/config_changes/With_RLS-32 265 46023904 ns/op
BenchmarkMain/Sample-25000/config_names/Without_RLS-32 710 16903926 ns/op
BenchmarkMain/Sample-25000/config_names/With_RLS-32 562 21754243 ns/op
BenchmarkMain/Sample-25000/config_summary/Without_RLS-32 278 44553859 ns/op
BenchmarkMain/Sample-25000/config_summary/With_RLS-32 62 197152798 ns/op
BenchmarkMain/Sample-25000/analysis_types/Without_RLS-32 4063 2961000 ns/op
BenchmarkMain/Sample-25000/analysis_types/With_RLS-32 3919 2941717 ns/op
BenchmarkMain/Sample-25000/analyzer_types/Without_RLS-32 4131 2833564 ns/op
BenchmarkMain/Sample-25000/analyzer_types/With_RLS-32 4158 2828328 ns/op
BenchmarkMain/Sample-25000/change_types/Without_RLS-32 3079 3888320 ns/op
BenchmarkMain/Sample-25000/change_types/With_RLS-32 3114 3862728 ns/op
BenchmarkMain/Sample-25000/config_classes/Without_RLS-32 4621 2445431 ns/op
BenchmarkMain/Sample-25000/config_classes/With_RLS-32 601 20055039 ns/op
BenchmarkMain/Sample-25000/config_types/Without_RLS-32 4089 2896618 ns/op
BenchmarkMain/Sample-25000/config_types/With_RLS-32 612 20079241 ns/op
BenchmarkMain/Sample-50000/config_changes/Without_RLS-32 1368 8782963 ns/op
BenchmarkMain/Sample-50000/config_changes/With_RLS-32 128 93016624 ns/op
BenchmarkMain/Sample-50000/config_names/Without_RLS-32 368 32866269 ns/op
BenchmarkMain/Sample-50000/config_names/With_RLS-32 278 43000173 ns/op
BenchmarkMain/Sample-50000/config_summary/Without_RLS-32 126 93310303 ns/op
BenchmarkMain/Sample-50000/config_summary/With_RLS-32 31 392169273 ns/op
BenchmarkMain/Sample-50000/analysis_types/Without_RLS-32 2122 5733683 ns/op
BenchmarkMain/Sample-50000/analysis_types/With_RLS-32 2062 5816716 ns/op
BenchmarkMain/Sample-50000/analyzer_types/Without_RLS-32 2172 5527429 ns/op
BenchmarkMain/Sample-50000/analyzer_types/With_RLS-32 2143 5595383 ns/op
BenchmarkMain/Sample-50000/change_types/Without_RLS-32 1597 7700942 ns/op
BenchmarkMain/Sample-50000/change_types/With_RLS-32 1552 7696690 ns/op
BenchmarkMain/Sample-50000/config_classes/Without_RLS-32 2396 4892954 ns/op
BenchmarkMain/Sample-50000/config_classes/With_RLS-32 300 39181990 ns/op
BenchmarkMain/Sample-50000/config_types/Without_RLS-32 2108 5691502 ns/op
BenchmarkMain/Sample-50000/config_types/With_RLS-32 302 39215572 ns/op
BenchmarkMain/Sample-100000/config_changes/Without_RLS-32 644 16781381 ns/op
BenchmarkMain/Sample-100000/config_changes/With_RLS-32 63 186536096 ns/op
BenchmarkMain/Sample-100000/config_names/Without_RLS-32 168 69874148 ns/op
BenchmarkMain/Sample-100000/config_names/With_RLS-32 142 84443880 ns/op
BenchmarkMain/Sample-100000/config_summary/Without_RLS-32 70 170671524 ns/op
BenchmarkMain/Sample-100000/config_summary/With_RLS-32 13 820575742 ns/op
BenchmarkMain/Sample-100000/analysis_types/Without_RLS-32 1042 11419541 ns/op
BenchmarkMain/Sample-100000/analysis_types/With_RLS-32 1041 11417138 ns/op
BenchmarkMain/Sample-100000/analyzer_types/Without_RLS-32 1060 11179349 ns/op
BenchmarkMain/Sample-100000/analyzer_types/With_RLS-32 1058 11257873 ns/op
BenchmarkMain/Sample-100000/change_types/Without_RLS-32 651 16418673 ns/op
BenchmarkMain/Sample-100000/change_types/With_RLS-32 648 16374548 ns/op
BenchmarkMain/Sample-100000/config_classes/Without_RLS-32 1182 9982530 ns/op
BenchmarkMain/Sample-100000/config_classes/With_RLS-32 152 78867207 ns/op
BenchmarkMain/Sample-100000/config_types/Without_RLS-32 1083 11456634 ns/op
BenchmarkMain/Sample-100000/config_types/With_RLS-32 150 78740119 ns/op
```
Loading

0 comments on commit 40a1b42

Please sign in to comment.