Skip to content

Commit 05ead22

Browse files
dleviminzidleviminzi
authored
add options for sentinel mode (#32)
* local setup using valkey * add master name to config and sentinel mode for clients * pass mode through and master name
1 parent 72fd4dc commit 05ead22

File tree

7 files changed

+131
-35
lines changed

7 files changed

+131
-35
lines changed

‎Makefile

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,12 @@ publish-chart:
2727
testclients:
2828
GOOS=linux GOARCH=amd64 go build -o bin/throughput e2e/throughput/main.go
2929
GOOS=linux GOARCH=amd64 go build -o bin/fs e2e/fs/main.go
30+
31+
32+
setup: build
33+
@if [ "$(shell kubectl config current-context)" != "k3d-beta9" ]; then \
34+
echo "Current context is not k3d-beta9"; \
35+
exit 1; \
36+
fi
37+
helm install blobcache-valkey oci://registry-1.docker.io/bitnamicharts/valkey --set architecture=standalone --set auth.password=password
38+
cd hack; kubectl apply -f deployment.yaml; cd ..

‎go.mod

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,13 @@
11
module github.com/beam-cloud/blobcache-v2
22

3-
go 1.22.0
4-
5-
toolchain go1.22.4
3+
go 1.22.10
64

75
require (
86
github.com/beam-cloud/ristretto v0.0.0-20241013204426-d1403e359aa2
97
github.com/bsm/redislock v0.9.4
108
github.com/deckarep/golang-set/v2 v2.6.0
119
github.com/djherbis/atime v1.1.0
10+
github.com/go-faker/faker/v4 v4.6.0
1211
github.com/google/uuid v1.6.0
1312
github.com/hanwen/go-fuse/v2 v2.5.1
1413
github.com/knadh/koanf/parsers/json v0.1.0
@@ -21,6 +20,7 @@ require (
2120
github.com/shirou/gopsutil v2.21.11+incompatible
2221
google.golang.org/grpc v1.62.0
2322
google.golang.org/protobuf v1.33.0
23+
gotest.tools v2.2.0+incompatible
2424
tailscale.com v1.72.1
2525
)
2626

@@ -82,7 +82,7 @@ require (
8282
github.com/miekg/dns v1.1.58 // indirect
8383
github.com/mitchellh/copystructure v1.2.0 // indirect
8484
github.com/mitchellh/go-ps v1.0.0 // indirect
85-
github.com/mitchellh/mapstructure v1.5.0 // indirect
85+
github.com/mitchellh/mapstructure v1.5.0
8686
github.com/mitchellh/reflectwalk v1.0.2 // indirect
8787
github.com/pierrec/lz4/v4 v4.1.21 // indirect
8888
github.com/pkg/errors v0.9.1 // indirect
@@ -110,10 +110,10 @@ require (
110110
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect
111111
golang.org/x/mod v0.19.0 // indirect
112112
golang.org/x/net v0.27.0 // indirect
113-
golang.org/x/sync v0.7.0 // indirect
113+
golang.org/x/sync v0.8.0 // indirect
114114
golang.org/x/sys v0.25.0 // indirect
115115
golang.org/x/term v0.22.0 // indirect
116-
golang.org/x/text v0.16.0 // indirect
116+
golang.org/x/text v0.18.0 // indirect
117117
golang.org/x/time v0.5.0 // indirect
118118
golang.org/x/tools v0.23.0 // indirect
119119
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect

‎go.sum

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,8 @@ github.com/gaissmai/bart v0.11.1 h1:5Uv5XwsaFBRo4E5VBcb9TzY8B7zxFf+U7isDxqOrRfc=
9292
github.com/gaissmai/bart v0.11.1/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg=
9393
github.com/github/fakeca v0.1.0 h1:Km/MVOFvclqxPM9dZBC4+QE564nU4gz4iZ0D9pMw28I=
9494
github.com/github/fakeca v0.1.0/go.mod h1:+bormgoGMMuamOscx7N91aOuUST7wdaJ2rNjeohylyo=
95+
github.com/go-faker/faker/v4 v4.6.0 h1:6aOPzNptRiDwD14HuAnEtlTa+D1IfFuEHO8+vEFwjTs=
96+
github.com/go-faker/faker/v4 v4.6.0/go.mod h1:ZmrHuVtTTm2Em9e0Du6CJ9CADaLEzGXW62z1YqFH0m0=
9597
github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0 h1:ymLjT4f35nQbASLnvxEde4XOBL+Sn7rFuV+FOJqkljg=
9698
github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0/go.mod h1:6daplAwHHGbUGib4990V3Il26O0OC4aRyvewaaAihaA=
9799
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
@@ -276,8 +278,8 @@ golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
276278
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
277279
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
278280
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
279-
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
280-
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
281+
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
282+
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
281283
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
282284
golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
283285
golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -293,8 +295,8 @@ golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
293295
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
294296
golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
295297
golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
296-
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
297-
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
298+
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
299+
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
298300
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
299301
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
300302
golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
@@ -317,6 +319,8 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
317319
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
318320
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
319321
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
322+
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
323+
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
320324
gvisor.dev/gvisor v0.0.0-20240722211153-64c016c92987 h1:TU8z2Lh3Bbq77w0t1eG8yRlLcNHzZu3x6mhoH2Mk0c8=
321325
gvisor.dev/gvisor v0.0.0-20240722211153-64c016c92987/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU=
322326
honnef.co/go/tools v0.4.6 h1:oFEHCKeID7to/3autwsWfnuv69j3NsfcXbvJKuIcep8=

‎pkg/metadata.go

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,16 +21,18 @@ type BlobCacheMetadata struct {
2121
lock *RedisLock
2222
}
2323

24-
const (
25-
redisMode RedisMode = "single"
26-
)
27-
2824
func NewBlobCacheMetadata(cfg MetadataConfig) (*BlobCacheMetadata, error) {
25+
redisMode := RedisModeSingle
26+
if cfg.RedisMode != "" {
27+
redisMode = cfg.RedisMode
28+
}
29+
2930
rdb, err := NewRedisClient(RedisConfig{
3031
Addrs: []string{cfg.RedisAddr},
3132
Mode: redisMode,
3233
Password: cfg.RedisPasswd,
3334
EnableTLS: cfg.RedisTLSEnabled,
35+
MasterName: cfg.RedisMasterName,
3436
InsecureSkipVerify: true, // HOTFIX: tailscale certs don't match in-cluster certs
3537
})
3638
if err != nil {

‎pkg/redis.go

Lines changed: 4 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"time"
1414

1515
"github.com/bsm/redislock"
16+
"github.com/mitchellh/mapstructure"
1617
"github.com/redis/go-redis/v9"
1718
)
1819

@@ -42,7 +43,7 @@ func WithClientName(name string) func(*redis.UniversalOptions) {
4243

4344
func NewRedisClient(config RedisConfig, options ...func(*redis.UniversalOptions)) (*RedisClient, error) {
4445
opts := &redis.UniversalOptions{}
45-
CopyStruct(&config, opts)
46+
mapstructure.Decode(config, opts)
4647

4748
for _, opt := range options {
4849
opt(opts)
@@ -60,6 +61,8 @@ func NewRedisClient(config RedisConfig, options ...func(*redis.UniversalOptions)
6061
client = redis.NewClient(opts.Simple())
6162
case RedisModeCluster:
6263
client = redis.NewClusterClient(opts.Cluster())
64+
case RedisModeSentinel:
65+
client = redis.NewFailoverClient(opts.Failover())
6366
default:
6467
return nil, ErrUnknownRedisMode
6568
}
@@ -306,21 +309,6 @@ func (l *RedisLock) Release(key string) error {
306309
return redislock.ErrLockNotHeld
307310
}
308311

309-
// Attempts to copy field values of the same name from the src to the dst struct.
310-
func CopyStruct(src, dst interface{}) {
311-
srcVal := reflect.ValueOf(src).Elem()
312-
dstVal := reflect.ValueOf(dst).Elem()
313-
314-
for i := 0; i < srcVal.NumField(); i++ {
315-
srcField := srcVal.Type().Field(i).Name
316-
dstField := dstVal.FieldByName(srcField)
317-
318-
if dstField.IsValid() && dstField.CanSet() {
319-
dstField.Set(srcVal.Field(i))
320-
}
321-
}
322-
}
323-
324312
// Copies the result of HGetAll to a provided struct.
325313
// If a field cannot be parsed, we use Go's default value.
326314
// Struct fields must have the redis tag on them otherwise they will be ignored.

‎pkg/redis_test.go

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package blobcache
2+
3+
import (
4+
"testing"
5+
6+
"github.com/go-faker/faker/v4"
7+
"github.com/mitchellh/mapstructure"
8+
redis "github.com/redis/go-redis/v9"
9+
"gotest.tools/assert"
10+
)
11+
12+
func Test_CopyConfig(t *testing.T) {
13+
14+
var config RedisConfig
15+
faker.FakeData(&config)
16+
17+
tests := []struct {
18+
name string
19+
sentinel bool
20+
inputConfig RedisConfig
21+
expectedOptions redis.UniversalOptions
22+
}{
23+
{
24+
name: "simple test",
25+
inputConfig: RedisConfig{
26+
Addrs: []string{"localhost:6379"},
27+
},
28+
expectedOptions: redis.UniversalOptions{
29+
Addrs: []string{"localhost:6379"},
30+
},
31+
},
32+
{
33+
name: "full config",
34+
inputConfig: config,
35+
expectedOptions: redis.UniversalOptions{
36+
Addrs: config.Addrs,
37+
ClientName: config.ClientName,
38+
Username: config.Username,
39+
Password: config.Password,
40+
MinIdleConns: config.MinIdleConns,
41+
MaxIdleConns: config.MaxIdleConns,
42+
ConnMaxIdleTime: config.ConnMaxIdleTime,
43+
ConnMaxLifetime: config.ConnMaxLifetime,
44+
DialTimeout: config.DialTimeout,
45+
ReadTimeout: config.ReadTimeout,
46+
WriteTimeout: config.WriteTimeout,
47+
MaxRedirects: config.MaxRedirects,
48+
MaxRetries: config.MaxRetries,
49+
PoolSize: config.PoolSize,
50+
RouteByLatency: config.RouteByLatency,
51+
},
52+
},
53+
{
54+
name: "sentinel config",
55+
sentinel: true,
56+
inputConfig: RedisConfig{
57+
Addrs: []string{"localhost:26379"},
58+
MasterName: "mymaster",
59+
RouteByLatency: true,
60+
},
61+
expectedOptions: redis.UniversalOptions{
62+
Addrs: []string{"localhost:26379"},
63+
MasterName: "mymaster",
64+
},
65+
},
66+
}
67+
68+
for _, test := range tests {
69+
t.Run(test.name, func(t *testing.T) {
70+
var actualOptions redis.UniversalOptions
71+
mapstructure.Decode(test.inputConfig, &actualOptions)
72+
assert.Equal(t, test.expectedOptions.Addrs[0], actualOptions.Addrs[0])
73+
assert.Equal(t, test.expectedOptions.ClientName, actualOptions.ClientName)
74+
assert.Equal(t, test.expectedOptions.Username, actualOptions.Username)
75+
assert.Equal(t, test.expectedOptions.Password, actualOptions.Password)
76+
assert.Equal(t, test.expectedOptions.MinIdleConns, actualOptions.MinIdleConns)
77+
assert.Equal(t, test.expectedOptions.MaxIdleConns, actualOptions.MaxIdleConns)
78+
assert.Equal(t, test.expectedOptions.ConnMaxIdleTime, actualOptions.ConnMaxIdleTime)
79+
assert.Equal(t, test.expectedOptions.ConnMaxLifetime, actualOptions.ConnMaxLifetime)
80+
81+
if test.sentinel {
82+
failoverOpts := actualOptions.Failover()
83+
assert.Equal(t, test.inputConfig.Addrs[0], failoverOpts.SentinelAddrs[0])
84+
assert.Equal(t, test.inputConfig.MasterName, failoverOpts.MasterName)
85+
}
86+
})
87+
}
88+
89+
}

‎pkg/types.go

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,16 +57,19 @@ type TailscaleConfig struct {
5757
}
5858

5959
type MetadataConfig struct {
60-
RedisAddr string `key:"redisAddr" json:"redis_addr"`
61-
RedisPasswd string `key:"redisPasswd" json:"redis_passwd"`
62-
RedisTLSEnabled bool `key:"redisTLSEnabled" json:"redis_tls_enabled"`
60+
RedisAddr string `key:"redisAddr" json:"redis_addr"`
61+
RedisPasswd string `key:"redisPasswd" json:"redis_passwd"`
62+
RedisTLSEnabled bool `key:"redisTLSEnabled" json:"redis_tls_enabled"`
63+
RedisMode RedisMode `key:"redisMode" json:"redis_mode"`
64+
RedisMasterName string `key:"redisMasterName" json:"redis_master_name"`
6365
}
6466

6567
type RedisMode string
6668

6769
var (
68-
RedisModeSingle RedisMode = "single"
69-
RedisModeCluster RedisMode = "cluster"
70+
RedisModeSingle RedisMode = "single"
71+
RedisModeCluster RedisMode = "cluster"
72+
RedisModeSentinel RedisMode = "sentinel"
7073
)
7174

7275
type RedisConfig struct {
@@ -88,6 +91,7 @@ type RedisConfig struct {
8891
Username string `key:"username" json:"username"`
8992
Password string `key:"password" json:"password"`
9093
RouteByLatency bool `key:"routeByLatency" json:"route_by_latency"`
94+
MasterName string `key:"masterName" json:"master_name"`
9195
}
9296

9397
type BlobFsConfig struct {

0 commit comments

Comments
 (0)