Skip to content

Commit 5980744

Browse files
kyleishiekyleishie
authored
Merge pull request #47 from ctrliq/mirror-configs
Update mirror plugin
2 parents fc77f54 + 26ef4b9 commit 5980744

File tree

26 files changed

+1761
-235
lines changed

26 files changed

+1761
-235
lines changed

‎charts/beskar-mirror/values.yaml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,4 +127,7 @@ configData:
127127
gcs:
128128
bucket: beskar-mirror
129129
azure:
130-
container: beskar-mirror
130+
container: beskar-mirror
131+
132+
sync:
133+
max_worker_count: 50

‎go.mod

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ require (
88
github.com/RussellLuo/kun v0.4.5
99
github.com/RussellLuo/validating/v3 v3.0.0-beta.1
1010
github.com/adlio/schema v1.3.4
11-
github.com/antoniomika/go-rsync v0.0.0-20220817021523-f831db35f9a3
1211
github.com/aws/aws-sdk-go v1.48.10
1312
github.com/cavaliergopher/rpm v1.2.0
13+
github.com/cenkalti/backoff v2.2.1+incompatible
1414
github.com/cenkalti/backoff/v4 v4.2.1
1515
github.com/distribution/distribution/v3 v3.0.0-alpha.1
1616
github.com/distribution/reference v0.5.0
@@ -34,6 +34,7 @@ require (
3434
github.com/twmb/murmur3 v1.1.8
3535
github.com/ulikunitz/xz v0.5.11
3636
github.com/vishvananda/netlink v1.2.1-beta.2
37+
go.ciq.dev/go-rsync v0.0.0-20240304021629-0a3bb196e6d1
3738
go.opentelemetry.io/otel v1.21.0
3839
go.opentelemetry.io/otel/trace v1.21.0
3940
gocloud.dev v0.32.0

‎go.sum

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
5757
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
5858
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
5959
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
60-
github.com/antoniomika/go-rsync v0.0.0-20220817021523-f831db35f9a3 h1:5mL0NCuUVoX5omCZhvf7yPW1wzgRlo3cWGLDUnN6kkM=
61-
github.com/antoniomika/go-rsync v0.0.0-20220817021523-f831db35f9a3/go.mod h1:rKzRO3ppwfCUpHMf/IEnJuwuGsr6yi0rlG7/RE32oEY=
6260
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
6361
github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
6462
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
@@ -129,6 +127,7 @@ github.com/bytecodealliance/wasmtime-go/v3 v3.0.2/go.mod h1:RnUjnIXxEJcL6BgCvNyz
129127
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
130128
github.com/cavaliergopher/rpm v1.2.0 h1:s0h+QeVK252QFTolkhGiMeQ1f+tMeIMhGl8B1HUmGUc=
131129
github.com/cavaliergopher/rpm v1.2.0/go.mod h1:R0q3vTqa7RUvPofAZYrnjJ63hh2vngjFfphuXiExVos=
130+
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
132131
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
133132
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
134133
github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
@@ -690,6 +689,8 @@ github.com/yashtewari/glob-intersection v0.2.0/go.mod h1:LK7pIC3piUjovexikBbJ26Y
690689
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
691690
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
692691
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
692+
go.ciq.dev/go-rsync v0.0.0-20240304021629-0a3bb196e6d1 h1:lYxtzhvoRGnoET/RcKJDnRnmaHuGKBCUIj3D1ZubBNg=
693+
go.ciq.dev/go-rsync v0.0.0-20240304021629-0a3bb196e6d1/go.mod h1:xOHMiPHUTm8AQpxu4n14T8bRuT/izQISy8ycm/Q3LLY=
693694
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
694695
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
695696
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=

‎internal/plugins/mirror/api.go

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,13 +53,27 @@ func (p *Plugin) SyncRepository(ctx context.Context, repository string, wait boo
5353
return p.repositoryManager.Get(ctx, repository).SyncRepository(ctx, wait)
5454
}
5555

56+
func (p *Plugin) GenerateRepository(ctx context.Context, repository string) (err error) {
57+
if err := checkRepository(repository); err != nil {
58+
return err
59+
}
60+
return p.repositoryManager.Get(ctx, repository).GenerateRepository(ctx)
61+
}
62+
5663
func (p *Plugin) GetRepositorySyncStatus(ctx context.Context, repository string) (syncStatus *apiv1.SyncStatus, err error) {
5764
if err := checkRepository(repository); err != nil {
5865
return nil, err
5966
}
6067
return p.repositoryManager.Get(ctx, repository).GetRepositorySyncStatus(ctx)
6168
}
6269

70+
func (p *Plugin) GetRepositorySyncPlan(ctx context.Context, repository string) (syncPlan *apiv1.RepositorySyncPlan, err error) {
71+
if err := checkRepository(repository); err != nil {
72+
return nil, err
73+
}
74+
return p.repositoryManager.Get(ctx, repository).GetRepositorySyncPlan(ctx)
75+
}
76+
6377
func (p *Plugin) ListRepositoryLogs(ctx context.Context, repository string, page *apiv1.Page) (logs []apiv1.RepositoryLog, err error) {
6478
if err := checkRepository(repository); err != nil {
6579
return nil, err
@@ -80,3 +94,17 @@ func (p *Plugin) GetRepositoryFile(ctx context.Context, repository, file string)
8094
}
8195
return p.repositoryManager.Get(ctx, repository).GetRepositoryFile(ctx, file)
8296
}
97+
98+
func (p *Plugin) GetRepositoryFileCount(ctx context.Context, repository string) (count int, err error) {
99+
if err := checkRepository(repository); err != nil {
100+
return 0, err
101+
}
102+
return p.repositoryManager.Get(ctx, repository).GetRepositoryFileCount(ctx)
103+
}
104+
105+
func (p *Plugin) DeleteRepositoryFile(ctx context.Context, repository, file string) (err error) {
106+
if err := checkRepository(repository); err != nil {
107+
return err
108+
}
109+
return p.repositoryManager.Get(ctx, repository).DeleteRepositoryFile(ctx, file)
110+
}

‎internal/plugins/mirror/pkg/config/beskar-mirror.go

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,15 @@ const (
3131
var defaultBeskarMirrorConfig string
3232

3333
type BeskarMirrorConfig struct {
34-
Version string `yaml:"version"`
35-
Log log.Config `yaml:"log"`
36-
Addr string `yaml:"addr"`
37-
Gossip gossip.Config `yaml:"gossip"`
38-
Storage storage.Config `yaml:"storage"`
39-
Profiling bool `yaml:"profiling"`
40-
DataDir string `yaml:"datadir"`
41-
ConfigDirectory string `yaml:"-"`
34+
Version string `yaml:"version"`
35+
Log log.Config `yaml:"log"`
36+
Addr string `yaml:"addr"`
37+
Gossip gossip.Config `yaml:"gossip"`
38+
Storage storage.Config `yaml:"storage"`
39+
Profiling bool `yaml:"profiling"`
40+
DataDir string `yaml:"datadir"`
41+
ConfigDirectory string `yaml:"-"`
42+
Sync config.SyncConfig `yaml:"sync"`
4243
}
4344

4445
func (bc BeskarMirrorConfig) ListenIP() (string, error) {

‎internal/plugins/mirror/pkg/config/beskar-mirror_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,6 @@ func TestParseBeskarMirrorConfig(t *testing.T) {
4747
require.Equal(t, "0.0.0.0:5501", bc.Gossip.Addr)
4848
require.Equal(t, "XD1IOhcp0HWFgZJ/HAaARqMKJwfMWtz284Yj7wxmerA=", bc.Gossip.Key)
4949
require.Equal(t, []string{"127.0.0.1:5102"}, bc.Gossip.Peers)
50+
51+
require.Equal(t, 50, bc.Sync.MaxWorkerCount)
5052
}

‎internal/plugins/mirror/pkg/config/default/beskar-mirror.yaml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,4 +34,7 @@ storage:
3434
azure:
3535
container: beskar-mirror
3636
account-name: account_name
37-
account-key: base64_encoded_account_key
37+
account-key: base64_encoded_account_key
38+
39+
sync:
40+
max_worker_count: 50

‎internal/plugins/mirror/pkg/index/embedded/index.html.tpl

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@ td, th {
1111
<hr>
1212
<table>
1313
<tbody>
14+
{{- if .Previous }}
1415
<td><a href="{{ .Previous }}">../</td>
16+
{{- end }}
1517
{{- range $dir := .Directories }}
1618
<tr>
1719
<td><a href="{{ $dir.Ref }}/">{{ $dir.Name }}/</a></td>

‎internal/plugins/mirror/pkg/mirrordb/repository.go

Lines changed: 151 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,13 @@ var repositorySchemas embed.FS
2020
type RepositoryFile struct {
2121
Tag string `db:"tag"`
2222
Name string `db:"name"`
23+
Reference string `db:"reference"`
24+
Parent string `db:"parent"`
2325
Link string `db:"link"`
2426
ModifiedTime int64 `db:"modified_time"`
2527
Mode uint32 `db:"mode"`
2628
Size uint64 `db:"size"`
29+
ConfigID uint64 `db:"config_id"`
2730
}
2831

2932
type RepositoryDB struct {
@@ -56,15 +59,15 @@ func (db *RepositoryDB) AddFile(ctx context.Context, file *RepositoryFile) error
5659
}
5760

5861
//nolint:gosec
59-
s := md5.Sum([]byte(file.Name))
62+
s := md5.Sum([]byte(file.Reference))
6063
file.Tag = hex.EncodeToString(s[:])
6164

6265
db.Lock()
6366
result, err := db.NamedExecContext(
6467
ctx,
6568
// BE CAREFUL and respect the table's columns order !!
66-
"INSERT INTO files VALUES(:tag, :name, :link, :modified_time, :mode, :size) "+
67-
"ON CONFLICT (tag) DO UPDATE SET name = :name, link = :link, modified_time = :modified_time, mode = :mode, size = :size",
69+
"INSERT INTO files VALUES(:tag, :name, :reference, :parent, :link, :modified_time, :mode, :size, :config_id) "+
70+
"ON CONFLICT (tag) DO UPDATE SET name = :name, reference = :reference, parent = :parent, link = :link, modified_time = :modified_time, mode = :mode, size = :size, config_id = :config_id",
6871
file,
6972
)
7073
db.Unlock()
@@ -159,6 +162,56 @@ func (db *RepositoryDB) GetFileByName(ctx context.Context, name string) (*Reposi
159162
return file, nil
160163
}
161164

165+
func (db *RepositoryDB) GetFileByReference(ctx context.Context, reference string) (*RepositoryFile, error) {
166+
db.Reference.Add(1)
167+
defer db.Reference.Add(-1)
168+
169+
if err := db.Open(ctx); err != nil {
170+
return nil, err
171+
}
172+
173+
rows, err := db.QueryxContext(ctx, "SELECT * FROM files WHERE reference = ? LIMIT 1", reference)
174+
if err != nil {
175+
return nil, err
176+
}
177+
defer rows.Close()
178+
179+
file := new(RepositoryFile)
180+
181+
if !rows.Next() {
182+
return nil, sqlite.ErrNoEntryFound
183+
}
184+
if err := rows.StructScan(file); err != nil {
185+
return nil, err
186+
}
187+
188+
return file, nil
189+
}
190+
191+
func (db *RepositoryDB) DeleteFileByName(ctx context.Context, name string) error {
192+
db.Reference.Add(1)
193+
defer db.Reference.Add(-1)
194+
195+
if err := db.Open(ctx); err != nil {
196+
return err
197+
}
198+
199+
db.Lock()
200+
result, err := db.ExecContext(ctx, "DELETE FROM files WHERE name = ?", name)
201+
db.Unlock()
202+
203+
if err != nil {
204+
return err
205+
}
206+
207+
_, err = result.RowsAffected()
208+
if err != nil {
209+
return err
210+
}
211+
212+
return nil
213+
}
214+
162215
type WalkFileFunc func(*RepositoryFile) error
163216

164217
func (db *RepositoryDB) WalkFiles(ctx context.Context, walkFn WalkFileFunc) error {
@@ -223,6 +276,101 @@ func (db *RepositoryDB) WalkSymlinks(ctx context.Context, walkFn WalkFileFunc) e
223276
return nil
224277
}
225278

279+
func (db *RepositoryDB) WalkFilesByParent(ctx context.Context, parent string, walkFn WalkFileFunc) error {
280+
if walkFn == nil {
281+
return fmt.Errorf("no file walk function provided")
282+
}
283+
284+
db.Reference.Add(1)
285+
defer db.Reference.Add(-1)
286+
287+
if err := db.Open(ctx); err != nil {
288+
return err
289+
}
290+
291+
rows, err := db.QueryxContext(ctx, "SELECT * FROM files WHERE parent = ?", parent)
292+
if err != nil {
293+
return err
294+
}
295+
defer rows.Close()
296+
297+
for rows.Next() {
298+
file := new(RepositoryFile)
299+
err := rows.StructScan(file)
300+
if err != nil {
301+
return err
302+
} else if err := walkFn(file); err != nil {
303+
return err
304+
}
305+
}
306+
307+
return nil
308+
}
309+
310+
func (db *RepositoryDB) WalkFilesByConfigID(ctx context.Context, configID uint64, walkFn WalkFileFunc) error {
311+
if walkFn == nil {
312+
return fmt.Errorf("no file walk function provided")
313+
}
314+
315+
db.Reference.Add(1)
316+
defer db.Reference.Add(-1)
317+
318+
if err := db.Open(ctx); err != nil {
319+
return err
320+
}
321+
322+
rows, err := db.QueryxContext(ctx, "SELECT * FROM files WHERE config_id = ?", configID)
323+
if err != nil {
324+
return err
325+
}
326+
defer rows.Close()
327+
328+
for rows.Next() {
329+
file := new(RepositoryFile)
330+
err := rows.StructScan(file)
331+
if err != nil {
332+
return err
333+
} else if err := walkFn(file); err != nil {
334+
return err
335+
}
336+
}
337+
338+
return nil
339+
}
340+
341+
type WalkStringFunc func(*string) error
342+
343+
func (db *RepositoryDB) WalkFilesByDistinctParent(ctx context.Context, walkFn WalkStringFunc) error {
344+
if walkFn == nil {
345+
return fmt.Errorf("no file walk function provided")
346+
}
347+
348+
db.Reference.Add(1)
349+
defer db.Reference.Add(-1)
350+
351+
if err := db.Open(ctx); err != nil {
352+
return err
353+
}
354+
355+
rows, err := db.QueryxContext(ctx, "SELECT DISTINCT parent FROM files")
356+
if err != nil {
357+
return err
358+
}
359+
defer rows.Close()
360+
361+
for rows.Next() {
362+
parent := new(string)
363+
err := rows.Scan(parent)
364+
if err != nil {
365+
return err
366+
} else if err := walkFn(parent); err != nil {
367+
return err
368+
}
369+
}
370+
371+
return nil
372+
}
373+
226374
func (db *RepositoryDB) CountFiles(ctx context.Context) (int, error) {
227375
db.Reference.Add(1)
228376
defer db.Reference.Add(-1)

‎internal/plugins/mirror/pkg/mirrordb/schema/repository/000000_create_files_table.sql

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
11
CREATE TABLE IF NOT EXISTS files (
22
tag TEXT PRIMARY KEY,
33
name TEXT,
4+
reference TEXT,
5+
parent TEXT,
46
link TEXT,
57
modified_time INTEGER,
68
mode INTEGER,
7-
size INTEGER
9+
size INTEGER,
10+
config_id INTEGER
811
);
912

10-
CREATE INDEX filename_idx ON files(name);
13+
CREATE INDEX files_name_idx ON files(name);
14+
CREATE INDEX files_reference_idx ON files(reference);
15+
CREATE INDEX files_parent_idx ON files(parent);
16+
CREATE INDEX files_config_id_idx ON files(config_id);

0 commit comments

Comments
 (0)