diff --git a/.common-ci.yml b/.common-ci.yml index 6e257546a..6a37d7989 100644 --- a/.common-ci.yml +++ b/.common-ci.yml @@ -150,7 +150,7 @@ trigger-pipeline: # Download the regctl binary for use in the release steps .regctl-setup: before_script: - - export REGCTL_VERSION=v0.7.1 + - export REGCTL_VERSION=v0.7.2 - apk add --no-cache curl - mkdir -p bin - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64 diff --git a/go.mod b/go.mod index 407a36a0e..2aeab9cd5 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/operator-framework/api v0.27.0 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.78.1 github.com/prometheus/client_golang v1.20.5 - github.com/regclient/regclient v0.7.1 + github.com/regclient/regclient v0.7.2 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.9.0 github.com/urfave/cli/v2 v2.27.5 @@ -102,7 +102,7 @@ require ( github.com/jmoiron/sqlx v1.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/lib/pq v1.10.9 // indirect diff --git a/go.sum b/go.sum index 157a91a8e..4d07496b4 100644 --- a/go.sum +++ b/go.sum @@ -207,8 +207,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -269,8 +269,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/olareg/olareg v0.1.0 h1:1dXBOgPrig5N7zoXyIZVQqU0QBo6sD9pbL6UYjY75CA= -github.com/olareg/olareg v0.1.0/go.mod h1:RBuU7JW7SoIIxZKzLRhq8sVtQeAHzCAtRrXEBx2KlM4= +github.com/olareg/olareg v0.1.1 h1:Ui7q93zjcoF+U9U71sgqgZWByDoZOpqHitUXEu2xV+g= +github.com/olareg/olareg v0.1.1/go.mod h1:w8NP4SWrHHtxsFaUiv1lnCnYPm4sN1seCd2h7FK/dc0= github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= @@ -317,8 +317,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/regclient/regclient v0.7.1 h1:qEsJrTmZd98fZKjueAbrZCSNGU+ifnr6xjlSAs3WOPs= -github.com/regclient/regclient v0.7.1/go.mod h1:+w/BFtJuw0h0nzIw/z2+1FuA2/dVXBzDq4rYmziJpMc= +github.com/regclient/regclient v0.7.2 h1:vcldDAwBMLtighYVMeb6qNt5+0hKg3AN2IkCc0JIJNM= +github.com/regclient/regclient v0.7.2/go.mod h1:QlA7W9/pvmbblOXM4d49JgfuOTwVXcUMKt3bFuOSVIQ= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= diff --git a/vendor/github.com/klauspost/compress/.goreleaser.yml b/vendor/github.com/klauspost/compress/.goreleaser.yml index a22953805..4528059ca 100644 --- a/vendor/github.com/klauspost/compress/.goreleaser.yml +++ b/vendor/github.com/klauspost/compress/.goreleaser.yml @@ -1,5 +1,5 @@ -# This is an example goreleaser.yaml file with some sane defaults. -# Make sure to check the documentation at http://goreleaser.com +version: 2 + before: hooks: - ./gen.sh @@ -99,7 +99,7 @@ archives: checksum: name_template: 'checksums.txt' snapshot: - name_template: "{{ .Tag }}-next" + version_template: "{{ .Tag }}-next" changelog: sort: asc filters: diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index 05c7359e4..de264c85a 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -16,6 +16,27 @@ This package provides various compression algorithms. # changelog +* Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10) + * gzhttp: Add TransportAlwaysDecompress option. https://github.com/klauspost/compress/pull/978 + * gzhttp: Add supported decompress request body by @mirecl in https://github.com/klauspost/compress/pull/1002 + * s2: Add EncodeBuffer buffer recycling callback https://github.com/klauspost/compress/pull/982 + * zstd: Improve memory usage on small streaming encodes https://github.com/klauspost/compress/pull/1007 + * flate: read data written with partial flush by @vajexal in https://github.com/klauspost/compress/pull/996 + +* Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9) + * s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949 + * flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963 + * Upgrade zip/zlib to 1.22.4 upstream https://github.com/klauspost/compress/pull/970 https://github.com/klauspost/compress/pull/971 + * zstd: BuildDict fails with RLE table https://github.com/klauspost/compress/pull/951 + +* Apr 9th, 2024 - [1.17.8](https://github.com/klauspost/compress/releases/tag/v1.17.8) + * zstd: Reject blocks where reserved values are not 0 https://github.com/klauspost/compress/pull/885 + * zstd: Add RLE detection+encoding https://github.com/klauspost/compress/pull/938 + +* Feb 21st, 2024 - [1.17.7](https://github.com/klauspost/compress/releases/tag/v1.17.7) + * s2: Add AsyncFlush method: Complete the block without flushing by @Jille in https://github.com/klauspost/compress/pull/927 + * s2: Fix literal+repeat exceeds dst crash https://github.com/klauspost/compress/pull/930 + * Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6) * zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923 * s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925 @@ -81,7 +102,7 @@ https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/comp * zstd: Various minor improvements by @greatroar in https://github.com/klauspost/compress/pull/788 https://github.com/klauspost/compress/pull/794 https://github.com/klauspost/compress/pull/795 * s2: Fix huge block overflow https://github.com/klauspost/compress/pull/779 * s2: Allow CustomEncoder fallback https://github.com/klauspost/compress/pull/780 - * gzhttp: Suppport ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799 + * gzhttp: Support ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799 * Mar 13, 2023 - [v1.16.1](https://github.com/klauspost/compress/releases/tag/v1.16.1) * zstd: Speed up + improve best encoder by @greatroar in https://github.com/klauspost/compress/pull/776 @@ -136,7 +157,7 @@ https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/comp * zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649 * Add Go 1.19 - deprecate Go 1.16 https://github.com/klauspost/compress/pull/651 * flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656 - * zstd: Improve "better" compresssion https://github.com/klauspost/compress/pull/657 + * zstd: Improve "better" compression https://github.com/klauspost/compress/pull/657 * s2: Improve "best" compression https://github.com/klauspost/compress/pull/658 * s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635 * s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646 @@ -339,7 +360,7 @@ While the release has been extensively tested, it is recommended to testing when * s2: Fix binaries. * Feb 25, 2021 (v1.11.8) - * s2: Fixed occational out-of-bounds write on amd64. Upgrade recommended. + * s2: Fixed occasional out-of-bounds write on amd64. Upgrade recommended. * s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315) * s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322) * zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314) @@ -518,7 +539,7 @@ While the release has been extensively tested, it is recommended to testing when * Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster. * Feb 19, 2016: Handle small payloads faster in level 1-3. * Feb 19, 2016: Added faster level 2 + 3 compression modes. -* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progresssion in terms of compression. New default level is 5. +* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progression in terms of compression. New default level is 5. * Feb 14, 2016: Snappy: Merge upstream changes. * Feb 14, 2016: Snappy: Fix aggressive skipping. * Feb 14, 2016: Snappy: Update benchmark. diff --git a/vendor/github.com/klauspost/compress/fse/decompress.go b/vendor/github.com/klauspost/compress/fse/decompress.go index cc05d0f7e..0c7dd4ffe 100644 --- a/vendor/github.com/klauspost/compress/fse/decompress.go +++ b/vendor/github.com/klauspost/compress/fse/decompress.go @@ -15,7 +15,7 @@ const ( // It is possible, but by no way guaranteed that corrupt data will // return an error. // It is up to the caller to verify integrity of the returned data. -// Use a predefined Scrach to set maximum acceptable output size. +// Use a predefined Scratch to set maximum acceptable output size. func Decompress(b []byte, s *Scratch) ([]byte, error) { s, err := s.prepare(b) if err != nil { diff --git a/vendor/github.com/klauspost/compress/huff0/decompress.go b/vendor/github.com/klauspost/compress/huff0/decompress.go index 54bd08b25..0f56b02d7 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress.go @@ -1136,7 +1136,7 @@ func (s *Scratch) matches(ct cTable, w io.Writer) { errs++ } if errs > 0 { - fmt.Fprintf(w, "%d errros in base, stopping\n", errs) + fmt.Fprintf(w, "%d errors in base, stopping\n", errs) continue } // Ensure that all combinations are covered. @@ -1152,7 +1152,7 @@ func (s *Scratch) matches(ct cTable, w io.Writer) { errs++ } if errs > 20 { - fmt.Fprintf(w, "%d errros, stopping\n", errs) + fmt.Fprintf(w, "%d errors, stopping\n", errs) break } } diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index 03744fbc7..9c28840c3 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -598,7 +598,9 @@ func (b *blockDec) prepareSequences(in []byte, hist *history) (err error) { printf("RLE set to 0x%x, code: %v", symb, v) } case compModeFSE: - println("Reading table for", tableIndex(i)) + if debugDecoder { + println("Reading table for", tableIndex(i)) + } if seq.fse == nil || seq.fse.preDefined { seq.fse = fseDecoderPool.Get().(*fseDecoder) } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index a4f5bf91f..84a79fde7 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -179,9 +179,9 @@ encodeLoop: if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { // Consider history as well. var seq seq - lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src) + length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -210,12 +210,12 @@ encodeLoop: // Index match start+1 (long) -> s - 1 index0 := s + repOff - s += lenght + repOff + s += length + repOff nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop @@ -241,9 +241,9 @@ encodeLoop: if false && repIndex >= 0 && load6432(src, repIndex) == load6432(src, s+repOff) { // Consider history as well. var seq seq - lenght := 8 + e.matchlen(s+8+repOff2, repIndex+8, src) + length := 8 + e.matchlen(s+8+repOff2, repIndex+8, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -270,11 +270,11 @@ encodeLoop: } blk.sequences = append(blk.sequences, seq) - s += lenght + repOff2 + s += length + repOff2 nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop @@ -708,9 +708,9 @@ encodeLoop: if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { // Consider history as well. var seq seq - lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src) + length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -738,12 +738,12 @@ encodeLoop: blk.sequences = append(blk.sequences, seq) // Index match start+1 (long) -> s - 1 - s += lenght + repOff + s += length + repOff nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop @@ -772,9 +772,9 @@ encodeLoop: if false && repIndex >= 0 && load6432(src, repIndex) == load6432(src, s+repOff) { // Consider history as well. var seq seq - lenght := 8 + e.matchlen(s+8+repOff2, repIndex+8, src) + length := 8 + e.matchlen(s+8+repOff2, repIndex+8, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -801,11 +801,11 @@ encodeLoop: } blk.sequences = append(blk.sequences, seq) - s += lenght + repOff2 + s += length + repOff2 nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index a154c18f7..d36be7bd8 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -138,9 +138,9 @@ encodeLoop: if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { // Consider history as well. var seq seq - lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src) + length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -166,11 +166,11 @@ encodeLoop: println("repeat sequence", seq, "next s:", s) } blk.sequences = append(blk.sequences, seq) - s += lenght + repOff + s += length + repOff nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop @@ -798,9 +798,9 @@ encodeLoop: if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { // Consider history as well. var seq seq - lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src) + length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -826,11 +826,11 @@ encodeLoop: println("repeat sequence", seq, "next s:", s) } blk.sequences = append(blk.sequences, seq) - s += lenght + repOff + s += length + repOff nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index 72af7ef0f..8f8223cd3 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -6,6 +6,7 @@ package zstd import ( "crypto/rand" + "errors" "fmt" "io" "math" @@ -149,6 +150,9 @@ func (e *Encoder) ResetContentSize(w io.Writer, size int64) { // and write CRC if requested. func (e *Encoder) Write(p []byte) (n int, err error) { s := &e.state + if s.eofWritten { + return 0, ErrEncoderClosed + } for len(p) > 0 { if len(p)+len(s.filling) < e.o.blockSize { if e.o.crc { @@ -202,7 +206,7 @@ func (e *Encoder) nextBlock(final bool) error { return nil } if final && len(s.filling) > 0 { - s.current = e.EncodeAll(s.filling, s.current[:0]) + s.current = e.encodeAll(s.encoder, s.filling, s.current[:0]) var n2 int n2, s.err = s.w.Write(s.current) if s.err != nil { @@ -288,6 +292,9 @@ func (e *Encoder) nextBlock(final bool) error { s.filling, s.current, s.previous = s.previous[:0], s.filling, s.current s.nInput += int64(len(s.current)) s.wg.Add(1) + if final { + s.eofWritten = true + } go func(src []byte) { if debugEncoder { println("Adding block,", len(src), "bytes, final:", final) @@ -303,9 +310,6 @@ func (e *Encoder) nextBlock(final bool) error { blk := enc.Block() enc.Encode(blk, src) blk.last = final - if final { - s.eofWritten = true - } // Wait for pending writes. s.wWg.Wait() if s.writeErr != nil { @@ -401,12 +405,20 @@ func (e *Encoder) Flush() error { if len(s.filling) > 0 { err := e.nextBlock(false) if err != nil { + // Ignore Flush after Close. + if errors.Is(s.err, ErrEncoderClosed) { + return nil + } return err } } s.wg.Wait() s.wWg.Wait() if s.err != nil { + // Ignore Flush after Close. + if errors.Is(s.err, ErrEncoderClosed) { + return nil + } return s.err } return s.writeErr @@ -422,6 +434,9 @@ func (e *Encoder) Close() error { } err := e.nextBlock(true) if err != nil { + if errors.Is(s.err, ErrEncoderClosed) { + return nil + } return err } if s.frameContentSize > 0 { @@ -459,6 +474,11 @@ func (e *Encoder) Close() error { } _, s.err = s.w.Write(frame) } + if s.err == nil { + s.err = ErrEncoderClosed + return nil + } + return s.err } @@ -469,6 +489,15 @@ func (e *Encoder) Close() error { // Data compressed with EncodeAll can be decoded with the Decoder, // using either a stream or DecodeAll. func (e *Encoder) EncodeAll(src, dst []byte) []byte { + e.init.Do(e.initialize) + enc := <-e.encoders + defer func() { + e.encoders <- enc + }() + return e.encodeAll(enc, src, dst) +} + +func (e *Encoder) encodeAll(enc encoder, src, dst []byte) []byte { if len(src) == 0 { if e.o.fullZero { // Add frame header. @@ -491,13 +520,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { } return dst } - e.init.Do(e.initialize) - enc := <-e.encoders - defer func() { - // Release encoder reference to last block. - // If a non-single block is needed the encoder will reset again. - e.encoders <- enc - }() + // Use single segments when above minimum window and below window size. single := len(src) <= e.o.windowSize && len(src) > MinWindowSize if e.o.single != nil { diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index 53e160f7e..e47af66e7 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -146,7 +146,9 @@ func (d *frameDec) reset(br byteBuffer) error { } return err } - printf("raw: %x, mantissa: %d, exponent: %d\n", wd, wd&7, wd>>3) + if debugDecoder { + printf("raw: %x, mantissa: %d, exponent: %d\n", wd, wd&7, wd>>3) + } windowLog := 10 + (wd >> 3) windowBase := uint64(1) << windowLog windowAdd := (windowBase / 8) * uint64(wd&0x7) diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go index 8adabd828..c59f17e07 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go @@ -146,7 +146,7 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) default: - return true, fmt.Errorf("sequenceDecs_decode returned erronous code %d", errCode) + return true, fmt.Errorf("sequenceDecs_decode returned erroneous code %d", errCode) } s.seqSize += ctx.litRemain @@ -292,7 +292,7 @@ func (s *sequenceDecs) decode(seqs []seqVals) error { return io.ErrUnexpectedEOF } - return fmt.Errorf("sequenceDecs_decode_amd64 returned erronous code %d", errCode) + return fmt.Errorf("sequenceDecs_decode_amd64 returned erroneous code %d", errCode) } if ctx.litRemain < 0 { diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s index 5b06174b8..f5591fa1e 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -1814,7 +1814,7 @@ TEXT ·sequenceDecs_decodeSync_amd64(SB), $64-32 MOVQ 40(SP), AX ADDQ AX, 48(SP) - // Calculate poiter to s.out[cap(s.out)] (a past-end pointer) + // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) ADDQ R10, 32(SP) // outBase += outPosition @@ -2376,7 +2376,7 @@ TEXT ·sequenceDecs_decodeSync_bmi2(SB), $64-32 MOVQ 40(SP), CX ADDQ CX, 48(SP) - // Calculate poiter to s.out[cap(s.out)] (a past-end pointer) + // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) ADDQ R9, 32(SP) // outBase += outPosition @@ -2896,7 +2896,7 @@ TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $64-32 MOVQ 40(SP), AX ADDQ AX, 48(SP) - // Calculate poiter to s.out[cap(s.out)] (a past-end pointer) + // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) ADDQ R10, 32(SP) // outBase += outPosition @@ -3560,7 +3560,7 @@ TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $64-32 MOVQ 40(SP), CX ADDQ CX, 48(SP) - // Calculate poiter to s.out[cap(s.out)] (a past-end pointer) + // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) ADDQ R9, 32(SP) // outBase += outPosition diff --git a/vendor/github.com/klauspost/compress/zstd/zstd.go b/vendor/github.com/klauspost/compress/zstd/zstd.go index 4be7cc736..066bef2a4 100644 --- a/vendor/github.com/klauspost/compress/zstd/zstd.go +++ b/vendor/github.com/klauspost/compress/zstd/zstd.go @@ -88,6 +88,10 @@ var ( // Close has been called. ErrDecoderClosed = errors.New("decoder used after Close") + // ErrEncoderClosed will be returned if the Encoder was used after + // Close has been called. + ErrEncoderClosed = errors.New("encoder used after Close") + // ErrDecoderNilInput is returned when a nil Reader was provided // and an operation other than Reset/DecodeAll/Close was attempted. ErrDecoderNilInput = errors.New("nil input provided as reader") diff --git a/vendor/github.com/regclient/regclient/.osv-scanner.toml b/vendor/github.com/regclient/regclient/.osv-scanner.toml index 8380d3b7b..a4acf069a 100644 --- a/vendor/github.com/regclient/regclient/.osv-scanner.toml +++ b/vendor/github.com/regclient/regclient/.osv-scanner.toml @@ -1 +1 @@ -GoVersionOverride = "1.22.5" +GoVersionOverride = "1.23.2" diff --git a/vendor/github.com/regclient/regclient/.version-bump.lock b/vendor/github.com/regclient/regclient/.version-bump.lock index abeb05306..625cd5436 100644 --- a/vendor/github.com/regclient/regclient/.version-bump.lock +++ b/vendor/github.com/regclient/regclient/.version-bump.lock @@ -1,164 +1,52 @@ -{"name":"gha-uses-semver","key":"actions/checkout","version":"v4.1.7"} -{"name":"gha-uses-semver","key":"actions/setup-go","version":"v5.0.2"} -{"name":"gha-uses-semver","key":"actions/stale","version":"v9.0.0"} -{"name":"gha-uses-semver","key":"actions/upload-artifact","version":"v4.3.5"} -{"name":"gha-uses-semver","key":"anchore/sbom-action","version":"v0.17.0"} -{"name":"gha-uses-semver","key":"docker/build-push-action","version":"v6.5.0"} -{"name":"gha-uses-semver","key":"docker/login-action","version":"v3.3.0"} -{"name":"gha-uses-semver","key":"docker/setup-buildx-action","version":"v3.6.1"} -{"name":"gha-uses-semver","key":"fgrosse/go-coverage-report","version":"v1.0.0"} -{"name":"gha-uses-semver","key":"github/codeql-action","version":"v3.25.15"} -{"name":"gha-uses-semver","key":"ossf/scorecard-action","version":"v2.4.0"} -{"name":"gha-uses-semver","key":"sigstore/cosign-installer","version":"v3.5.0"} -{"name":"gha-uses-semver","key":"softprops/action-gh-release","version":"v2.0.8"} -{"name":"git-commit","key":"https://github.com/GoogleCloudPlatform/docker-credential-gcr.git:master","version":"62afb2723512fd6572c6300024e0c94f734b0ae3"} -{"name":"git-commit","key":"https://github.com/awslabs/amazon-ecr-credential-helper.git:main","version":"a8d7d3c42ca114b5accc1b373aad865f075ca3c1"} -{"name":"git-commit","key":"https://github.com/grafi-tt/lunajson.git:master","version":"3d10600874527d71519b33ecbb314eb93ccd1df6"} -{"name":"git-commit","key":"https://github.com/kikito/semver.lua.git:master","version":"af495adc857d51fd1507a112be18523828a1da0d"} -{"name":"git-tag-semver","key":"github.com/GoogleCloudPlatform/docker-credential-gcr","version":"v2.1.23"} -{"name":"git-tag-semver","key":"github.com/dominikh/go-tools","version":"v0.4.7"} -{"name":"git-tag-semver","key":"github.com/google/osv-scanner","version":"v1.8.2"} -{"name":"git-tag-semver","key":"github.com/icholy/gomajor","version":"v0.13.1"} -{"name":"git-tag-semver","key":"github.com/securego/gosec","version":"v2.20.0"} -{"name":"git-tag-semver","key":"github.com/sigstore/cosign","version":"v2.3.0"} -{"name":"git-tag-semver","key":"go.googlesource.com/vuln","version":"v1.1.3"} -{"name":"github-commit-match","key":"actions/checkout:v4.1.1","version":"b4ffde65f46336ab88eb53be808477a3936bae11"} -{"name":"github-commit-match","key":"actions/checkout:v4.1.2","version":"9bb56186c3b09b4f86b1c65136769dd318469633"} -{"name":"github-commit-match","key":"actions/checkout:v4.1.4","version":"0ad4b8fadaa221de15dcec353f45205ec38ea70b"} -{"name":"github-commit-match","key":"actions/checkout:v4.1.5","version":"44c2b7a8a4ea60a981eaca3cf939b5f4305c123b"} -{"name":"github-commit-match","key":"actions/checkout:v4.1.6","version":"a5ac7e51b41094c92402da3b24376905380afc29"} -{"name":"github-commit-match","key":"actions/checkout:v4.1.7","version":"692973e3d937129bcbf40652eb9f2f61becf3332"} -{"name":"github-commit-match","key":"actions/setup-go:v4.1.0","version":"93397bea11091df50f3d7e59dc26a7711a8bcfbe"} -{"name":"github-commit-match","key":"actions/setup-go:v5.0.0","version":"0c52d547c9bc32b1aa3301fd7a9cb496313a4491"} -{"name":"github-commit-match","key":"actions/setup-go:v5.0.1","version":"cdcb36043654635271a94b9a6d1392de5bb323a7"} -{"name":"github-commit-match","key":"actions/setup-go:v5.0.2","version":"0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32"} -{"name":"github-commit-match","key":"actions/stale:v8.0.0","version":"1160a2240286f5da8ec72b1c0816ce2481aabf84"} -{"name":"github-commit-match","key":"actions/stale:v9.0.0","version":"28ca1036281a5e5922ead5184a1bbf96e5fc984e"} -{"name":"github-commit-match","key":"actions/upload-artifact:v3.1.3","version":"a8a3f3ad30e3422c9c7b888a15615d19a852ae32"} -{"name":"github-commit-match","key":"actions/upload-artifact:v4.0.0","version":"c7d193f32edcb7bfad88892161225aeda64e9392"} -{"name":"github-commit-match","key":"actions/upload-artifact:v4.1.0","version":"1eb3cb2b3e0f29609092a73eb033bb759a334595"} -{"name":"github-commit-match","key":"actions/upload-artifact:v4.3.0","version":"26f96dfa697d77e81fd5907df203aa23a56210a8"} -{"name":"github-commit-match","key":"actions/upload-artifact:v4.3.1","version":"5d5d22a31266ced268874388b861e4b58bb5c2f3"} -{"name":"github-commit-match","key":"actions/upload-artifact:v4.3.3","version":"65462800fd760344b1a7b4382951275a0abb4808"} -{"name":"github-commit-match","key":"actions/upload-artifact:v4.3.4","version":"0b2256b8c012f0828dc542b3febcab082c67f72b"} -{"name":"github-commit-match","key":"actions/upload-artifact:v4.3.5","version":"89ef406dd8d7e03cfd12d9e0a4a378f454709029"} -{"name":"github-commit-match","key":"anchore/sbom-action:v0.14.3","version":"78fc58e266e87a38d4194b2137a3d4e9bcaf7ca1"} -{"name":"github-commit-match","key":"anchore/sbom-action:v0.15.0","version":"fd74a6fb98a204a1ad35bbfae0122c1a302ff88b"} -{"name":"github-commit-match","key":"anchore/sbom-action:v0.15.1","version":"5ecf649a417b8ae17dc8383dc32d46c03f2312df"} -{"name":"github-commit-match","key":"anchore/sbom-action:v0.15.10","version":"ab5d7b5f48981941c4c5d6bf33aeb98fe3bae38c"} -{"name":"github-commit-match","key":"anchore/sbom-action:v0.15.11","version":"7ccf588e3cf3cc2611714c2eeae48550fbc17552"} -{"name":"github-commit-match","key":"anchore/sbom-action:v0.15.3","version":"c7f031d9249a826a082ea14c79d3b686a51d485a"} -{"name":"github-commit-match","key":"anchore/sbom-action:v0.15.5","version":"24b0d5238516480139aa8bc6f92eeb7b54a9eb0a"} -{"name":"github-commit-match","key":"anchore/sbom-action:v0.15.8","version":"b6a39da80722a2cb0ef5d197531764a89b5d48c3"} -{"name":"github-commit-match","key":"anchore/sbom-action:v0.15.9","version":"9fece9e20048ca9590af301449208b2b8861333b"} -{"name":"github-commit-match","key":"anchore/sbom-action:v0.16.0","version":"e8d2a6937ecead383dfe75190d104edd1f9c5751"} -{"name":"github-commit-match","key":"anchore/sbom-action:v0.16.1","version":"95b086ac308035dc0850b3853be5b7ab108236a8"} -{"name":"github-commit-match","key":"anchore/sbom-action:v0.17.0","version":"d94f46e13c6c62f59525ac9a1e147a99dc0b9bf5"} -{"name":"github-commit-match","key":"docker/build-push-action:v5.1.0","version":"4a13e500e55cf31b7a5d59a38ab2040ab0f42f56"} -{"name":"github-commit-match","key":"docker/build-push-action:v5.2.0","version":"af5a7ed5ba88268d5278f7203fb52cd833f66d6e"} -{"name":"github-commit-match","key":"docker/build-push-action:v5.3.0","version":"2cdde995de11925a030ce8070c3d77a52ffcf1c0"} -{"name":"github-commit-match","key":"docker/build-push-action:v6.1.0","version":"31159d49c0d4756269a0940a750801a1ea5d7003"} -{"name":"github-commit-match","key":"docker/build-push-action:v6.3.0","version":"1a162644f9a7e87d8f4b053101d1d9a712edc18c"} -{"name":"github-commit-match","key":"docker/build-push-action:v6.5.0","version":"5176d81f87c23d6fc96624dfdbcd9f3830bbe445"} -{"name":"github-commit-match","key":"docker/login-action:v3.0.0","version":"343f7c4344506bcbf9b4de18042ae17996df046d"} -{"name":"github-commit-match","key":"docker/login-action:v3.1.0","version":"e92390c5fb421da1463c202d546fed0ec5c39f20"} -{"name":"github-commit-match","key":"docker/login-action:v3.2.0","version":"0d4c9c5ea7693da7b068278f7b52bda2a190a446"} -{"name":"github-commit-match","key":"docker/login-action:v3.3.0","version":"9780b0c442fbb1117ed29e0efdff1e18412f7567"} -{"name":"github-commit-match","key":"docker/setup-buildx-action:v3.0.0","version":"f95db51fddba0c2d1ec667646a06c2ce06100226"} -{"name":"github-commit-match","key":"docker/setup-buildx-action:v3.1.0","version":"0d103c3126aa41d772a8362f6aa67afac040f80c"} -{"name":"github-commit-match","key":"docker/setup-buildx-action:v3.2.0","version":"2b51285047da1547ffb1b2203d8be4c0af6b1f20"} -{"name":"github-commit-match","key":"docker/setup-buildx-action:v3.3.0","version":"d70bba72b1f3fd22344832f00baa16ece964efeb"} -{"name":"github-commit-match","key":"docker/setup-buildx-action:v3.4.0","version":"4fd812986e6c8c2a69e18311145f9371337f27d4"} -{"name":"github-commit-match","key":"docker/setup-buildx-action:v3.5.0","version":"aa33708b10e362ff993539393ff100fa93ed6a27"} -{"name":"github-commit-match","key":"docker/setup-buildx-action:v3.6.1","version":"988b5a0280414f521da01fcc63a27aeeb4b104db"} -{"name":"github-commit-match","key":"fgrosse/go-coverage-report:v1.0.0","version":"a284a4c69b3383da62629009e9e8e58976efbf6a"} -{"name":"github-commit-match","key":"github/codeql-action:v2.22.7","version":"66b90a5db151a8042fa97405c6cf843bbe433f7b"} -{"name":"github-commit-match","key":"github/codeql-action:v2.22.8","version":"407ffafae6a767df3e0230c3df91b6443ae8df75"} -{"name":"github-commit-match","key":"github/codeql-action:v2.22.9","version":"c0d1daa7f7e14667747d73a7dbbe8c074bc8bfe2"} -{"name":"github-commit-match","key":"github/codeql-action:v3.22.11","version":"b374143c1149a9115d881581d29b8390bbcbb59c"} -{"name":"github-commit-match","key":"github/codeql-action:v3.22.12","version":"012739e5082ff0c22ca6d6ab32e07c36df03c4a4"} -{"name":"github-commit-match","key":"github/codeql-action:v3.23.0","version":"e5f05b81d5b6ff8cfa111c80c22c5fd02a384118"} -{"name":"github-commit-match","key":"github/codeql-action:v3.23.1","version":"0b21cf2492b6b02c465a3e5d7c473717ad7721ba"} -{"name":"github-commit-match","key":"github/codeql-action:v3.23.2","version":"b7bf0a3ed3ecfa44160715d7c442788f65f0f923"} -{"name":"github-commit-match","key":"github/codeql-action:v3.24.0","version":"e8893c57a1f3a2b659b6b55564fdfdbbd2982911"} -{"name":"github-commit-match","key":"github/codeql-action:v3.24.10","version":"4355270be187e1b672a7a1c7c7bae5afdc1ab94a"} -{"name":"github-commit-match","key":"github/codeql-action:v3.24.3","version":"379614612a29c9e28f31f39a59013eb8012a51f0"} -{"name":"github-commit-match","key":"github/codeql-action:v3.24.5","version":"47b3d888fe66b639e431abf22ebca059152f1eea"} -{"name":"github-commit-match","key":"github/codeql-action:v3.24.6","version":"8a470fddafa5cbb6266ee11b37ef4d8aae19c571"} -{"name":"github-commit-match","key":"github/codeql-action:v3.24.8","version":"05963f47d870e2cb19a537396c1f668a348c7d8f"} -{"name":"github-commit-match","key":"github/codeql-action:v3.24.9","version":"1b1aada464948af03b950897e5eb522f92603cc2"} -{"name":"github-commit-match","key":"github/codeql-action:v3.25.0","version":"df5a14dc28094dc936e103b37d749c6628682b60"} -{"name":"github-commit-match","key":"github/codeql-action:v3.25.10","version":"23acc5c183826b7a8a97bce3cecc52db901f8251"} -{"name":"github-commit-match","key":"github/codeql-action:v3.25.11","version":"b611370bb5703a7efb587f9d136a52ea24c5c38c"} -{"name":"github-commit-match","key":"github/codeql-action:v3.25.12","version":"4fa2a7953630fd2f3fb380f21be14ede0169dd4f"} -{"name":"github-commit-match","key":"github/codeql-action:v3.25.15","version":"afb54ba388a7dca6ecae48f608c4ff05ff4cc77a"} -{"name":"github-commit-match","key":"github/codeql-action:v3.25.3","version":"d39d31e687223d841ef683f52467bd88e9b21c14"} -{"name":"github-commit-match","key":"github/codeql-action:v3.25.4","version":"ccf74c947955fd1cf117aef6a0e4e66191ef6f61"} -{"name":"github-commit-match","key":"github/codeql-action:v3.25.5","version":"b7cec7526559c32f1616476ff32d17ba4c59b2d6"} -{"name":"github-commit-match","key":"github/codeql-action:v3.25.6","version":"9fdb3e49720b44c48891d036bb502feb25684276"} -{"name":"github-commit-match","key":"github/codeql-action:v3.25.7","version":"f079b8493333aace61c81488f8bd40919487bd9f"} -{"name":"github-commit-match","key":"github/codeql-action:v3.25.8","version":"2e230e8fe0ad3a14a340ad0815ddb96d599d2aff"} -{"name":"github-commit-match","key":"ossf/scorecard-action:v2.3.1","version":"0864cf19026789058feabb7e87baa5f140aac736"} -{"name":"github-commit-match","key":"ossf/scorecard-action:v2.3.3","version":"dc50aa9510b46c811795eb24b2f1ba02a914e534"} -{"name":"github-commit-match","key":"ossf/scorecard-action:v2.4.0","version":"62b2cac7ed8198b15735ed49ab1e5cf35480ba46"} -{"name":"github-commit-match","key":"regclient/actions:main","version":"35bc5829dd3d37ace2717971f3151894b43bfabc"} -{"name":"github-commit-match","key":"sigstore/cosign-installer:v3.2.0","version":"1fc5bd396d372bee37d608f955b336615edf79c8"} -{"name":"github-commit-match","key":"sigstore/cosign-installer:v3.3.0","version":"9614fae9e5c5eddabb09f90a270fcb487c9f7149"} -{"name":"github-commit-match","key":"sigstore/cosign-installer:v3.4.0","version":"e1523de7571e31dbe865fd2e80c5c7c23ae71eb4"} -{"name":"github-commit-match","key":"sigstore/cosign-installer:v3.5.0","version":"59acb6260d9c0ba8f4a2f9d9b48431a222b68e20"} -{"name":"github-commit-match","key":"softprops/action-gh-release:v0.1.15","version":"de2c0eb89ae2a093876385947365aca7b0e5f844"} -{"name":"github-commit-match","key":"softprops/action-gh-release:v2.0.3","version":"3198ee18f814cdf787321b4a32a26ddbf37acc52"} -{"name":"github-commit-match","key":"softprops/action-gh-release:v2.0.4","version":"9d7c94cfd0a1f3ed45544c887983e9fa900f0564"} -{"name":"github-commit-match","key":"softprops/action-gh-release:v2.0.5","version":"69320dbe05506a9a39fc8ae11030b214ec2d1f87"} -{"name":"github-commit-match","key":"softprops/action-gh-release:v2.0.6","version":"a74c6b72af54cfa997e81df42d94703d6313a2d0"} -{"name":"github-commit-match","key":"softprops/action-gh-release:v2.0.8","version":"c062e08bd532815e2082a85e87e3ef29c3e6d191"} -{"name":"registry-digest-arg","key":"docker.io/library/alpine:3.18.4","version":"sha256:eece025e432126ce23f223450a0326fbebde39cdf496a85d8c016293fc851978"} -{"name":"registry-digest-arg","key":"docker.io/library/alpine:3.18.5","version":"sha256:34871e7290500828b39e22294660bee86d966bc0017544e848dd9a255cdf59e0"} -{"name":"registry-digest-arg","key":"docker.io/library/alpine:3.19.0","version":"sha256:51b67269f354137895d43f3b3d810bfacd3945438e94dc5ac55fdac340352f48"} -{"name":"registry-digest-arg","key":"docker.io/library/alpine:3.19.1","version":"sha256:c5b1261d6d3e43071626931fc004f70149baeba2c8ec672bd4f27761f8e1ad6b"} -{"name":"registry-digest-arg","key":"docker.io/library/golang:1.21.4-alpine","version":"sha256:70afe55365a265f0762257550bc38440e0d6d6b97020d3f8c85328f00200dd8e"} -{"name":"registry-digest-arg","key":"docker.io/library/golang:1.21.5-alpine","version":"sha256:4db4aac30880b978cae5445dd4a706215249ad4f43d28bd7cdf7906e9be8dd6b"} -{"name":"registry-digest-arg","key":"docker.io/library/golang:1.21.6-alpine","version":"sha256:a6a7f1fcf12f5efa9e04b1e75020931a616cd707f14f62ab5262bfbe109aa84a"} -{"name":"registry-digest-arg","key":"docker.io/library/golang:1.22.0-alpine","version":"sha256:8e96e6cff6a388c2f70f5f662b64120941fcd7d4b89d62fec87520323a316bd9"} -{"name":"registry-digest-arg-match","key":"docker.io/library/alpine:3.19.1","version":"sha256:c5b1261d6d3e43071626931fc004f70149baeba2c8ec672bd4f27761f8e1ad6b"} -{"name":"registry-digest-arg-match","key":"docker.io/library/alpine:3.20.0","version":"sha256:77726ef6b57ddf65bb551896826ec38bc3e53f75cdde31354fbffb4f25238ebd"} -{"name":"registry-digest-arg-match","key":"docker.io/library/alpine:3.20.1","version":"sha256:b89d9c93e9ed3597455c90a0b88a8bbb5cb7188438f70953fede212a0c4394e0"} -{"name":"registry-digest-arg-match","key":"docker.io/library/alpine:3.20.2","version":"sha256:0a4eaa0eecf5f8c050e5bba433f58c052be7587ee8af3e8b3910ef9ab5fbe9f5"} -{"name":"registry-digest-arg-match","key":"docker.io/library/golang:1.22.0-alpine","version":"sha256:8e96e6cff6a388c2f70f5f662b64120941fcd7d4b89d62fec87520323a316bd9"} -{"name":"registry-digest-arg-match","key":"docker.io/library/golang:1.22.1-alpine","version":"sha256:0466223b8544fb7d4ff04748acc4d75a608234bf4e79563bff208d2060c0dd79"} -{"name":"registry-digest-arg-match","key":"docker.io/library/golang:1.22.2-alpine","version":"sha256:cdc86d9f363e8786845bea2040312b4efa321b828acdeb26f393faa864d887b0"} -{"name":"registry-digest-arg-match","key":"docker.io/library/golang:1.22.3-alpine","version":"sha256:7e788330fa9ae95c68784153b7fd5d5076c79af47651e992a3cdeceeb5dd1df0"} -{"name":"registry-digest-arg-match","key":"docker.io/library/golang:1.22.4-alpine","version":"sha256:ace6cc3fe58d0c7b12303c57afe6d6724851152df55e08057b43990b927ad5e8"} -{"name":"registry-digest-arg-match","key":"docker.io/library/golang:1.22.5-alpine","version":"sha256:0d3653dd6f35159ec6e3d10263a42372f6f194c3dea0b35235d72aabde86486e"} -{"name":"registry-digest-match","key":"anchore/syft:v0.100.0","version":"sha256:df7b07bfadff45e0135d74f22478f47b16ac6aff4e8dbd93133fcae3bbbb790d"} -{"name":"registry-digest-match","key":"anchore/syft:v0.101.1","version":"sha256:ea3df0f0d7b85b352911f2b4a06fd08e2f19cdc793812c8766f7813b6a1e98cb"} -{"name":"registry-digest-match","key":"anchore/syft:v0.102.0","version":"sha256:fbbdb5f60a9db9400d49801bf70b19c29ac054b370dbccf538399918bbdf38a7"} -{"name":"registry-digest-match","key":"anchore/syft:v0.103.1","version":"sha256:96c56b2554079d90e5fc8999278638ecca6454713fceebd26321d0698975b243"} -{"name":"registry-digest-match","key":"anchore/syft:v0.104.0","version":"sha256:aeb052d18121587de9138ca715fc3207ef86358a56bd49ebd2d3ba27169c09d1"} -{"name":"registry-digest-match","key":"anchore/syft:v0.105.0","version":"sha256:e3dbedff17aaec7d06c6509fa42c6454ee2ed346299606fce6589096dc9efd70"} -{"name":"registry-digest-match","key":"anchore/syft:v0.97.1","version":"sha256:abc8d4310c54b56dd1e789d5f60b8ebc43f472652b34971d4b0d0dbed7f4ebda"} -{"name":"registry-digest-match","key":"anchore/syft:v0.98.0","version":"sha256:b353bf516310fcbc86676bb20849929298034e80f15873e63da18acdf7080b4e"} -{"name":"registry-digest-match","key":"anchore/syft:v0.99.0","version":"sha256:07d598b6a95280ed6ecc128685192173a00f370b5326cf50c62500d559075e1d"} -{"name":"registry-digest-match","key":"anchore/syft:v1.0.1","version":"sha256:d49defada853900861d55491ba549ab334148d51b11f23942abecb39ea83d4db"} -{"name":"registry-digest-match","key":"anchore/syft:v1.1.0","version":"sha256:878a95c76e139fdcdf58aa14ba7594ad41971dfc834245501347276b600aa81e"} -{"name":"registry-digest-match","key":"anchore/syft:v1.1.1","version":"sha256:201752e0aa1233f7e2fbc478e3c345ac98b1613a6e0005fb26c8c3ee84d3809c"} -{"name":"registry-digest-match","key":"anchore/syft:v1.10.0","version":"sha256:4243162c3ac33d107a8d9981e2d41b7888b66f12d9bd547124644391be796763"} -{"name":"registry-digest-match","key":"anchore/syft:v1.2.0","version":"sha256:6e70eb6e34380ae2e9397f7dbe1b0e1e329a53e71b18fc3b1d2089e367fc114a"} -{"name":"registry-digest-match","key":"anchore/syft:v1.3.0","version":"sha256:93384e4f46c62cc827960f0e3323516576590811d246dd97556ebbf71112db49"} -{"name":"registry-digest-match","key":"anchore/syft:v1.4.1","version":"sha256:24feb76496d558c52a09a859de569fc71cb147d9aff01edab885accae5363150"} -{"name":"registry-digest-match","key":"anchore/syft:v1.5.0","version":"sha256:7e622b5d92a6ec0727fb4bd48046b644f459c33b54e9ea9025a764d324177cd2"} -{"name":"registry-digest-match","key":"anchore/syft:v1.7.0","version":"sha256:35372cf4e44776927f9222f620ab7dcfb78fa422685d02e10240255feee3c00c"} -{"name":"registry-digest-match","key":"anchore/syft:v1.8.0","version":"sha256:37552ed2e74c0b7c944501d901110076505984c312c82c06633259d492ea87e1"} -{"name":"registry-digest-match","key":"anchore/syft:v1.9.0","version":"sha256:205d527263647054b457a0622b27aa8e584351c9bae66846bbaba1e4fb0e6561"} -{"name":"registry-golang-latest","key":"golang-latest","version":"1.22"} -{"name":"registry-golang-matrix","key":"golang-matrix","version":"[\"1.20\", \"1.21\", \"1.22\"]"} -{"name":"registry-golang-oldest","key":"golang-oldest","version":"1.20"} -{"name":"registry-tag-arg-semver","key":"anchore/syft","version":"v1.10.0"} -{"name":"registry-tag-arg-semver","key":"davidanson/markdownlint-cli2","version":"v0.13.0"} -{"name":"registry-tag-arg-semver","key":"docker.io/library/alpine","version":"3.20.2"} -{"name":"registry-tag-arg-semver","key":"docker.io/library/golang","version":"1.22.5"} -{"name":"registry-tag-arg-semver","key":"docker.io/library/registry","version":"2.8.3"} -{"name":"registry-tag-arg-semver","key":"ghcr.io/igorshubovych/markdownlint-cli","version":"v0.35.0"} -{"name":"registry-tag-arg-semver","key":"ghcr.io/project-zot/zot-linux-amd64","version":"v2.1.0"} -{"name":"registry-tag-arg-semver-major","key":"docker.io/library/alpine","version":"3"} -{"name":"registry-tag-match-semver","key":"anchore/syft","version":"v1.10.0"} +{"name":"docker-arg-alpine-digest","key":"docker.io/library/alpine:3.20.3","version":"sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d"} +{"name":"docker-arg-alpine-tag","key":"docker.io/library/alpine","version":"3.20.3"} +{"name":"docker-arg-ecr","key":"https://github.com/awslabs/amazon-ecr-credential-helper.git:main","version":"fb978450068958e4e5874e36a3cc2b69798132f9"} +{"name":"docker-arg-gcr","key":"https://github.com/GoogleCloudPlatform/docker-credential-gcr.git","version":"v2.1.25"} +{"name":"docker-arg-go-digest","key":"docker.io/library/golang:1.23.2-alpine","version":"sha256:9dd2625a1ff2859b8d8b01d8f7822c0f528942fe56cfe7a1e7c38d3b8d72d679"} +{"name":"docker-arg-go-tag","key":"docker.io/library/golang","version":"1.23.2"} +{"name":"docker-arg-lunajson","key":"https://github.com/grafi-tt/lunajson.git:master","version":"3d10600874527d71519b33ecbb314eb93ccd1df6"} +{"name":"docker-arg-semver","key":"https://github.com/kikito/semver.lua.git:master","version":"af495adc857d51fd1507a112be18523828a1da0d"} +{"name":"gha-alpine-digest","key":"docker.io/library/alpine:3.20.3","version":"sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d"} +{"name":"gha-alpine-tag-base","key":"docker.io/library/alpine","version":"3"} +{"name":"gha-alpine-tag-comment","key":"docker.io/library/alpine","version":"3.20.3"} +{"name":"gha-cosign-version","key":"https://github.com/sigstore/cosign.git","version":"v2.4.1"} +{"name":"gha-golang-matrix","key":"golang-matrix","version":"[\"1.21\", \"1.22\", \"1.23\"]"} +{"name":"gha-golang-release","key":"golang-latest","version":"1.23"} +{"name":"gha-syft-version","key":"docker.io/anchore/syft","version":"v1.15.0"} +{"name":"gha-uses-commit","key":"https://github.com/actions/checkout.git:v4.2.2","version":"11bd71901bbe5b1630ceea73d27597364c9af683"} +{"name":"gha-uses-commit","key":"https://github.com/actions/setup-go.git:v5.1.0","version":"41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed"} +{"name":"gha-uses-commit","key":"https://github.com/actions/stale.git:v9.0.0","version":"28ca1036281a5e5922ead5184a1bbf96e5fc984e"} +{"name":"gha-uses-commit","key":"https://github.com/actions/upload-artifact.git:v4.4.3","version":"b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882"} +{"name":"gha-uses-commit","key":"https://github.com/anchore/sbom-action.git:v0.17.6","version":"251a468eed47e5082b105c3ba6ee500c0e65a764"} +{"name":"gha-uses-commit","key":"https://github.com/docker/build-push-action.git:v6.9.0","version":"4f58ea79222b3b9dc2c8bbdd6debcef730109a75"} +{"name":"gha-uses-commit","key":"https://github.com/docker/login-action.git:v3.3.0","version":"9780b0c442fbb1117ed29e0efdff1e18412f7567"} +{"name":"gha-uses-commit","key":"https://github.com/docker/setup-buildx-action.git:v3.7.1","version":"c47758b77c9736f4b2ef4073d4d51994fabfe349"} +{"name":"gha-uses-commit","key":"https://github.com/regclient/actions.git:main","version":"ce5fd131e371ffcdd7508b478cb223b3511a9183"} +{"name":"gha-uses-commit","key":"https://github.com/sigstore/cosign-installer.git:v3.7.0","version":"dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da"} +{"name":"gha-uses-commit","key":"https://github.com/softprops/action-gh-release.git:v2.0.9","version":"e7a8f85e1c67a31e6ed99a94b41bd0b71bbee6b8"} +{"name":"gha-uses-semver","key":"https://github.com/actions/checkout.git","version":"v4.2.2"} +{"name":"gha-uses-semver","key":"https://github.com/actions/setup-go.git","version":"v5.1.0"} +{"name":"gha-uses-semver","key":"https://github.com/actions/stale.git","version":"v9.0.0"} +{"name":"gha-uses-semver","key":"https://github.com/actions/upload-artifact.git","version":"v4.4.3"} +{"name":"gha-uses-semver","key":"https://github.com/anchore/sbom-action.git","version":"v0.17.6"} +{"name":"gha-uses-semver","key":"https://github.com/docker/build-push-action.git","version":"v6.9.0"} +{"name":"gha-uses-semver","key":"https://github.com/docker/login-action.git","version":"v3.3.0"} +{"name":"gha-uses-semver","key":"https://github.com/docker/setup-buildx-action.git","version":"v3.7.1"} +{"name":"gha-uses-semver","key":"https://github.com/sigstore/cosign-installer.git","version":"v3.7.0"} +{"name":"gha-uses-semver","key":"https://github.com/softprops/action-gh-release.git","version":"v2.0.9"} +{"name":"go-mod-golang-release","key":"golang-oldest","version":"1.21"} +{"name":"makefile-ci-distribution","key":"docker.io/library/registry","version":"2.8.3"} +{"name":"makefile-ci-zot","key":"ghcr.io/project-zot/zot-linux-amd64","version":"v2.1.1"} +{"name":"makefile-go-vulncheck","key":"https://go.googlesource.com/vuln.git","version":"v1.1.3"} +{"name":"makefile-gomajor","key":"https://github.com/icholy/gomajor.git","version":"v0.14.0"} +{"name":"makefile-gosec","key":"https://github.com/securego/gosec.git","version":"v2.21.4"} +{"name":"makefile-markdown-lint","key":"docker.io/davidanson/markdownlint-cli2","version":"v0.14.0"} +{"name":"makefile-osv-scanner","key":"https://github.com/google/osv-scanner.git","version":"v1.9.1"} +{"name":"makefile-staticcheck","key":"https://github.com/dominikh/go-tools.git","version":"v0.5.1"} +{"name":"makefile-syft-container-digest","key":"anchore/syft:v1.15.0","version":"sha256:92b229ac1d84cd9627624a951e26a78333b26a5f34c9999629ba96e90751c971"} +{"name":"makefile-syft-container-tag","key":"anchore/syft","version":"v1.15.0"} +{"name":"makefile-syft-version","key":"docker.io/anchore/syft","version":"v1.15.0"} +{"name":"osv-golang-release","key":"docker.io/library/golang","version":"1.23.2"} +{"name":"shell-alpine-digest","key":"docker.io/library/alpine:3.20.3","version":"sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d"} +{"name":"shell-alpine-tag-base","key":"docker.io/library/alpine","version":"3"} +{"name":"shell-alpine-tag-comment","key":"docker.io/library/alpine","version":"3.20.3"} diff --git a/vendor/github.com/regclient/regclient/.version-bump.yml b/vendor/github.com/regclient/regclient/.version-bump.yml index 597f2446c..e7421d5fd 100644 --- a/vendor/github.com/regclient/regclient/.version-bump.yml +++ b/vendor/github.com/regclient/regclient/.version-bump.yml @@ -1,21 +1,21 @@ files: "build/Dockerfile*": - scans: + processors: - docker-arg-alpine-tag - docker-arg-alpine-digest - docker-arg-go-tag - docker-arg-go-digest - - git-commit-ecr - - git-tag-gcr - - git-commit-lunajson - - git-commit-semver + - docker-arg-ecr + - docker-arg-gcr + - docker-arg-lunajson + - docker-arg-semver "build/oci-image.sh": - scans: - - shell-alpine-tag + processors: + - shell-alpine-tag-base - shell-alpine-tag-comment - shell-alpine-digest ".github/workflows/*.yml": - scans: + processors: - gha-golang-matrix - gha-golang-release - gha-uses-vx @@ -23,11 +23,11 @@ files: - gha-uses-commit - gha-syft-version - gha-cosign-version - - gha-alpine-tag + - gha-alpine-tag-base - gha-alpine-tag-comment - gha-alpine-digest "Makefile": - scans: + processors: - makefile-gomajor - makefile-go-vulncheck - makefile-markdown-lint @@ -35,337 +35,296 @@ files: - makefile-osv-scanner - makefile-staticcheck - makefile-syft-version - - makefile-syft-version2 - - makefile-syft-digest + - makefile-syft-container-tag + - makefile-syft-container-digest - makefile-ci-distribution - makefile-ci-zot "go.mod": - scans: + processors: - go-mod-golang-release ".osv-scanner.toml": - scans: + processors: - osv-golang-release -scans: +x-processor-tmpl: + git-commit: &git-commit + key: "{{ .SourceArgs.url }}:{{ .SourceArgs.ref }}" + scan: "regexp" + source: "git-commit" + filter: + expr: "^{{ .SourceArgs.ref }}$" + git-tag-semver: &git-tag-semver + key: "{{ .SourceArgs.url }}" + scan: "regexp" + source: "git-tag" + filter: + expr: '^v?\d+\.\d+\.\d+$' + sort: + method: "semver" + registry-digest: ®istry-digest + key: "{{ .SourceArgs.image }}" + scan: "regexp" + source: "registry-digest" + registry-tag-semver: ®istry-tag-semver + key: "{{ .SourceArgs.repo }}" + scan: "regexp" + source: "registry-tag" + filter: + expr: '^v?\d+\.\d+\.\d+$' + sort: + method: "semver" + +processors: docker-arg-alpine-tag: - type: "regexp" - source: "registry-tag-arg-semver" - args: - regexp: '^ARG ALPINE_VER=(?P\d+\.\d+\.\d+)@(?Psha256:[0-9a-f]+)\s*$' + <<: *registry-tag-semver + scanArgs: + regexp: '^ARG ALPINE_VER=(?Pv?\d+\.\d+\.\d+)@(?Psha256:[0-9a-f]+)\s*$' + sourceArgs: repo: "docker.io/library/alpine" docker-arg-alpine-digest: - type: "regexp" - source: "registry-digest-arg-match" - args: - regexp: '^ARG ALPINE_VER=(?P\d+\.\d+\.\d+)@(?Psha256:[0-9a-f]+)\s*$' - image: "docker.io/library/alpine" + <<: *registry-digest + scanArgs: + regexp: '^ARG ALPINE_VER=(?Pv?\d+\.\d+\.\d+)@(?Psha256:[0-9a-f]+)\s*$' + sourceArgs: + image: "docker.io/library/alpine:{{.ScanMatch.Tag}}" docker-arg-go-tag: - type: "regexp" - source: "registry-tag-arg-semver" - args: + <<: *registry-tag-semver + scanArgs: regexp: '^ARG GO_VER=(?P[a-z0-9\-\.]+)-alpine@(?Psha256:[0-9a-f]+)\s*$' + sourceArgs: repo: "docker.io/library/golang" docker-arg-go-digest: - type: "regexp" - source: "registry-digest-arg-match" - args: + <<: *registry-digest + scanArgs: regexp: '^ARG GO_VER=(?P[a-z0-9\-\.]+)@(?Psha256:[0-9a-f]+)\s*$' - image: "docker.io/library/golang" - git-commit-ecr: - type: "regexp" - source: "git-commit" - args: + sourceArgs: + image: "docker.io/library/golang:{{.ScanMatch.Tag}}" + docker-arg-ecr: + <<: *git-commit + scanArgs: regexp: '^ARG ECR_HELPER_VER=(?P[0-9a-f]+)\s*$' - repo: "https://github.com/awslabs/amazon-ecr-credential-helper.git" + sourceArgs: + url: "https://github.com/awslabs/amazon-ecr-credential-helper.git" ref: main - git-tag-gcr: - type: "regexp" - source: "git-tag-semver" - args: - regexp: '^ARG GCR_HELPER_VER=(?P[^\s]+)\s*$' - repo: "github.com/GoogleCloudPlatform/docker-credential-gcr" - ref: master - git-commit-lunajson: - type: "regexp" - source: "git-commit" - args: + docker-arg-gcr: + <<: *git-tag-semver + scanArgs: + regexp: '^ARG GCR_HELPER_VER=(?Pv?\d+\.\d+\.\d+)\s*$' + sourceArgs: + url: "https://github.com/GoogleCloudPlatform/docker-credential-gcr.git" + docker-arg-lunajson: + <<: *git-commit + scanArgs: regexp: '^ARG LUNAJSON_COMMIT=(?P[0-9a-f]+)\s*$' - repo: "https://github.com/grafi-tt/lunajson.git" + sourceArgs: + url: "https://github.com/grafi-tt/lunajson.git" ref: master - git-commit-semver: - type: "regexp" - source: "git-commit" - args: + docker-arg-semver: + <<: *git-commit + scanArgs: regexp: '^ARG SEMVER_COMMIT=(?P[0-9a-f]+)\s*$' - repo: "https://github.com/kikito/semver.lua.git" + sourceArgs: + url: "https://github.com/kikito/semver.lua.git" ref: master - gha-uses-vx: - type: "regexp" - source: "gha-uses-vx" - args: - regexp: '^\s+-?\s+uses: (?P[^@/]+/[^@/]+)[^@]*@(?P[0-9a-f]+)\s+#\s+(?Pv\d+)\s*$' - gha-uses-semver: - type: "regexp" - source: "gha-uses-semver" - args: - regexp: '^\s+-?\s+uses: (?P[^@/]+/[^@/]+)[^@]*@(?P[0-9a-f]+)\s+#\s+(?Pv\d+\.\d+\.\d+)\s*$' - gha-uses-commit: - type: "regexp" - source: "github-commit-match" - args: - regexp: '^\s+-?\s+uses: (?P[^@/]+/[^@/]+)[^@]*@(?P[0-9a-f]+)\s+#\s+(?P[\w\d\.]+)\s*$' + + gha-alpine-digest: + <<: *registry-digest + scanArgs: + regexp: '^\s*ALPINE_DIGEST: "(?Psha256:[0-9a-f]+)"\s*#\s*(?P\d+\.\d+\.\d+)\s*$' + sourceArgs: + image: "docker.io/library/alpine:{{ .ScanMatch.Tag }}" + gha-alpine-tag-base: + <<: *registry-tag-semver + scanArgs: + regexp: '^\s*ALPINE_NAME: "alpine:(?Pv?\d+)"\s*$' + sourceArgs: + repo: "docker.io/library/alpine" + # only return the major version number in the tag to support detecting a change in the base image + template: '{{ index ( split .Version "." ) 0 }}' + gha-alpine-tag-comment: + <<: *registry-tag-semver + scanArgs: + regexp: '^\s*ALPINE_DIGEST: "(?Psha256:[0-9a-f]+)"\s*#\s*(?Pv?\d+\.\d+\.\d+)\s*$' + sourceArgs: + repo: "docker.io/library/alpine" + gha-cosign-version: + <<: *git-tag-semver + scanArgs: + regexp: '^\s*cosign-release: "(?Pv?[0-9\.]+)"\s*$' + sourceArgs: + url: "https://github.com/sigstore/cosign.git" gha-golang-matrix: - type: "regexp" - source: "registry-golang-matrix" - args: + <<: *registry-tag-semver + key: "golang-matrix" + scanArgs: regexp: '^\s*gover: (?P\[["0-9, \.]+\])\s*$' + sourceArgs: + repo: "docker.io/library/golang" + filter: + expr: '^v?\d+\.\d+$' + template: '["{{ index .VerMap ( index .VerList 2 ) }}", "{{ index .VerMap ( index .VerList 1 ) }}", "{{ index .VerMap ( index .VerList 0 ) }}"]' gha-golang-release: - type: "regexp" - source: "registry-golang-latest" - args: - regexp: '^\s*RELEASE_GO_VER: "(?P[0-9\.]+)"\s*$' + <<: *registry-tag-semver + key: "golang-latest" + scanArgs: + regexp: '^\s*RELEASE_GO_VER: "(?Pv?[0-9\.]+)"\s*$' + sourceArgs: + repo: "docker.io/library/golang" + filter: + expr: '^v?\d+\.\d+$' gha-syft-version: - type: "regexp" - source: "registry-tag-arg-semver" - args: - regexp: '^\s*syft-version: "(?Pv[0-9\.]+)"\s*$' - repo: "anchore/syft" - gha-cosign-version: - type: "regexp" - source: "git-tag-semver" - args: - regexp: '^\s*cosign-release: "(?Pv[0-9\.]+)"\s*$' - repo: "github.com/sigstore/cosign" - gha-alpine-tag: - type: "regexp" - source: "registry-tag-arg-semver-major" - args: - regexp: '^\s*ALPINE_NAME: "alpine:(?P\d+)"\s*$' - repo: "docker.io/library/alpine" - gha-alpine-tag-comment: - type: "regexp" - source: "registry-tag-arg-semver" - args: - regexp: '^\s*ALPINE_DIGEST: "(?Psha256:[0-9a-f]+)"\s*#\s*(?P\d+\.\d+\.\d+)\s*$' - repo: "docker.io/library/alpine" - gha-alpine-digest: - type: "regexp" - source: "registry-digest-arg-match" - args: - regexp: '^\s*ALPINE_DIGEST: "(?Psha256:[0-9a-f]+)"\s*#\s*(?P\d+\.\d+\.\d+)\s*$' - image: "docker.io/library/alpine" + <<: *registry-tag-semver + scanArgs: + regexp: '^\s*syft-version: "(?Pv?[0-9\.]+)"\s*$' + sourceArgs: + repo: "docker.io/anchore/syft" + gha-uses-vx: + <<: *git-tag-semver + scanArgs: + regexp: '^\s+-?\s+uses: (?P[^@/]+/[^@/]+)[^@]*@(?P[0-9a-f]+)\s+#\s+(?Pv?\d+)\s*$' + sourceArgs: + url: "https://github.com/{{ .ScanMatch.Repo }}.git" + filter: + expr: '^v?\d+$' + gha-uses-semver: + <<: *git-tag-semver + scanArgs: + regexp: '^\s+-?\s+uses: (?P[^@/]+/[^@/]+)[^@]*@(?P[0-9a-f]+)\s+#\s+(?Pv?\d+\.\d+\.\d+)\s*$' + sourceArgs: + url: "https://github.com/{{ .ScanMatch.Repo }}.git" + gha-uses-commit: + <<: *git-commit + scanArgs: + regexp: '^\s+-?\s+uses: (?P[^@/]+/[^@/]+)[^@]*@(?P[0-9a-f]+)\s+#\s+(?P[\w\d\.]+)\s*$' + sourceArgs: + url: "https://github.com/{{ .ScanMatch.Repo }}.git" + ref: "{{ .ScanMatch.Ref }}" + go-mod-golang-release: - type: "regexp" - source: "registry-golang-oldest" - args: + <<: *registry-tag-semver + key: "golang-oldest" + scanArgs: regexp: '^go (?P[0-9\.]+)\s*$' + sourceArgs: + repo: "docker.io/library/golang" + filter: + expr: '^\d+\.\d+$' + template: '{{ index .VerMap ( index .VerList 2 ) }}' + makefile-ci-distribution: - type: "regexp" - source: "registry-tag-arg-semver" - args: + <<: *registry-tag-semver + scanArgs: regexp: '^CI_DISTRIBUTION_VER\?=(?Pv?[0-9\.]+)\s*$' + sourceArgs: repo: "docker.io/library/registry" makefile-ci-zot: - type: "regexp" - source: "registry-tag-arg-semver" - args: + <<: *registry-tag-semver + scanArgs: regexp: '^CI_ZOT_VER\?=(?Pv?[0-9\.]+)\s*$' + sourceArgs: repo: "ghcr.io/project-zot/zot-linux-amd64" makefile-gomajor: - type: "regexp" - source: "git-tag-semver" - args: - regexp: '^GOMAJOR_VER\?=(?Pv[0-9\.]+)\s*$' - repo: "github.com/icholy/gomajor" - makefile-go-vulncheck: - type: "regexp" - source: "git-tag-semver" - args: - regexp: '^GO_VULNCHECK_VER\?=(?Pv[0-9\.]+)\s*$' - repo: "go.googlesource.com/vuln" + <<: *git-tag-semver + scanArgs: + regexp: '^GOMAJOR_VER\?=(?Pv?[0-9\.]+)\s*$' + sourceArgs: + url: "https://github.com/icholy/gomajor.git" makefile-gosec: - type: "regexp" - source: "git-tag-semver" - args: - regexp: '^GOSEC_VER\?=(?Pv[0-9\.]+)\s*$' - repo: "github.com/securego/gosec" + <<: *git-tag-semver + scanArgs: + regexp: '^GOSEC_VER\?=(?Pv?[0-9\.]+)\s*$' + sourceArgs: + url: "https://github.com/securego/gosec.git" + makefile-go-vulncheck: + <<: *git-tag-semver + scanArgs: + regexp: '^GO_VULNCHECK_VER\?=(?Pv?[0-9\.]+)\s*$' + sourceArgs: + url: "https://go.googlesource.com/vuln.git" makefile-markdown-lint: - type: "regexp" - source: "registry-tag-arg-semver" - args: - regexp: '^MARKDOWN_LINT_VER\?=(?Pv[0-9\.]+)\s*$' - repo: "davidanson/markdownlint-cli2" + <<: *registry-tag-semver + scanArgs: + regexp: '^MARKDOWN_LINT_VER\?=(?Pv?[0-9\.]+)\s*$' + sourceArgs: + repo: "docker.io/davidanson/markdownlint-cli2" makefile-osv-scanner: - type: "regexp" - source: "git-tag-semver" - args: - regexp: '^OSV_SCANNER_VER\?=(?Pv[0-9\.]+)\s*$' - repo: "github.com/google/osv-scanner" + <<: *git-tag-semver + scanArgs: + regexp: '^OSV_SCANNER_VER\?=(?Pv?[0-9\.]+)\s*$' + sourceArgs: + url: "https://github.com/google/osv-scanner.git" makefile-staticcheck: - type: "regexp" - source: "git-tag-semver" - args: - regexp: '^STATICCHECK_VER\?=(?Pv[0-9\.]+)\s*$' - repo: "github.com/dominikh/go-tools" + <<: *git-tag-semver + scanArgs: + regexp: '^STATICCHECK_VER\?=(?Pv?[0-9\.]+)\s*$' + sourceArgs: + url: "https://github.com/dominikh/go-tools.git" + filter: + # repo also has dated tags, ignore versions without a preceding "v" + expr: '^v\d+\.\d+\.\d+$' + makefile-syft-container-tag: + <<: *registry-tag-semver + scanArgs: + regexp: '^SYFT_CONTAINER\?=(?P[^:]*):(?Pv?[0-9\.]+)@(?Psha256:[0-9a-f]+)\s*$' + sourceArgs: + repo: "{{ .ScanMatch.Repo }}" + makefile-syft-container-digest: + <<: *registry-digest + scanArgs: + regexp: '^SYFT_CONTAINER\?=(?P[^:]*):(?Pv?[0-9\.]+)@(?Psha256:[0-9a-f]+)\s*$' + sourceArgs: + image: "{{ .ScanMatch.Image }}:{{.ScanMatch.Tag}}" makefile-syft-version: - type: "regexp" - source: "registry-tag-arg-semver" - args: + <<: *registry-tag-semver + scanArgs: regexp: '^SYFT_VERSION\?=(?Pv[0-9\.]+)\s*$' - repo: "anchore/syft" - makefile-syft-version2: - type: "regexp" - source: "registry-tag-match-semver" - args: - regexp: '^SYFT_CONTAINER\?=(?P[^:]*):(?Pv[0-9\.]+)@(?Psha256:[0-9a-f]+)\s*$' - makefile-syft-digest: - type: "regexp" - source: "registry-digest-match" - args: - regexp: '^SYFT_CONTAINER\?=(?P[^:]*):(?Pv[0-9\.]+)@(?Psha256:[0-9a-f]+)\s*$' + sourceArgs: + repo: "docker.io/anchore/syft" + osv-golang-release: - type: "regexp" - source: "registry-tag-arg-semver" - args: - regexp: '^GoVersionOverride = "(?P[0-9\.]+)"\s*$' + <<: *registry-tag-semver + scanArgs: + regexp: '^GoVersionOverride = "(?Pv?[0-9\.]+)"\s*$' + sourceArgs: repo: "docker.io/library/golang" - shell-alpine-tag: - type: "regexp" - source: "registry-tag-arg-semver-major" - args: - regexp: '^\s*ALPINE_NAME="alpine:(?P\d+)"\s*$' + + shell-alpine-tag-base: + <<: *registry-tag-semver + scanArgs: + regexp: '^\s*ALPINE_NAME="alpine:(?Pv?\d+)"\s*$' + sourceArgs: repo: "docker.io/library/alpine" + # only return the major version number in the tag to support detecting a change in the base image + template: '{{ index ( split .Version "." ) 0 }}' shell-alpine-tag-comment: - type: "regexp" - source: "registry-tag-arg-semver" - args: - regexp: '^\s*ALPINE_DIGEST="(?Psha256:[0-9a-f]+)"\s*#\s*(?P\d+\.\d+\.\d+)\s*$' + <<: *registry-tag-semver + scanArgs: + regexp: '^\s*ALPINE_DIGEST="(?Psha256:[0-9a-f]+)"\s*#\s*(?Pv?\d+\.\d+\.\d+)\s*$' + sourceArgs: repo: "docker.io/library/alpine" shell-alpine-digest: - type: "regexp" - source: "registry-digest-arg-match" - args: + <<: *registry-digest + scanArgs: regexp: '^\s*ALPINE_DIGEST="(?Psha256:[0-9a-f]+)"\s*#\s*(?P\d+\.\d+\.\d+)\s*$' - image: "docker.io/library/alpine" + sourceArgs: + image: "docker.io/library/alpine:{{ .ScanMatch.Tag }}" + +scans: + regexp: + type: "regexp" sources: - registry-tag-arg-semver: - type: "registry" - key: "{{ .ScanArgs.repo }}" - args: - type: "tag" - repo: "{{ .ScanArgs.repo }}" - filter: - expr: '^v?[0-9]+\.[0-9]+\.[0-9]+$' - sort: - method: "semver" - registry-tag-arg-semver-major: - type: "registry" - key: "{{ .ScanArgs.repo }}" - args: - type: "tag" - repo: "{{ .ScanArgs.repo }}" - filter: - expr: '^v?[0-9]+\.[0-9]+\.[0-9]+$' - sort: - method: "semver" - template: '{{ index ( split .Version "." ) 0 }}' - registry-tag-match-semver: - type: "registry" - key: "{{ .ScanMatch.Repo }}" - args: - type: "tag" - repo: "{{ .ScanMatch.Repo }}" - filter: - expr: '^v?[0-9]+\.[0-9]+\.[0-9]+$' - sort: - method: "semver" - registry-digest-arg-match: - type: "registry" - key: "{{ .ScanArgs.image }}:{{.ScanMatch.Tag}}" - args: - image: "{{ .ScanArgs.image }}:{{.ScanMatch.Tag}}" - registry-digest-match: - type: "registry" - key: "{{ .ScanMatch.Image }}:{{.ScanMatch.Tag}}" - args: - image: "{{ .ScanMatch.Image }}:{{.ScanMatch.Tag}}" - registry-golang-latest: - type: "registry" - key: "golang-latest" - args: - repo: "golang" - type: "tag" - filter: - expr: '^\d+\.\d+$' - sort: - method: "semver" - registry-golang-oldest: - type: "registry" - key: "golang-oldest" - args: - repo: "golang" - type: "tag" - filter: - expr: '^\d+\.\d+$' - sort: - method: "semver" - template: '{{ index .VerMap ( index .VerList 2 ) }}' - registry-golang-matrix: - type: "registry" - key: "golang-matrix" - args: - repo: "golang" - type: "tag" - filter: - expr: '^\d+\.\d+$' - sort: - method: "semver" - template: '["{{ index .VerMap ( index .VerList 2 ) }}", "{{ index .VerMap ( index .VerList 1 ) }}", "{{ index .VerMap ( index .VerList 0 ) }}"]' - gha-uses-vx: - type: "git" - key: "{{ .ScanMatch.Repo }}" - args: - type: "tag" - url: "https://github.com/{{ .ScanMatch.Repo }}.git" - filter: - expr: '^v\d+$' - sort: - method: "semver" - gha-uses-semver: - type: "git" - key: "{{ .ScanMatch.Repo }}" - args: - type: "tag" - url: "https://github.com/{{ .ScanMatch.Repo }}.git" - filter: - expr: '^v\d+\.\d+\.\d+$' - sort: - method: "semver" git-commit: type: "git" - key: "{{ .ScanArgs.repo }}:{{ .ScanArgs.ref }}" args: type: "commit" - url: "{{ .ScanArgs.repo }}" - filter: - expr: '^{{ .ScanArgs.ref }}$' - git-tag-semver: + git-tag: type: "git" - key: "{{ .ScanArgs.repo }}" args: type: "tag" - url: "https://{{ .ScanArgs.repo }}.git" - filter: - expr: '^v[0-9]+\.[0-9]+\.[0-9]+$' - sort: - method: "semver" - github-commit-match: - type: "git" - key: "{{ .ScanMatch.Repo }}:{{ .ScanMatch.Ref }}" + registry-digest: + type: "registry" + registry-tag: + type: "registry" args: - type: "commit" - url: "https://github.com/{{ .ScanMatch.Repo }}.git" - ref: "{{ .ScanMatch.Ref }}" - filter: - expr: "^{{ .ScanMatch.Ref }}$" + type: "tag" diff --git a/vendor/github.com/regclient/regclient/Makefile b/vendor/github.com/regclient/regclient/Makefile index 558c1a42a..8d40b7394 100644 --- a/vendor/github.com/regclient/regclient/Makefile +++ b/vendor/github.com/regclient/regclient/Makefile @@ -33,24 +33,24 @@ ifeq "$(strip $(VER_BUMP))" '' -u "$(shell id -u):$(shell id -g)" \ $(VER_BUMP_CONTAINER) endif -MARKDOWN_LINT_VER?=v0.13.0 -GOMAJOR_VER?=v0.13.1 -GOSEC_VER?=v2.20.0 +MARKDOWN_LINT_VER?=v0.14.0 +GOMAJOR_VER?=v0.14.0 +GOSEC_VER?=v2.21.4 GO_VULNCHECK_VER?=v1.1.3 -OSV_SCANNER_VER?=v1.8.2 +OSV_SCANNER_VER?=v1.9.1 SYFT?=$(shell command -v syft 2>/dev/null) SYFT_CMD_VER:=$(shell [ -x "$(SYFT)" ] && echo "v$$($(SYFT) version | awk '/^Version: / {print $$2}')" || echo "0") -SYFT_VERSION?=v1.10.0 -SYFT_CONTAINER?=anchore/syft:v1.10.0@sha256:4243162c3ac33d107a8d9981e2d41b7888b66f12d9bd547124644391be796763 +SYFT_VERSION?=v1.15.0 +SYFT_CONTAINER?=anchore/syft:v1.15.0@sha256:92b229ac1d84cd9627624a951e26a78333b26a5f34c9999629ba96e90751c971 ifneq "$(SYFT_CMD_VER)" "$(SYFT_VERSION)" SYFT=docker run --rm \ -v "$(shell pwd)/:$(shell pwd)/" -w "$(shell pwd)" \ -u "$(shell id -u):$(shell id -g)" \ $(SYFT_CONTAINER) endif -STATICCHECK_VER?=v0.4.7 +STATICCHECK_VER?=v0.5.1 CI_DISTRIBUTION_VER?=2.8.3 -CI_ZOT_VER?=v2.1.0 +CI_ZOT_VER?=v2.1.1 .PHONY: .FORCE .FORCE: diff --git a/vendor/github.com/regclient/regclient/README.md b/vendor/github.com/regclient/regclient/README.md index bfc90bd9d..6bf58ce80 100644 --- a/vendor/github.com/regclient/regclient/README.md +++ b/vendor/github.com/regclient/regclient/README.md @@ -8,8 +8,6 @@ [![Go Reference](https://pkg.go.dev/badge/github.com/regclient/regclient.svg)](https://pkg.go.dev/github.com/regclient/regclient) ![License](https://img.shields.io/github/license/regclient/regclient) [![Go Report Card](https://goreportcard.com/badge/github.com/regclient/regclient)](https://goreportcard.com/report/github.com/regclient/regclient) -[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/regclient/regclient/badge)](https://securityscorecards.dev/viewer/?uri=github.com/regclient/regclient) -[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8088/badge)](https://www.bestpractices.dev/projects/8088) [![GitHub Downloads](https://img.shields.io/github/downloads/regclient/regclient/total?label=GitHub%20downloads)](https://github.com/regclient/regclient/releases) Client interface for the registry API. diff --git a/vendor/github.com/regclient/regclient/blob.go b/vendor/github.com/regclient/regclient/blob.go index c3cd7b698..62f08acce 100644 --- a/vendor/github.com/regclient/regclient/blob.go +++ b/vendor/github.com/regclient/regclient/blob.go @@ -10,13 +10,15 @@ import ( "github.com/sirupsen/logrus" - "github.com/regclient/regclient/internal/throttle" + "github.com/regclient/regclient/internal/pqueue" + "github.com/regclient/regclient/internal/reqmeta" "github.com/regclient/regclient/scheme" "github.com/regclient/regclient/types" "github.com/regclient/regclient/types/blob" "github.com/regclient/regclient/types/descriptor" "github.com/regclient/regclient/types/errs" "github.com/regclient/regclient/types/ref" + "github.com/regclient/regclient/types/warning" ) const blobCBFreq = time.Millisecond * 100 @@ -49,6 +51,10 @@ func (rc *RegClient) BlobCopy(ctx context.Context, refSrc ref.Ref, refTgt ref.Re for _, optFn := range opts { optFn(&opt) } + // dedup warnings + if w := warning.FromContext(ctx); w == nil { + ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()}) + } tDesc := d tDesc.URLs = []string{} // ignore URLs when pushing to target if opt.callback != nil { @@ -78,7 +84,7 @@ func (rc *RegClient) BlobCopy(ctx context.Context, refSrc ref.Ref, refTgt ref.Re return nil } // acquire throttle for both src and tgt to avoid deadlocks - tList := []*throttle.Throttle{} + tList := []*pqueue.Queue[reqmeta.Data]{} schemeSrcAPI, err := rc.schemeGet(refSrc.Scheme) if err != nil { return err @@ -94,11 +100,14 @@ func (rc *RegClient) BlobCopy(ctx context.Context, refSrc ref.Ref, refTgt ref.Re tList = append(tList, tTgt.Throttle(refTgt, true)...) } if len(tList) > 0 { - ctx, err = throttle.AcquireMulti(ctx, tList) + ctxMulti, done, err := pqueue.AcquireMulti[reqmeta.Data](ctx, reqmeta.Data{Kind: reqmeta.Blob, Size: d.Size}, tList...) if err != nil { return err } - defer throttle.ReleaseMulti(ctx, tList) + if done != nil { + defer done() + } + ctx = ctxMulti } // try mounting blob from the source repo is the registry is the same diff --git a/vendor/github.com/regclient/regclient/config/host.go b/vendor/github.com/regclient/regclient/config/host.go index 5ed3eecb4..17d4e20e7 100644 --- a/vendor/github.com/regclient/regclient/config/host.go +++ b/vendor/github.com/regclient/regclient/config/host.go @@ -6,12 +6,10 @@ import ( "fmt" "io" "strings" - "sync" "time" "github.com/sirupsen/logrus" - "github.com/regclient/regclient/internal/throttle" "github.com/regclient/regclient/internal/timejson" ) @@ -43,15 +41,11 @@ const ( // defaultConcurrent is the default number of concurrent registry connections. defaultConcurrent = 3 // defaultReqPerSec is the default maximum frequency to send requests to a registry. - defaultReqPerSec = 10 + defaultReqPerSec = 0 // tokenUser is the username returned by credential helpers that indicates the password is an identity token. tokenUser = "" ) -var ( - mu = sync.Mutex{} -) - // MarshalJSON converts TLSConf to a json string using MarshalText. func (t TLSConf) MarshalJSON() ([]byte, error) { s, err := t.MarshalText() @@ -105,31 +99,30 @@ func (t *TLSConf) UnmarshalText(b []byte) error { // Host defines settings for connecting to a registry. type Host struct { - Name string `json:"-" yaml:"registry,omitempty"` // Name of the registry (required) (yaml configs pass this as a field, json provides this from the object key) - TLS TLSConf `json:"tls,omitempty" yaml:"tls"` // TLS setting: enabled (default), disabled, insecure - RegCert string `json:"regcert,omitempty" yaml:"regcert"` // public pem cert of registry - ClientCert string `json:"clientCert,omitempty" yaml:"clientCert"` // public pem cert for client (mTLS) - ClientKey string `json:"clientKey,omitempty" yaml:"clientKey"` // private pem cert for client (mTLS) - Hostname string `json:"hostname,omitempty" yaml:"hostname"` // hostname of registry, default is the registry name - User string `json:"user,omitempty" yaml:"user"` // username, not used with credHelper - Pass string `json:"pass,omitempty" yaml:"pass"` // password, not used with credHelper - Token string `json:"token,omitempty" yaml:"token"` // token, experimental for specific APIs - CredHelper string `json:"credHelper,omitempty" yaml:"credHelper"` // credential helper command for requesting logins - CredExpire timejson.Duration `json:"credExpire,omitempty" yaml:"credExpire"` // time until credential expires - CredHost string `json:"credHost" yaml:"credHost"` // used when a helper hostname doesn't match Hostname - PathPrefix string `json:"pathPrefix,omitempty" yaml:"pathPrefix"` // used for mirrors defined within a repository namespace - Mirrors []string `json:"mirrors,omitempty" yaml:"mirrors"` // list of other Host Names to use as mirrors - Priority uint `json:"priority,omitempty" yaml:"priority"` // priority when sorting mirrors, higher priority attempted first - RepoAuth bool `json:"repoAuth,omitempty" yaml:"repoAuth"` // tracks a separate auth per repo - API string `json:"api,omitempty" yaml:"api"` // experimental: registry API to use - APIOpts map[string]string `json:"apiOpts,omitempty" yaml:"apiOpts"` // options for APIs - BlobChunk int64 `json:"blobChunk,omitempty" yaml:"blobChunk"` // size of each blob chunk - BlobMax int64 `json:"blobMax,omitempty" yaml:"blobMax"` // threshold to switch to chunked upload, -1 to disable, 0 for regclient.blobMaxPut - ReqPerSec float64 `json:"reqPerSec,omitempty" yaml:"reqPerSec"` // requests per second, default is defaultReqPerSec(10) - ReqConcurrent int64 `json:"reqConcurrent,omitempty" yaml:"reqConcurrent"` // concurrent requests, default is defaultConcurrent(3) - Scheme string `json:"scheme,omitempty" yaml:"scheme"` // Deprecated: use TLS instead - credRefresh time.Time `json:"-" yaml:"-"` // internal use, when to refresh credentials - throttle *throttle.Throttle `json:"-" yaml:"-"` // internal use, limit for concurrent requests + Name string `json:"-" yaml:"registry,omitempty"` // Name of the registry (required) (yaml configs pass this as a field, json provides this from the object key) + TLS TLSConf `json:"tls,omitempty" yaml:"tls"` // TLS setting: enabled (default), disabled, insecure + RegCert string `json:"regcert,omitempty" yaml:"regcert"` // public pem cert of registry + ClientCert string `json:"clientCert,omitempty" yaml:"clientCert"` // public pem cert for client (mTLS) + ClientKey string `json:"clientKey,omitempty" yaml:"clientKey"` // private pem cert for client (mTLS) + Hostname string `json:"hostname,omitempty" yaml:"hostname"` // hostname of registry, default is the registry name + User string `json:"user,omitempty" yaml:"user"` // username, not used with credHelper + Pass string `json:"pass,omitempty" yaml:"pass"` // password, not used with credHelper + Token string `json:"token,omitempty" yaml:"token"` // token, experimental for specific APIs + CredHelper string `json:"credHelper,omitempty" yaml:"credHelper"` // credential helper command for requesting logins + CredExpire timejson.Duration `json:"credExpire,omitempty" yaml:"credExpire"` // time until credential expires + CredHost string `json:"credHost,omitempty" yaml:"credHost"` // used when a helper hostname doesn't match Hostname + PathPrefix string `json:"pathPrefix,omitempty" yaml:"pathPrefix"` // used for mirrors defined within a repository namespace + Mirrors []string `json:"mirrors,omitempty" yaml:"mirrors"` // list of other Host Names to use as mirrors + Priority uint `json:"priority,omitempty" yaml:"priority"` // priority when sorting mirrors, higher priority attempted first + RepoAuth bool `json:"repoAuth,omitempty" yaml:"repoAuth"` // tracks a separate auth per repo + API string `json:"api,omitempty" yaml:"api"` // Deprecated: registry API to use + APIOpts map[string]string `json:"apiOpts,omitempty" yaml:"apiOpts"` // options for APIs + BlobChunk int64 `json:"blobChunk,omitempty" yaml:"blobChunk"` // size of each blob chunk + BlobMax int64 `json:"blobMax,omitempty" yaml:"blobMax"` // threshold to switch to chunked upload, -1 to disable, 0 for regclient.blobMaxPut + ReqPerSec float64 `json:"reqPerSec,omitempty" yaml:"reqPerSec"` // requests per second + ReqConcurrent int64 `json:"reqConcurrent,omitempty" yaml:"reqConcurrent"` // concurrent requests, default is defaultConcurrent(3) + Scheme string `json:"scheme,omitempty" yaml:"scheme"` // Deprecated: use TLS instead + credRefresh time.Time `json:"-" yaml:"-"` // internal use, when to refresh credentials } // Cred defines a user credential for accessing a registry. @@ -148,16 +141,48 @@ func HostNew() *Host { return &h } -// HostNewName creates a default Host with a hostname. -func HostNewName(name string) *Host { - h := HostNew() +// HostNewDefName creates a host using provided defaults and hostname. +func HostNewDefName(def *Host, name string) *Host { + var h Host + if def == nil { + h = *HostNew() + } else { + h = *def + // configure required defaults + if h.TLS == TLSUndefined { + h.TLS = TLSEnabled + } + if h.APIOpts == nil { + h.APIOpts = map[string]string{} + } + if h.ReqConcurrent == 0 { + h.ReqConcurrent = int64(defaultConcurrent) + } + if h.ReqPerSec == 0 { + h.ReqPerSec = float64(defaultReqPerSec) + } + // copy any fields that are not passed by value + if len(h.APIOpts) > 0 { + orig := h.APIOpts + h.APIOpts = map[string]string{} + for k, v := range orig { + h.APIOpts[k] = v + } + } + if h.Mirrors != nil { + orig := h.Mirrors + h.Mirrors = make([]string, len(orig)) + copy(h.Mirrors, orig) + } + } + // configure host origName := name // Docker Hub is a special case if name == DockerRegistryAuth || name == DockerRegistryDNS || name == DockerRegistry { h.Name = DockerRegistry h.Hostname = DockerRegistryDNS h.CredHost = DockerRegistryAuth - return h + return &h } // handle http/https prefix i := strings.Index(name, "://") @@ -178,7 +203,12 @@ func HostNewName(name string) *Host { if origName != name { h.CredHost = origName } - return h + return &h +} + +// HostNewName creates a default Host with a hostname. +func HostNewName(name string) *Host { + return HostNewDefName(nil, name) } // GetCred returns the credential, fetching from a credential helper if needed. @@ -207,6 +237,35 @@ func (host *Host) refreshHelper() { } } +// IsZero returns true if the struct is set to the zero value or the result of [HostNew]. +func (host Host) IsZero() bool { + if host.Name != "" || + (host.TLS != TLSUndefined && host.TLS != TLSEnabled) || + host.RegCert != "" || + host.ClientCert != "" || + host.ClientKey != "" || + host.Hostname != "" || + host.User != "" || + host.Pass != "" || + host.Token != "" || + host.CredHelper != "" || + host.CredExpire != 0 || + host.CredHost != "" || + host.PathPrefix != "" || + len(host.Mirrors) != 0 || + host.Priority != 0 || + host.RepoAuth || + len(host.APIOpts) != 0 || + host.BlobChunk != 0 || + host.BlobMax != 0 || + (host.ReqPerSec != 0 && host.ReqPerSec != float64(defaultReqPerSec)) || + (host.ReqConcurrent != 0 && host.ReqConcurrent != int64(defaultConcurrent)) || + !host.credRefresh.IsZero() { + return false + } + return true +} + // Merge adds fields from a new config host entry. func (host *Host) Merge(newHost Host, log *logrus.Logger) error { name := newHost.Name @@ -390,15 +449,12 @@ func (host *Host) Merge(newHost Host, log *logrus.Logger) error { host.RepoAuth = newHost.RepoAuth } + // TODO: eventually delete if newHost.API != "" { - if host.API != "" && host.API != newHost.API { - log.WithFields(logrus.Fields{ - "orig": host.API, - "new": newHost.API, - "host": name, - }).Warn("Changing API settings for registry") - } - host.API = newHost.API + log.WithFields(logrus.Fields{ + "api": newHost.API, + "host": name, + }).Warn("API field has been deprecated") } if len(newHost.APIOpts) > 0 { @@ -443,7 +499,7 @@ func (host *Host) Merge(newHost Host, log *logrus.Logger) error { host.BlobMax = newHost.BlobMax } - if newHost.ReqPerSec > 0 { + if newHost.ReqPerSec != 0 { if host.ReqPerSec != 0 && host.ReqPerSec != newHost.ReqPerSec { log.WithFields(logrus.Fields{ "orig": host.ReqPerSec, @@ -456,14 +512,6 @@ func (host *Host) Merge(newHost Host, log *logrus.Logger) error { if newHost.ReqConcurrent > 0 { if host.ReqConcurrent != 0 && host.ReqConcurrent != newHost.ReqConcurrent { - if host.throttle != nil { - log.WithFields(logrus.Fields{ - "orig": host.ReqConcurrent, - "new": newHost.ReqConcurrent, - "host": name, - }).Warn("Unable to change ReqConcurrent after throttle is created") - return fmt.Errorf("unable to change ReqConcurrent after throttle is created") - } log.WithFields(logrus.Fields{ "orig": host.ReqConcurrent, "new": newHost.ReqConcurrent, @@ -476,18 +524,6 @@ func (host *Host) Merge(newHost Host, log *logrus.Logger) error { return nil } -func (host *Host) Throttle() *throttle.Throttle { - if host.ReqConcurrent <= 0 { - return nil - } - mu.Lock() - defer mu.Unlock() - if host.throttle == nil { - host.throttle = throttle.New(int(host.ReqConcurrent)) - } - return host.throttle -} - func copyMapString(src map[string]string) map[string]string { copy := map[string]string{} for k, v := range src { diff --git a/vendor/github.com/regclient/regclient/image.go b/vendor/github.com/regclient/regclient/image.go index efa872527..b6698d870 100644 --- a/vendor/github.com/regclient/regclient/image.go +++ b/vendor/github.com/regclient/regclient/image.go @@ -235,6 +235,10 @@ func (rc *RegClient) ImageCheckBase(ctx context.Context, r ref.Ref, opts ...Imag var m manifest.Manifest var err error + // dedup warnings + if w := warning.FromContext(ctx); w == nil { + ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()}) + } // if the base name is not provided, check image for base annotations if opt.checkBaseRef == "" { m, err = rc.ManifestGet(ctx, r) @@ -446,6 +450,10 @@ func (rc *RegClient) ImageConfig(ctx context.Context, r ref.Ref, opts ...ImageOp for _, optFn := range opts { optFn(&opt) } + // dedup warnings + if w := warning.FromContext(ctx); w == nil { + ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()}) + } p, err := platform.Parse(opt.platform) if err != nil { return nil, fmt.Errorf("failed to parse platform %s: %w", opt.platform, err) @@ -672,6 +680,111 @@ func (rc *RegClient) imageCopyOpt(ctx context.Context, refSrc ref.Ref, refTgt re } } + // If source is image, copy blobs + if mSrcImg, ok := mSrc.(manifest.Imager); ok && mSrc.IsSet() && !ref.EqualRepository(refSrc, refTgt) { + // copy the config + cd, err := mSrcImg.GetConfig() + if err != nil { + // docker schema v1 does not have a config object, ignore if it's missing + if !errors.Is(err, errs.ErrUnsupportedMediaType) { + rc.log.WithFields(logrus.Fields{ + "ref": refSrc.Reference, + "err": err, + }).Warn("Failed to get config digest from manifest") + return fmt.Errorf("failed to get config digest for %s: %w", refSrc.CommonName(), err) + } + } else { + waitCount++ + go func() { + rc.log.WithFields(logrus.Fields{ + "source": refSrc.Reference, + "target": refTgt.Reference, + "digest": cd.Digest.String(), + }).Info("Copy config") + err := rc.imageCopyBlob(ctx, refSrc, refTgt, cd, opt, bOpt...) + if err != nil && !errors.Is(err, context.Canceled) { + rc.log.WithFields(logrus.Fields{ + "source": refSrc.Reference, + "target": refTgt.Reference, + "digest": cd.Digest.String(), + "err": err, + }).Warn("Failed to copy config") + } + waitCh <- err + }() + } + + // copy filesystem layers + l, err := mSrcImg.GetLayers() + if err != nil { + return err + } + for _, layerSrc := range l { + if len(layerSrc.URLs) > 0 && !opt.includeExternal { + // skip blobs where the URLs are defined, these aren't hosted and won't be pulled from the source + rc.log.WithFields(logrus.Fields{ + "source": refSrc.Reference, + "target": refTgt.Reference, + "layer": layerSrc.Digest.String(), + "external-urls": layerSrc.URLs, + }).Debug("Skipping external layer") + continue + } + waitCount++ + layerSrc := layerSrc + go func() { + rc.log.WithFields(logrus.Fields{ + "source": refSrc.Reference, + "target": refTgt.Reference, + "layer": layerSrc.Digest.String(), + }).Info("Copy layer") + err := rc.imageCopyBlob(ctx, refSrc, refTgt, layerSrc, opt, bOpt...) + if err != nil && !errors.Is(err, context.Canceled) { + rc.log.WithFields(logrus.Fields{ + "source": refSrc.Reference, + "target": refTgt.Reference, + "layer": layerSrc.Digest.String(), + "err": err, + }).Warn("Failed to copy layer") + } + waitCh <- err + }() + } + } + + // check for any errors and abort early if found + err = nil + done := false + for !done && waitCount > 0 { + if err == nil { + select { + case err = <-waitCh: + if err != nil { + cancel() + } + default: + done = true // happy path + } + } else { + if errors.Is(err, context.Canceled) { + // try to find a better error message than context canceled + err = <-waitCh + } else { + <-waitCh + } + } + if !done { + waitCount-- + } + } + if err != nil { + rc.log.WithFields(logrus.Fields{ + "err": err, + "sDig": sDig, + }).Debug("child manifest copy failed") + return err + } + // copy referrers referrerTags := []string{} if opt.referrerConfs != nil { @@ -796,111 +909,6 @@ func (rc *RegClient) imageCopyOpt(ctx context.Context, refSrc ref.Ref, refTgt re } } - // check for any errors and abort early if found - err = nil - done := false - for !done && waitCount > 0 { - if err == nil { - select { - case err = <-waitCh: - if err != nil { - cancel() - } - default: - done = true // happy path - } - } else { - if errors.Is(err, context.Canceled) { - // try to find a better error message than context canceled - err = <-waitCh - } else { - <-waitCh - } - } - if !done { - waitCount-- - } - } - if err != nil { - rc.log.WithFields(logrus.Fields{ - "err": err, - "sDig": sDig, - }).Debug("child manifest copy failed") - return err - } - - // If source is image, copy blobs - if mSrcImg, ok := mSrc.(manifest.Imager); ok && mSrc.IsSet() && !ref.EqualRepository(refSrc, refTgt) { - // copy the config - cd, err := mSrcImg.GetConfig() - if err != nil { - // docker schema v1 does not have a config object, ignore if it's missing - if !errors.Is(err, errs.ErrUnsupportedMediaType) { - rc.log.WithFields(logrus.Fields{ - "ref": refSrc.Reference, - "err": err, - }).Warn("Failed to get config digest from manifest") - return fmt.Errorf("failed to get config digest for %s: %w", refSrc.CommonName(), err) - } - } else { - waitCount++ - go func() { - rc.log.WithFields(logrus.Fields{ - "source": refSrc.Reference, - "target": refTgt.Reference, - "digest": cd.Digest.String(), - }).Info("Copy config") - err := rc.imageCopyBlob(ctx, refSrc, refTgt, cd, opt, bOpt...) - if err != nil && !errors.Is(err, context.Canceled) { - rc.log.WithFields(logrus.Fields{ - "source": refSrc.Reference, - "target": refTgt.Reference, - "digest": cd.Digest.String(), - "err": err, - }).Warn("Failed to copy config") - } - waitCh <- err - }() - } - - // copy filesystem layers - l, err := mSrcImg.GetLayers() - if err != nil { - return err - } - for _, layerSrc := range l { - if len(layerSrc.URLs) > 0 && !opt.includeExternal { - // skip blobs where the URLs are defined, these aren't hosted and won't be pulled from the source - rc.log.WithFields(logrus.Fields{ - "source": refSrc.Reference, - "target": refTgt.Reference, - "layer": layerSrc.Digest.String(), - "external-urls": layerSrc.URLs, - }).Debug("Skipping external layer") - continue - } - waitCount++ - layerSrc := layerSrc - go func() { - rc.log.WithFields(logrus.Fields{ - "source": refSrc.Reference, - "target": refTgt.Reference, - "layer": layerSrc.Digest.String(), - }).Info("Copy layer") - err := rc.imageCopyBlob(ctx, refSrc, refTgt, layerSrc, opt, bOpt...) - if err != nil && !errors.Is(err, context.Canceled) { - rc.log.WithFields(logrus.Fields{ - "source": refSrc.Reference, - "target": refTgt.Reference, - "layer": layerSrc.Digest.String(), - "err": err, - }).Warn("Failed to copy layer") - } - waitCh <- err - }() - } - } - // wait for background tasks to finish err = nil for waitCount > 0 { @@ -1037,6 +1045,10 @@ func (rc *RegClient) ImageExport(ctx context.Context, r ref.Ref, outStream io.Wr opt.exportRef = r } + // dedup warnings + if w := warning.FromContext(ctx); w == nil { + ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()}) + } // create tar writer object out := outStream if opt.exportCompress { @@ -1270,6 +1282,10 @@ func (rc *RegClient) ImageImport(ctx context.Context, r ref.Ref, rs io.ReadSeeke optFn(&opt) } + // dedup warnings + if w := warning.FromContext(ctx); w == nil { + ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()}) + } trd := &tarReadData{ name: opt.importName, handlers: map[string]tarFileHandler{}, diff --git a/vendor/github.com/regclient/regclient/internal/auth/auth.go b/vendor/github.com/regclient/regclient/internal/auth/auth.go index 3da146183..0955d2dcc 100644 --- a/vendor/github.com/regclient/regclient/internal/auth/auth.go +++ b/vendor/github.com/regclient/regclient/internal/auth/auth.go @@ -49,58 +49,56 @@ func init() { } // CredsFn is passed to lookup credentials for a given hostname, response is a username and password or empty strings -type CredsFn func(string) Cred +type CredsFn func(host string) Cred -// Cred is returned by the CredsFn +// Cred is returned by the CredsFn. +// If Token is provided and auth method is bearer, it will attempt to use it as a refresh token. +// Else if user and password are provided, they are attempted with all auth methods. +// Else if neither are provided and auth method is bearer, an anonymous login is attempted. type Cred struct { - User, Password, Token string + User, Password string // clear text username and password + Token string // refresh token only used for bearer auth } -// Auth manages authorization requests/responses for http requests -type Auth interface { - AddScope(host, scope string) error - HandleResponse(*http.Response) error - UpdateRequest(*http.Request) error -} - -// Challenge is the extracted contents of the WWW-Authenticate header -type Challenge struct { +// challenge is the extracted contents of the WWW-Authenticate header. +type challenge struct { authType string params map[string]string } -// Handler handles a challenge for a host to return an auth header -type Handler interface { +// handler handles a challenge for a host to return an auth header +type handler interface { AddScope(scope string) error - ProcessChallenge(Challenge) error + ProcessChallenge(challenge) error GenerateAuth() (string, error) } -// HandlerBuild is used to make a new handler for a specific authType and URL -type HandlerBuild func(client *http.Client, clientID, host string, credFn CredsFn, log *logrus.Logger) Handler +// handlerBuild is used to make a new handler for a specific authType and URL +type handlerBuild func(client *http.Client, clientID, host string, credFn CredsFn, log *logrus.Logger) handler // Opts configures options for NewAuth -type Opts func(*auth) +type Opts func(*Auth) -type auth struct { +// Auth is used to handle authentication requests. +type Auth struct { httpClient *http.Client clientID string credsFn CredsFn - hbs map[string]HandlerBuild // handler builders based on authType - hs map[string]map[string]Handler // handlers based on url and authType + hbs map[string]handlerBuild // handler builders based on authType + hs map[string]map[string]handler // handlers based on url and authType authTypes []string log *logrus.Logger mu sync.Mutex } // NewAuth creates a new Auth -func NewAuth(opts ...Opts) Auth { - a := &auth{ +func NewAuth(opts ...Opts) *Auth { + a := &Auth{ httpClient: &http.Client{}, clientID: defaultClientID, credsFn: DefaultCredsFn, - hbs: map[string]HandlerBuild{}, - hs: map[string]map[string]Handler{}, + hbs: map[string]handlerBuild{}, + hs: map[string]map[string]handler{}, authTypes: []string{}, } a.log = &logrus.Logger{ @@ -123,7 +121,7 @@ func NewAuth(opts ...Opts) Auth { // WithCreds provides a user/pass lookup for a url func WithCreds(f CredsFn) Opts { - return func(a *auth) { + return func(a *Auth) { if f != nil { a.credsFn = f } @@ -132,7 +130,7 @@ func WithCreds(f CredsFn) Opts { // WithHTTPClient uses a specific http client with requests func WithHTTPClient(h *http.Client) Opts { - return func(a *auth) { + return func(a *Auth) { if h != nil { a.httpClient = h } @@ -141,14 +139,14 @@ func WithHTTPClient(h *http.Client) Opts { // WithClientID uses a client ID with request headers func WithClientID(clientID string) Opts { - return func(a *auth) { + return func(a *Auth) { a.clientID = clientID } } // WithHandler includes a handler for a specific auth type -func WithHandler(authType string, hb HandlerBuild) Opts { - return func(a *auth) { +func WithHandler(authType string, hb handlerBuild) Opts { + return func(a *Auth) { lcat := strings.ToLower(authType) a.hbs[lcat] = hb a.authTypes = append(a.authTypes, lcat) @@ -157,14 +155,14 @@ func WithHandler(authType string, hb HandlerBuild) Opts { // WithDefaultHandlers includes a Basic and Bearer handler, this is automatically added with "WithHandler" is not called func WithDefaultHandlers() Opts { - return func(a *auth) { + return func(a *Auth) { a.addDefaultHandlers() } } // WithLog injects a logrus Logger func WithLog(log *logrus.Logger) Opts { - return func(a *auth) { + return func(a *Auth) { a.log = log } } @@ -172,7 +170,7 @@ func WithLog(log *logrus.Logger) Opts { // AddScope extends an existing auth with additional scopes. // This is used to pre-populate scopes with the Docker convention rather than // depend on the registry to respond with the correct http status and headers. -func (a *auth) AddScope(host, scope string) error { +func (a *Auth) AddScope(host, scope string) error { a.mu.Lock() defer a.mu.Unlock() success := false @@ -202,7 +200,7 @@ func (a *auth) AddScope(host, scope string) error { // HandleResponse parses the 401 response, extracting the WWW-Authenticate // header and verifying the requirement is different from what was included in // the last request -func (a *auth) HandleResponse(resp *http.Response) error { +func (a *Auth) HandleResponse(resp *http.Response) error { a.mu.Lock() defer a.mu.Unlock() // verify response is an access denied @@ -233,7 +231,7 @@ func (a *auth) HandleResponse(resp *http.Response) error { } // setup a handler for the host and auth type if _, ok := a.hs[host]; !ok { - a.hs[host] = map[string]Handler{} + a.hs[host] = map[string]handler{} } if _, ok := a.hs[host][c.authType]; !ok { h := a.hbs[c.authType](a.httpClient, a.clientID, host, a.credsFn, a.log) @@ -266,7 +264,7 @@ func (a *auth) HandleResponse(resp *http.Response) error { } // UpdateRequest adds Authorization headers to a request -func (a *auth) UpdateRequest(req *http.Request) error { +func (a *Auth) UpdateRequest(req *http.Request) error { a.mu.Lock() defer a.mu.Unlock() host := req.URL.Host @@ -296,7 +294,7 @@ func (a *auth) UpdateRequest(req *http.Request) error { return nil } -func (a *auth) addDefaultHandlers() { +func (a *Auth) addDefaultHandlers() { if _, ok := a.hbs["basic"]; !ok { a.hbs["basic"] = NewBasicHandler a.authTypes = append(a.authTypes, "basic") @@ -305,11 +303,6 @@ func (a *auth) addDefaultHandlers() { a.hbs["bearer"] = NewBearerHandler a.authTypes = append(a.authTypes, "bearer") } - // jwt is considered experimental, used for some Hub specific API's - if _, ok := a.hbs["jwt"]; !ok { - a.hbs["jwt"] = NewJWTHandler - a.authTypes = append(a.authTypes, "jwt") - } } // DefaultCredsFn is used to return no credentials when auth is not configured with a CredsFn @@ -319,10 +312,10 @@ func DefaultCredsFn(h string) Cred { } // ParseAuthHeaders extracts the scheme and realm from WWW-Authenticate headers -func ParseAuthHeaders(ahl []string) ([]Challenge, error) { - var cl []Challenge +func ParseAuthHeaders(ahl []string) ([]challenge, error) { + var cl []challenge for _, ah := range ahl { - c, err := ParseAuthHeader(ah) + c, err := parseAuthHeader(ah) if err != nil { return nil, fmt.Errorf("failed to parse challenge header: %s, %w", ah, err) } @@ -331,13 +324,13 @@ func ParseAuthHeaders(ahl []string) ([]Challenge, error) { return cl, nil } -// ParseAuthHeader parses a single header line for WWW-Authenticate +// parseAuthHeader parses a single header line for WWW-Authenticate // Example values: // Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:samalba/my-app:pull,push" // Basic realm="GitHub Package Registry" -func ParseAuthHeader(ah string) ([]Challenge, error) { - var cl []Challenge - var c *Challenge +func parseAuthHeader(ah string) ([]challenge, error) { + var cl []challenge + var c *challenge var eb, atb, kb, vb []byte // eb is element bytes, atb auth type, kb key, vb value state := "string" @@ -370,7 +363,7 @@ func ParseAuthHeader(ah string) ([]Challenge, error) { // space ends the element atb = eb eb = []byte{} - c = &Challenge{authType: strings.ToLower(string(atb)), params: map[string]string{}} + c = &challenge{authType: strings.ToLower(string(atb)), params: map[string]string{}} cl = append(cl, *c) } else { // unknown char @@ -439,7 +432,7 @@ func ParseAuthHeader(ah string) ([]Challenge, error) { case "string": if len(eb) != 0 { atb = eb - c = &Challenge{authType: strings.ToLower(string(atb)), params: map[string]string{}} + c = &challenge{authType: strings.ToLower(string(atb)), params: map[string]string{}} cl = append(cl, *c) } case "value": @@ -453,16 +446,16 @@ func ParseAuthHeader(ah string) ([]Challenge, error) { return cl, nil } -// BasicHandler supports Basic auth type requests -type BasicHandler struct { +// basicHandler supports Basic auth type requests +type basicHandler struct { realm string host string credsFn CredsFn } // NewBasicHandler creates a new BasicHandler -func NewBasicHandler(client *http.Client, clientID, host string, credsFn CredsFn, log *logrus.Logger) Handler { - return &BasicHandler{ +func NewBasicHandler(client *http.Client, clientID, host string, credsFn CredsFn, log *logrus.Logger) handler { + return &basicHandler{ realm: "", host: host, credsFn: credsFn, @@ -470,12 +463,12 @@ func NewBasicHandler(client *http.Client, clientID, host string, credsFn CredsFn } // AddScope is not valid for BasicHandler -func (b *BasicHandler) AddScope(scope string) error { +func (b *basicHandler) AddScope(scope string) error { return ErrNoNewChallenge } // ProcessChallenge for BasicHandler is a noop -func (b *BasicHandler) ProcessChallenge(c Challenge) error { +func (b *basicHandler) ProcessChallenge(c challenge) error { if _, ok := c.params["realm"]; !ok { return ErrInvalidChallenge } @@ -487,7 +480,7 @@ func (b *BasicHandler) ProcessChallenge(c Challenge) error { } // GenerateAuth for BasicHandler generates base64 encoded user/pass for a host -func (b *BasicHandler) GenerateAuth() (string, error) { +func (b *basicHandler) GenerateAuth() (string, error) { cred := b.credsFn(b.host) if cred.User == "" || cred.Password == "" { return "", fmt.Errorf("no credentials available: %w", errs.ErrHTTPUnauthorized) @@ -496,20 +489,20 @@ func (b *BasicHandler) GenerateAuth() (string, error) { return fmt.Sprintf("Basic %s", auth), nil } -// BearerHandler supports Bearer auth type requests -type BearerHandler struct { +// bearerHandler supports Bearer auth type requests +type bearerHandler struct { client *http.Client clientID string realm, service string host string credsFn CredsFn scopes []string - token BearerToken + token bearerToken log *logrus.Logger } -// BearerToken is the json response to the Bearer request -type BearerToken struct { +// bearerToken is the json response to the Bearer request +type bearerToken struct { Token string `json:"token"` AccessToken string `json:"access_token"` ExpiresIn int `json:"expires_in"` @@ -519,8 +512,8 @@ type BearerToken struct { } // NewBearerHandler creates a new BearerHandler -func NewBearerHandler(client *http.Client, clientID, host string, credsFn CredsFn, log *logrus.Logger) Handler { - return &BearerHandler{ +func NewBearerHandler(client *http.Client, clientID, host string, credsFn CredsFn, log *logrus.Logger) handler { + return &bearerHandler{ client: client, clientID: clientID, host: host, @@ -533,7 +526,7 @@ func NewBearerHandler(client *http.Client, clientID, host string, credsFn CredsF } // AddScope appends a new scope if it doesn't already exist -func (b *BearerHandler) AddScope(scope string) error { +func (b *bearerHandler) AddScope(scope string) error { if b.scopeExists(scope) { if b.token.Token == "" || !b.isExpired() { return ErrNoNewChallenge @@ -543,7 +536,7 @@ func (b *BearerHandler) AddScope(scope string) error { return b.addScope(scope) } -func (b *BearerHandler) addScope(scope string) error { +func (b *bearerHandler) addScope(scope string) error { replaced := false for i, cur := range b.scopes { // extend an existing scope with more actions @@ -564,7 +557,7 @@ func (b *BearerHandler) addScope(scope string) error { // ProcessChallenge handles WWW-Authenticate header for bearer tokens // Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:samalba/my-app:pull,push" -func (b *BearerHandler) ProcessChallenge(c Challenge) error { +func (b *bearerHandler) ProcessChallenge(c challenge) error { if _, ok := c.params["realm"]; !ok { return ErrInvalidChallenge } @@ -598,7 +591,7 @@ func (b *BearerHandler) ProcessChallenge(c Challenge) error { } // GenerateAuth for BasicHandler generates base64 encoded user/pass for a host -func (b *BearerHandler) GenerateAuth() (string, error) { +func (b *bearerHandler) GenerateAuth() (string, error) { // if unexpired token already exists, return it if b.token.Token != "" && !b.isExpired() { return fmt.Sprintf("Bearer %s", b.token.Token), nil @@ -623,7 +616,7 @@ func (b *BearerHandler) GenerateAuth() (string, error) { // isExpired returns true when token issue date is either 0, token has expired, // or will expire within buffer time -func (b *BearerHandler) isExpired() bool { +func (b *bearerHandler) isExpired() bool { if b.token.IssuedAt.IsZero() { return true } @@ -633,7 +626,7 @@ func (b *BearerHandler) isExpired() bool { } // tryGet requests a new token with a GET request -func (b *BearerHandler) tryGet() error { +func (b *bearerHandler) tryGet() error { cred := b.credsFn(b.host) req, err := http.NewRequest("GET", b.realm, nil) if err != nil { @@ -669,7 +662,7 @@ func (b *BearerHandler) tryGet() error { } // tryPost requests a new token via a POST request -func (b *BearerHandler) tryPost() error { +func (b *bearerHandler) tryPost() error { cred := b.credsFn(b.host) form := url.Values{} if len(b.scopes) > 0 { @@ -708,7 +701,7 @@ func (b *BearerHandler) tryPost() error { } // scopeExists check if the scope already exists within the list of scopes -func (b *BearerHandler) scopeExists(search string) bool { +func (b *bearerHandler) scopeExists(search string) bool { if search == "" { return true } @@ -722,14 +715,14 @@ func (b *BearerHandler) scopeExists(search string) bool { } // validateResponse extracts the returned token -func (b *BearerHandler) validateResponse(resp *http.Response) error { +func (b *bearerHandler) validateResponse(resp *http.Response) error { if resp.StatusCode != 200 { return ErrUnauthorized } // decode response and if successful, update token decoder := json.NewDecoder(resp.Body) - decoded := BearerToken{} + decoded := bearerToken{} if err := decoder.Decode(&decoded); err != nil { return err } @@ -756,8 +749,8 @@ func (b *BearerHandler) validateResponse(resp *http.Response) error { return nil } -// JWTHubHandler supports JWT auth type requests -type JWTHubHandler struct { +// jwtHubHandler supports JWT auth type requests. +type jwtHubHandler struct { client *http.Client clientID string realm string @@ -776,11 +769,11 @@ type jwtHubResp struct { RefreshToken string `json:"refresh_token"` } -// NewJWTHandler creates a new JWTHandler -func NewJWTHandler(client *http.Client, clientID, host string, credsFn CredsFn, log *logrus.Logger) Handler { +// NewJWTHubHandler creates a new JWTHandler for Docker Hub. +func NewJWTHubHandler(client *http.Client, clientID, host string, credsFn CredsFn, log *logrus.Logger) handler { // JWT handler is only tested against Hub, and the API is Hub specific if host == "hub.docker.com" { - return &JWTHubHandler{ + return &jwtHubHandler{ client: client, clientID: clientID, host: host, @@ -792,12 +785,12 @@ func NewJWTHandler(client *http.Client, clientID, host string, credsFn CredsFn, } // AddScope is not valid for JWTHubHandler -func (j *JWTHubHandler) AddScope(scope string) error { +func (j *jwtHubHandler) AddScope(scope string) error { return ErrNoNewChallenge } // ProcessChallenge handles WWW-Authenticate header for JWT auth on Docker Hub -func (j *JWTHubHandler) ProcessChallenge(c Challenge) error { +func (j *jwtHubHandler) ProcessChallenge(c challenge) error { cred := j.credsFn(j.host) // use token if provided if cred.Token != "" { @@ -844,7 +837,7 @@ func (j *JWTHubHandler) ProcessChallenge(c Challenge) error { } // GenerateAuth for JWTHubHandler adds JWT header -func (j *JWTHubHandler) GenerateAuth() (string, error) { +func (j *jwtHubHandler) GenerateAuth() (string, error) { if len(j.jwt) > 0 { return fmt.Sprintf("JWT %s", j.jwt), nil } diff --git a/vendor/github.com/regclient/regclient/internal/pqueue/pqueue.go b/vendor/github.com/regclient/regclient/internal/pqueue/pqueue.go new file mode 100644 index 000000000..4d493f2f0 --- /dev/null +++ b/vendor/github.com/regclient/regclient/internal/pqueue/pqueue.go @@ -0,0 +1,286 @@ +// Package pqueue implements a priority queue. +package pqueue + +import ( + "context" + "fmt" + "sync" +) + +type Queue[T any] struct { + mu sync.Mutex + max int + next func(queued, active []*T) int + active []*T + queued []*T + wait []*chan struct{} +} + +// Opts is used to configure a new priority queue. +type Opts[T any] struct { + Max int // maximum concurrent entries, defaults to 1. + Next func(queued, active []*T) int // function to lookup index of next queued entry to release, defaults to oldest entry. +} + +// New creates a new priority queue. +func New[T any](opts Opts[T]) *Queue[T] { + if opts.Max <= 0 { + opts.Max = 1 + } + return &Queue[T]{ + max: opts.Max, + next: opts.Next, + } +} + +// Acquire adds a new entry to the queue and returns once it is ready. +// The returned function must be called when the queued job completes to release the next entry. +// If there is any error, the returned function will be nil. +func (q *Queue[T]) Acquire(ctx context.Context, e T) (func(), error) { + if q == nil { + return func() {}, nil + } + found, err := q.checkContext(ctx) + if err != nil { + return nil, err + } + if found { + return func() {}, nil + } + q.mu.Lock() + if len(q.active)+len(q.queued) < q.max { + q.active = append(q.active, &e) + q.mu.Unlock() + return q.releaseFn(&e), nil + } + // limit reached, add to queue and wait + w := make(chan struct{}, 1) + q.queued = append(q.queued, &e) + q.wait = append(q.wait, &w) + q.mu.Unlock() + // wait on both context and queue + select { + case <-ctx.Done(): + // context abort, remove queued entry + q.mu.Lock() + for i := range q.queued { + if q.queued[i] == &e { + if len(q.queued) >= i+1 { + q.queued = q.queued[:i] + q.wait = q.wait[:i] + } else { + q.queued = append(q.queued[:i], q.queued[i+1:]...) + q.wait = append(q.wait[:i], q.wait[i+1:]...) + } + q.mu.Unlock() + return nil, ctx.Err() + } + } + q.mu.Unlock() + // queued entry found, assume race condition with context and entry being released, release next entry + q.release(&e) + return nil, ctx.Err() + case <-w: + return q.releaseFn(&e), nil + } +} + +// TryAcquire attempts to add an entry on to the list of active entries. +// If the returned function is nil, the queue was not available. +// If the returned function is not nil, it must be called when the job is complete to release the next entry. +func (q *Queue[T]) TryAcquire(ctx context.Context, e T) (func(), error) { + if q == nil { + return func() {}, nil + } + found, err := q.checkContext(ctx) + if err != nil { + return nil, err + } + if found { + return func() {}, nil + } + q.mu.Lock() + defer q.mu.Unlock() + if len(q.active)+len(q.queued) < q.max { + q.active = append(q.active, &e) + return q.releaseFn(&e), nil + } + return nil, nil +} + +// release next entry or noop. +func (q *Queue[T]) release(prev *T) { + q.mu.Lock() + defer q.mu.Unlock() + // remove prev entry from active list + for i := range q.active { + if q.active[i] == prev { + if i == len(q.active)+1 { + q.active = q.active[:i] + } else { + q.active = append(q.active[:i], q.active[i+1:]...) + } + break + } + } + // skip checks when at limit or nothing queued + if len(q.queued) == 0 { + if len(q.active) == 0 { + // free up slices if this was the last active entry + q.active = nil + q.queued = nil + q.wait = nil + } + return + } + if len(q.active) >= q.max { + return + } + i := 0 + if q.next != nil && len(q.queued) > 1 { + i = q.next(q.queued, q.active) + // validate response + if i < 0 { + i = 0 + } + if i >= len(q.queued) { + i = len(q.queued) - 1 + } + } + // release queued entry, move to active list, and remove from queued/wait lists + close(*q.wait[i]) + q.active = append(q.active, q.queued[i]) + if i == len(q.queued)-1 { + q.queued = q.queued[:i] + q.wait = q.wait[:i] + } else { + q.queued = append(q.queued[:i], q.queued[i+1:]...) + q.wait = append(q.wait[:i], q.wait[i+1:]...) + } +} + +// releaseFn is a convenience wrapper around [release]. +func (q *Queue[T]) releaseFn(prev *T) func() { + return func() { + q.release(prev) + } +} + +// TODO: is there a way to make a different context key for each generic type? +type ctxType int + +var ctxKey ctxType + +type valMulti[T any] struct { + qList []*Queue[T] +} + +// AcquireMulti is used to simultaneously lock multiple queues without the risk of deadlock. +// The returned context needs to be used on calls to [Acquire] or [TryAcquire] which will immediately succeed since the resource is already acquired. +// Attempting to acquire other resources with [Acquire], [TryAcquire], or [AcquireMulti] using the returned context and will fail for being outside of the transaction. +// The returned function must be called to release the resources. +// The returned function is not thread safe, ensure no other simultaneous calls to [Acquire] or [TryAcquire] using the returned context have finished before it is called. +func AcquireMulti[T any](ctx context.Context, e T, qList ...*Queue[T]) (context.Context, func(), error) { + // verify context not already holding locks + qCtx := ctx.Value(ctxKey) + if qCtx != nil { + if qCtxVal, ok := qCtx.(*valMulti[T]); !ok || qCtxVal.qList != nil { + return ctx, nil, fmt.Errorf("context already used by another AcquireMulti request") + } + } + // delete nil entries + for i := len(qList) - 1; i >= 0; i-- { + if qList[i] == nil { + if i == len(qList)-1 { + qList = qList[:i] + } else { + qList = append(qList[:i], qList[i+1:]...) + } + } + } + // empty/nil list is a noop + if len(qList) == 0 { + return ctx, func() {}, nil + } + // dedup entries from the list + for i := len(qList) - 2; i >= 0; i-- { + for j := len(qList) - 1; j > i; j-- { + if qList[i] == qList[j] { + qList[j] = qList[len(qList)-1] + qList = qList[:len(qList)-1] + } + } + } + // Loop through queues to acquire, waiting on the first, and attempting the remaining. + // If any of the remaining entries cannot be immediately acquired, reset and make it the new queue to wait on. + lockI := 0 + doneList := make([]func(), len(qList)) + for { + acquired := true + i := 0 + done, err := qList[lockI].Acquire(ctx, e) + if err != nil { + return ctx, nil, err + } + doneList[lockI] = done + for i < len(qList) { + if i != lockI { + doneList[i], err = qList[i].TryAcquire(ctx, e) + if doneList[i] == nil || err != nil { + acquired = false + break + } + } + i++ + } + if err == nil && acquired { + break + } + // cleanup on failed attempt + if lockI > i { + doneList[lockI]() + } + // track blocking index for a retry + lockI = i + for i > 0 { + i-- + doneList[i]() + } + // abort on errors + if err != nil { + return ctx, nil, err + } + } + // success, update context + ctxVal := valMulti[T]{qList: qList} + newCtx := context.WithValue(ctx, ctxKey, &ctxVal) + cleanup := func() { + ctxVal.qList = nil + // dequeue in reverse order to minimize chance of another AcquireMulti being freed and immediately blocking on the next queue + for i := len(doneList) - 1; i >= 0; i-- { + doneList[i]() + } + } + return newCtx, cleanup, nil +} + +func (q *Queue[T]) checkContext(ctx context.Context) (bool, error) { + qCtx := ctx.Value(ctxKey) + if qCtx == nil { + return false, nil + } + qCtxVal, ok := qCtx.(*valMulti[T]) + if !ok { + return false, nil // another type is using the context, treat it as unset + } + if qCtxVal.qList == nil { + return false, nil + } + for _, cur := range qCtxVal.qList { + if cur == q { + // instance already locked + return true, nil + } + } + return true, fmt.Errorf("cannot acquire new locks during a transaction") +} diff --git a/vendor/github.com/regclient/regclient/internal/reghttp/http.go b/vendor/github.com/regclient/regclient/internal/reghttp/http.go index f5da2f7b4..e7bd207cb 100644 --- a/vendor/github.com/regclient/regclient/internal/reghttp/http.go +++ b/vendor/github.com/regclient/regclient/internal/reghttp/http.go @@ -25,99 +25,94 @@ import ( _ "crypto/sha256" _ "crypto/sha512" - "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" "github.com/regclient/regclient/config" "github.com/regclient/regclient/internal/auth" - "github.com/regclient/regclient/internal/throttle" + "github.com/regclient/regclient/internal/pqueue" + "github.com/regclient/regclient/internal/reqmeta" "github.com/regclient/regclient/types/errs" "github.com/regclient/regclient/types/warning" ) -var defaultDelayInit, _ = time.ParseDuration("1s") +var defaultDelayInit, _ = time.ParseDuration("0.1s") var defaultDelayMax, _ = time.ParseDuration("30s") var warnRegexp = regexp.MustCompile(`^299\s+-\s+"([^"]+)"`) const ( - DefaultRetryLimit = 3 + DefaultRetryLimit = 5 // number of times a request will be retried + backoffResetCount = 5 // number of successful requests needed to reduce the backoff ) -// Client is an HTTP client wrapper -// It handles features like authentication, retries, backoff delays, TLS settings +// Client is an HTTP client wrapper. +// It handles features like authentication, retries, backoff delays, TLS settings. type Client struct { - getConfigHost func(string) *config.Host - host map[string]*clientHost - httpClient *http.Client - rootCAPool [][]byte - rootCADirs []string - retryLimit int - delayInit time.Duration - delayMax time.Duration - log *logrus.Logger - userAgent string - mu sync.Mutex + httpClient *http.Client // upstream [http.Client], this is wrapped per repository for an auth handler on redirects + getConfigHost func(string) *config.Host // call-back to get the [config.Host] for a specific registry + host map[string]*clientHost // host specific settings, wrap access with a mutex lock + rootCAPool [][]byte // list of root CAs for configuring the http.Client transport + rootCADirs []string // list of directories for additional root CAs + retryLimit int // number of retries before failing a request, this applies to each host, and each request + delayInit time.Duration // how long to initially delay requests on a failure + delayMax time.Duration // maximum time to delay a request + log *logrus.Logger // logging for tracing and failures + userAgent string // user agent to specify in http request headers + mu sync.Mutex // mutex to prevent data races } type clientHost struct { - initialized bool - backoffCur int - backoffUntil time.Time - config *config.Host - httpClient *http.Client - auth map[string]auth.Auth - newAuth func() auth.Auth - mu sync.Mutex - ratelimit *time.Ticker -} - -// Req is a request to send to a registry + config *config.Host // config entry + httpClient *http.Client // modified http client for registry specific settings + userAgent string // user agent to specify in http request headers + log *logrus.Logger // logging for tracing and failures + auth map[string]*auth.Auth // map of auth handlers by repository + backoffCur int // current count of backoffs for this host + backoffLast time.Time // time the last request was released, this may be in the future if there is a queue, or zero if no delay is needed + backoffReset int // count of successful requests when a backoff is experienced, once [backoffResetCount] is reached, [backoffCur] is reduced by one and this is reset to 0 + reqFreq time.Duration // how long between submitting requests for this host + reqNext time.Time // time to release the next request + throttle *pqueue.Queue[reqmeta.Data] // limit concurrent requests to the host + mu sync.Mutex // mutex to prevent data races +} + +// Req is a request to send to a registry. type Req struct { - Host string - NoMirrors bool - APIs map[string]ReqAPI // allow different types of registries (registry/2.0, OCI, default to empty string) -} - -// ReqAPI handles API specific settings in a request -type ReqAPI struct { - Method string - DirectURL *url.URL - NoPrefix bool - Repository string - Path string - Query url.Values - BodyLen int64 - BodyBytes []byte - BodyFunc func() (io.ReadCloser, error) - Headers http.Header - Digest digest.Digest - IgnoreErr bool -} - -// Resp is used to handle the result of a request -type Resp interface { - io.ReadSeekCloser - HTTPResponse() *http.Response -} - -type clientResp struct { + MetaKind reqmeta.Kind // kind of request for the priority queue + Host string // registry name, hostname and mirrors will be looked up from host configuration + Method string // http method to call + DirectURL *url.URL // url to query, overrides repository, path, and query + Repository string // repository to scope the request + Path string // path of the request within a repository + Query url.Values // url query parameters + BodyLen int64 // length of body to send + BodyBytes []byte // bytes of the body, overridden by BodyFunc + BodyFunc func() (io.ReadCloser, error) // function to return a new body + Headers http.Header // headers to send in the request + NoPrefix bool // do not include the repository prefix + NoMirrors bool // do not send request to a mirror + ExpectLen int64 // expected size of the returned body + TransactLen int64 // size of an overall transaction for the priority queue + IgnoreErr bool // ignore http errors and do not trigger backoffs +} + +// Resp is used to handle the result of a request. +type Resp struct { ctx context.Context client *Client req *Req resp *http.Response mirror string done bool - digest digest.Digest - digester digest.Digester reader io.Reader readCur, readMax int64 - throttle *throttle.Throttle + retryCount int + throttleDone func() } -// Opts is used to configure client options +// Opts is used to configure client options. type Opts func(*Client) -// NewClient returns a client for handling requests +// NewClient returns a client for handling requests. func NewClient(opts ...Opts) *Client { c := Client{ httpClient: &http.Client{}, @@ -135,21 +130,21 @@ func NewClient(opts ...Opts) *Client { return &c } -// WithCerts adds certificates +// WithCerts adds certificates. func WithCerts(certs [][]byte) Opts { return func(c *Client) { c.rootCAPool = append(c.rootCAPool, certs...) } } -// WithCertDirs adds directories to check for host specific certs +// WithCertDirs adds directories to check for host specific certs. func WithCertDirs(dirs []string) Opts { return func(c *Client) { c.rootCADirs = append(c.rootCADirs, dirs...) } } -// WithCertFiles adds certificates by filename +// WithCertFiles adds certificates by filename. func WithCertFiles(files []string) Opts { return func(c *Client) { for _, f := range files { @@ -167,14 +162,15 @@ func WithCertFiles(files []string) Opts { } } -// WithConfigHost adds the callback to request a config.Host struct -func WithConfigHost(gch func(string) *config.Host) Opts { +// WithConfigHostFn adds the callback to request a [config.Host] struct. +// The function must normalize the hostname for Docker Hub support. +func WithConfigHostFn(gch func(string) *config.Host) Opts { return func(c *Client) { c.getConfigHost = gch } } -// WithDelay initial time to wait between retries (increased with exponential backoff) +// WithDelay initial time to wait between retries (increased with exponential backoff). func WithDelay(delayInit time.Duration, delayMax time.Duration) Opts { return func(c *Client) { if delayInit > 0 { @@ -191,14 +187,14 @@ func WithDelay(delayInit time.Duration, delayMax time.Duration) Opts { } } -// WithHTTPClient uses a specific http client with retryable requests +// WithHTTPClient uses a specific http client with retryable requests. func WithHTTPClient(hc *http.Client) Opts { return func(c *Client) { c.httpClient = hc } } -// WithRetryLimit restricts the number of retries (defaults to 5) +// WithRetryLimit restricts the number of retries (defaults to 5). func WithRetryLimit(rl int) Opts { return func(c *Client) { if rl > 0 { @@ -207,40 +203,42 @@ func WithRetryLimit(rl int) Opts { } } -// WithLog injects a logrus Logger configuration +// WithLog injects a logrus Logger configuration. func WithLog(log *logrus.Logger) Opts { return func(c *Client) { c.log = log } } -// WithTransport uses a specific http transport with retryable requests +// WithTransport uses a specific http transport with retryable requests. func WithTransport(t *http.Transport) Opts { return func(c *Client) { c.httpClient = &http.Client{Transport: t} } } -// WithUserAgent sets a user agent header +// WithUserAgent sets a user agent header. func WithUserAgent(ua string) Opts { return func(c *Client) { c.userAgent = ua } } -// Do runs a request, returning the response result -func (c *Client) Do(ctx context.Context, req *Req) (Resp, error) { - resp := &clientResp{ - ctx: ctx, - client: c, - req: req, +// Do runs a request, returning the response result. +func (c *Client) Do(ctx context.Context, req *Req) (*Resp, error) { + resp := &Resp{ + ctx: ctx, + client: c, + req: req, + readCur: 0, + readMax: req.ExpectLen, } - err := resp.Next() + err := resp.next() return resp, err } -// Next sends requests until a mirror responds or all requests fail -func (resp *clientResp) Next() error { +// next sends requests until a mirror responds or all requests fail. +func (resp *Resp) next() error { var err error c := resp.client req := resp.req @@ -272,11 +270,11 @@ func (resp *clientResp) Next() error { } h := hosts[curHost] resp.mirror = h.config.Name - - api, okAPI := req.APIs[h.config.API] - if !okAPI { - api, okAPI = req.APIs[""] + // there is an intentional extra retry in this check to allow for auth requests + if resp.retryCount > c.retryLimit { + return errs.ErrRetryLimitExceeded } + resp.retryCount++ // check that context isn't canceled/done ctxErr := resp.ctx.Err() @@ -284,7 +282,10 @@ func (resp *clientResp) Next() error { return ctxErr } // wait for other concurrent requests to this host - throttleErr := h.config.Throttle().Acquire(resp.ctx) + throttleDone, throttleErr := h.throttle.Acquire(resp.ctx, reqmeta.Data{ + Kind: req.MetaKind, + Size: req.BodyLen + req.ExpectLen + req.TransactLen, + }) if throttleErr != nil { return throttleErr } @@ -292,11 +293,7 @@ func (resp *clientResp) Next() error { // try each host in a closure to handle all the backoff/dropHost from one place loopErr := func() error { var err error - if !okAPI { - dropHost = true - return fmt.Errorf("failed looking up api \"%s\" for host \"%s\": %w", h.config.API, h.config.Name, errs.ErrAPINotFound) - } - if api.Method == "HEAD" && h.config.APIOpts != nil { + if req.Method == "HEAD" && h.config.APIOpts != nil { var disableHead bool disableHead, err = strconv.ParseBool(h.config.APIOpts["disableHead"]) if err == nil && disableHead { @@ -305,16 +302,10 @@ func (resp *clientResp) Next() error { } } - // store the desired digest and setup digester at first byte - resp.digest = api.Digest - if resp.readCur == 0 && resp.digest.Validate() == nil { - resp.digester = resp.digest.Algorithm().Digester() - } - // build the url var u url.URL - if api.DirectURL != nil { - u = *api.DirectURL + if req.DirectURL != nil { + u = *req.DirectURL } else { u = url.URL{ Host: h.config.Hostname, @@ -322,19 +313,19 @@ func (resp *clientResp) Next() error { } path := strings.Builder{} path.WriteString("/v2") - if h.config.PathPrefix != "" && !api.NoPrefix { + if h.config.PathPrefix != "" && !req.NoPrefix { path.WriteString("/" + h.config.PathPrefix) } - if api.Repository != "" { - path.WriteString("/" + api.Repository) + if req.Repository != "" { + path.WriteString("/" + req.Repository) } - path.WriteString("/" + api.Path) + path.WriteString("/" + req.Path) u.Path = path.String() if h.config.TLS == config.TLSDisabled { u.Scheme = "http" } - if api.Query != nil { - u.RawQuery = api.Query.Encode() + if req.Query != nil { + u.RawQuery = req.Query.Encode() } } // close previous response @@ -342,13 +333,13 @@ func (resp *clientResp) Next() error { _ = resp.resp.Body.Close() } // delay for backoff if needed - bu := resp.backoffUntil() + bu := resp.backoffGet() if !bu.IsZero() && bu.After(time.Now()) { sleepTime := time.Until(bu) c.log.WithFields(logrus.Fields{ "Host": h.config.Name, "Seconds": sleepTime.Seconds(), - }).Warn("Sleeping for backoff") + }).Debug("Sleeping for backoff") select { case <-resp.ctx.Done(): return errs.ErrCanceled @@ -356,47 +347,48 @@ func (resp *clientResp) Next() error { } } var httpReq *http.Request - httpReq, err = http.NewRequestWithContext(resp.ctx, api.Method, u.String(), nil) + httpReq, err = http.NewRequestWithContext(resp.ctx, req.Method, u.String(), nil) if err != nil { dropHost = true return err } - if api.BodyFunc != nil { - body, err := api.BodyFunc() + if req.BodyFunc != nil { + body, err := req.BodyFunc() if err != nil { dropHost = true return err } httpReq.Body = body - httpReq.GetBody = api.BodyFunc - httpReq.ContentLength = api.BodyLen - } else if len(api.BodyBytes) > 0 { - body := io.NopCloser(bytes.NewReader(api.BodyBytes)) + httpReq.GetBody = req.BodyFunc + httpReq.ContentLength = req.BodyLen + } else if len(req.BodyBytes) > 0 { + body := io.NopCloser(bytes.NewReader(req.BodyBytes)) httpReq.Body = body httpReq.GetBody = func() (io.ReadCloser, error) { return body, nil } - httpReq.ContentLength = api.BodyLen + httpReq.ContentLength = req.BodyLen } - if len(api.Headers) > 0 { - httpReq.Header = api.Headers.Clone() + if len(req.Headers) > 0 { + httpReq.Header = req.Headers.Clone() } if c.userAgent != "" && httpReq.Header.Get("User-Agent") == "" { httpReq.Header.Add("User-Agent", c.userAgent) } if resp.readCur > 0 && resp.readMax > 0 { - if httpReq.Header.Get("Range") == "" { + if req.Headers.Get("Range") == "" { httpReq.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", resp.readCur, resp.readMax)) } else { + // TODO: support Seek within a range request dropHost = true return fmt.Errorf("unable to resume a connection within a range request") } } - hAuth := h.getAuth(api.Repository) + hAuth := h.getAuth(req.Repository) if hAuth != nil { // include docker generated scope to emulate docker clients - if api.Repository != "" { - scope := "repository:" + api.Repository + ":pull" - if api.Method != "HEAD" && api.Method != "GET" { + if req.Repository != "" { + scope := "repository:" + req.Repository + ":pull" + if req.Method != "HEAD" && req.Method != "GET" { scope = scope + ",push" } _ = hAuth.AddScope(h.config.Hostname, scope) @@ -414,20 +406,24 @@ func (resp *clientResp) Next() error { } // delay for the rate limit - if h.ratelimit != nil { - <-h.ratelimit.C + if h.reqFreq > 0 { + sleep := time.Duration(0) + h.mu.Lock() + if time.Now().Before(h.reqNext) { + sleep = time.Until(h.reqNext) + h.reqNext = h.reqNext.Add(h.reqFreq) + } else { + h.reqNext = time.Now().Add(h.reqFreq) + } + h.mu.Unlock() + if sleep > 0 { + time.Sleep(sleep) + } } - // update http client for insecure requests and root certs - httpClient := *h.httpClient - // send request - resp.client.log.WithFields(logrus.Fields{ - "url": httpReq.URL.String(), - "method": httpReq.Method, - "withAuth": (len(httpReq.Header.Values("Authorization")) > 0), - }).Debug("http req") - resp.resp, err = httpClient.Do(httpReq) + hc := h.getHTTPClient(req.Repository) + resp.resp, err = hc.Do(httpReq) if err != nil { c.log.WithFields(logrus.Fields{ @@ -437,13 +433,7 @@ func (resp *clientResp) Next() error { backoff = true return err } - // extract any warnings - for _, wh := range resp.resp.Header.Values("Warning") { - if match := warnRegexp.FindStringSubmatch(wh); len(match) == 2 { - // TODO: pass other fields (registry hostname) with structured logging - warning.Handle(resp.ctx, resp.client.log, match[1]) - } - } + statusCode := resp.resp.StatusCode if statusCode < 200 || statusCode >= 300 { switch statusCode { @@ -478,7 +468,7 @@ func (resp *clientResp) Next() error { case http.StatusRequestedRangeNotSatisfiable: // if range request error (blob push), drop mirror for this req, but other requests don't need backoff dropHost = true - case http.StatusTooManyRequests, http.StatusRequestTimeout, http.StatusGatewayTimeout, http.StatusInternalServerError: + case http.StatusTooManyRequests, http.StatusRequestTimeout, http.StatusGatewayTimeout, http.StatusBadGateway, http.StatusInternalServerError: // server is likely overloaded, backoff but still retry backoff = true default: @@ -486,27 +476,28 @@ func (resp *clientResp) Next() error { backoff = true dropHost = true } - c.log.WithFields(logrus.Fields{ - "URL": u.String(), - "Status": http.StatusText(statusCode), - }).Debug("Request failed") errHTTP := HTTPError(resp.resp.StatusCode) errBody, _ := io.ReadAll(resp.resp.Body) _ = resp.resp.Body.Close() return fmt.Errorf("request failed: %w: %s", errHTTP, errBody) } - // setup reader, with a digester if configured - if resp.digester == nil { - resp.reader = resp.resp.Body - } else { - resp.reader = io.TeeReader(resp.resp.Body, resp.digester.Hash()) - } + resp.reader = resp.resp.Body resp.done = false // set variables from headers if found - if resp.readCur == 0 && resp.readMax == 0 && resp.resp.Header.Get("Content-Length") != "" { - cl, parseErr := strconv.ParseInt(resp.resp.Header.Get("Content-Length"), 10, 64) - if parseErr == nil { + clHeader := resp.resp.Header.Get("Content-Length") + if resp.readCur == 0 && clHeader != "" { + cl, parseErr := strconv.ParseInt(clHeader, 10, 64) + if parseErr != nil { + c.log.WithFields(logrus.Fields{ + "err": err, + "header": clHeader, + }).Debug("failed to parse content-length header") + } else if resp.readMax > 0 { + if resp.readMax != cl { + return fmt.Errorf("unexpected content-length, expected %d, received %d", resp.readMax, cl) + } + } else { resp.readMax = cl } } @@ -520,16 +511,12 @@ func (resp *clientResp) Next() error { }() // return on success if loopErr == nil { - resp.throttle = h.config.Throttle() + resp.throttleDone = throttleDone return nil } // backoff, dropHost, and/or go to next host in the list - throttleErr = h.config.Throttle().Release(resp.ctx) - if throttleErr != nil { - return throttleErr - } if backoff { - if api.IgnoreErr { + if req.IgnoreErr { // don't set a backoff, immediately drop the host when errors ignored dropHost = true } else { @@ -540,6 +527,7 @@ func (resp *clientResp) Next() error { } } } + throttleDone() // when error does not allow retries, abort with the last known err value if err != nil && errors.Is(loopErr, errs.ErrNotRetryable) { return err @@ -553,11 +541,20 @@ func (resp *clientResp) Next() error { } } -func (resp *clientResp) HTTPResponse() *http.Response { +// GetThrottle returns the current [pqueue.Queue] for a host used to throttle connections. +// This can be used to acquire multiple throttles before performing a request across multiple hosts. +func (c *Client) GetThrottle(host string) *pqueue.Queue[reqmeta.Data] { + ch := c.getHost(host) + return ch.throttle +} + +// HTTPResponse returns the [http.Response] from the last request. +func (resp *Resp) HTTPResponse() *http.Response { return resp.resp } -func (resp *clientResp) Read(b []byte) (int, error) { +// Read provides a retryable read from the body of the response. +func (resp *Resp) Read(b []byte) (int, error) { if resp.done { return 0, io.EOF } @@ -569,7 +566,7 @@ func (resp *clientResp) Read(b []byte) (int, error) { resp.readCur += int64(i) if err == io.EOF || err == io.ErrUnexpectedEOF { if resp.resp.Request.Method == "HEAD" || resp.readCur >= resp.readMax { - resp.backoffClear() + resp.backoffReset() resp.done = true } else { // short read, retry? @@ -580,7 +577,7 @@ func (resp *clientResp) Read(b []byte) (int, error) { // retry respErr := resp.backoffSet() if respErr == nil { - respErr = resp.Next() + respErr = resp.next() } // unrecoverable EOF if respErr != nil { @@ -593,17 +590,6 @@ func (resp *clientResp) Read(b []byte) (int, error) { // retry successful, no EOF return i, nil } - // validate the digest if specified - if resp.resp.Request.Method != "HEAD" && resp.digester != nil && resp.digest.Validate() == nil && resp.digest != resp.digester.Digest() { - resp.client.log.WithFields(logrus.Fields{ - "expected": resp.digest, - "computed": resp.digester.Digest(), - }).Warn("Digest mismatch") - _ = resp.backoffSet() - resp.done = true - return i, fmt.Errorf("%w, expected %s, computed %s", errs.ErrDigestMismatch, - resp.digest.String(), resp.digester.Digest().String()) - } } if err == nil { @@ -612,22 +598,24 @@ func (resp *clientResp) Read(b []byte) (int, error) { return i, err } -func (resp *clientResp) Close() error { - if resp.throttle != nil { - _ = resp.throttle.Release(resp.ctx) - resp.throttle = nil +// Close frees up resources from the request. +func (resp *Resp) Close() error { + if resp.throttleDone != nil { + resp.throttleDone() + resp.throttleDone = nil } if resp.resp == nil { return errs.ErrNotFound } if !resp.done { - resp.backoffClear() + resp.backoffReset() } resp.done = true return resp.resp.Body.Close() } -func (resp *clientResp) Seek(offset int64, whence int) (int64, error) { +// Seek provides a limited ability seek within the request response. +func (resp *Resp) Seek(offset int64, whence int) (int64, error) { newOffset := resp.readCur switch whence { case io.SeekStart: @@ -644,60 +632,66 @@ func (resp *clientResp) Seek(offset int64, whence int) (int64, error) { default: return resp.readCur, fmt.Errorf("unknown value of whence: %d", whence) } - if newOffset == 0 { - resp.readCur = 0 + if newOffset != resp.readCur { + resp.readCur = newOffset // rerun the request to restart - err := resp.Next() + resp.retryCount-- // do not count a seek as a retry + err := resp.next() if err != nil { return resp.readCur, err } - } else if newOffset != resp.readCur { - return resp.readCur, fmt.Errorf("seek to arbitrary position is not supported") } return resp.readCur, nil } -func (resp *clientResp) backoffClear() { +func (resp *Resp) backoffGet() time.Time { c := resp.client - c.mu.Lock() - defer c.mu.Unlock() - ch := c.host[resp.mirror] - if ch.backoffCur > c.retryLimit { - ch.backoffCur = c.retryLimit - } + ch := c.getHost(resp.mirror) + ch.mu.Lock() + defer ch.mu.Unlock() if ch.backoffCur > 0 { - ch.backoffCur-- - if ch.backoffCur == 0 { - ch.backoffUntil = time.Time{} + delay := c.delayInit << ch.backoffCur + if delay > c.delayMax { + delay = c.delayMax + } + next := ch.backoffLast.Add(delay) + now := time.Now() + if now.After(next) { + next = now } + ch.backoffLast = next + return next } + // reset a stale "retry-after" time + if !ch.backoffLast.IsZero() && ch.backoffLast.Before(time.Now()) { + ch.backoffLast = time.Time{} + } + return ch.backoffLast } -func (resp *clientResp) backoffSet() error { +func (resp *Resp) backoffSet() error { c := resp.client - c.mu.Lock() - defer c.mu.Unlock() - ch := c.host[resp.mirror] - ch.backoffCur++ - // sleep for backoff time - sleepTime := c.delayInit << ch.backoffCur - // limit to max delay - if sleepTime > c.delayMax { - sleepTime = c.delayMax - } - // check rate limit header + ch := c.getHost(resp.mirror) + ch.mu.Lock() + defer ch.mu.Unlock() + // check rate limit header and use that directly if possible if resp.resp != nil && resp.resp.Header.Get("Retry-After") != "" { ras := resp.resp.Header.Get("Retry-After") ra, _ := time.ParseDuration(ras + "s") - if ra > c.delayMax { - sleepTime = c.delayMax - } else if ra > sleepTime { - sleepTime = ra + if ra > 0 { + next := time.Now().Add(ra) + if ch.backoffLast.Before(next) { + ch.backoffLast = next + } + return nil } } - - ch.backoffUntil = time.Now().Add(sleepTime) - + // Else track the number of backoffs and fail when the limit is exceeded. + // New requests always get at least one try, but fail fast if the server has been throwing errors. + ch.backoffCur++ + if ch.backoffLast.IsZero() { + ch.backoffLast = time.Now() + } if ch.backoffCur >= c.retryLimit { return fmt.Errorf("%w: backoffs %d", errs.ErrBackoffLimit, ch.backoffCur) } @@ -705,118 +699,153 @@ func (resp *clientResp) backoffSet() error { return nil } -func (resp *clientResp) backoffUntil() time.Time { +func (resp *Resp) backoffReset() { c := resp.client - c.mu.Lock() - defer c.mu.Unlock() - ch := c.host[resp.mirror] - return ch.backoffUntil + ch := c.getHost(resp.mirror) + ch.mu.Lock() + defer ch.mu.Unlock() + if ch.backoffCur > 0 { + ch.backoffReset++ + // If enough successful requests are seen, lower the backoffCur count. + // This requires multiple successful requests of a flaky server, but quickly drops when above the retry limit. + if ch.backoffReset > backoffResetCount || ch.backoffCur > c.retryLimit { + ch.backoffReset = 0 + ch.backoffCur-- + if ch.backoffCur == 0 { + // reset the last time to the zero value + ch.backoffLast = time.Time{} + } + } + } } +// getHost looks up or creates a clientHost for a given registry. func (c *Client) getHost(host string) *clientHost { c.mu.Lock() defer c.mu.Unlock() - h, ok := c.host[host] - if ok && h.initialized { + if h, ok := c.host[host]; ok { return h } - if !ok { - h = &clientHost{} + var conf *config.Host + if c.getConfigHost != nil { + conf = c.getConfigHost(host) + } else { + conf = config.HostNewName(host) } - if h.config == nil { - if c.getConfigHost != nil { - h.config = c.getConfigHost(host) - } else { - h.config = config.HostNewName(host) - } - // check for normalized hostname - if h.config.Name != host { - host = h.config.Name - hNormal, ok := c.host[host] - if ok && hNormal.initialized { - return hNormal - } + if conf.Name != host { + if h, ok := c.host[conf.Name]; ok { + return h } } - if h.auth == nil { - h.auth = map[string]auth.Auth{} - } - if h.ratelimit == nil && h.config.ReqPerSec > 0 { - h.ratelimit = time.NewTicker(time.Duration(float64(time.Second) / h.config.ReqPerSec)) - } - - if h.httpClient == nil { - h.httpClient = c.httpClient - // update http client for insecure requests and root certs - if h.config.TLS == config.TLSInsecure || len(c.rootCAPool) > 0 || len(c.rootCADirs) > 0 || h.config.RegCert != "" || (h.config.ClientCert != "" && h.config.ClientKey != "") { - // create a new client and modify the transport - httpClient := *c.httpClient - if httpClient.Transport == nil { - httpClient.Transport = http.DefaultTransport.(*http.Transport).Clone() + h := &clientHost{ + config: conf, + userAgent: c.userAgent, + log: c.log, + auth: map[string]*auth.Auth{}, + } + if h.config.ReqPerSec > 0 { + h.reqFreq = time.Duration(float64(time.Second) / h.config.ReqPerSec) + } + if h.config.ReqConcurrent > 0 { + h.throttle = pqueue.New(pqueue.Opts[reqmeta.Data]{Max: int(h.config.ReqConcurrent), Next: reqmeta.DataNext}) + } + // copy the http client and configure registry specific settings + hc := *c.httpClient + h.httpClient = &hc + if h.httpClient.Transport == nil { + h.httpClient.Transport = http.DefaultTransport.(*http.Transport).Clone() + } + // configure transport for insecure requests and root certs + if h.config.TLS == config.TLSInsecure || len(c.rootCAPool) > 0 || len(c.rootCADirs) > 0 || h.config.RegCert != "" || (h.config.ClientCert != "" && h.config.ClientKey != "") { + t, ok := h.httpClient.Transport.(*http.Transport) + if ok { + var tlsc *tls.Config + if t.TLSClientConfig != nil { + tlsc = t.TLSClientConfig.Clone() + } else { + //#nosec G402 the default TLS 1.2 minimum version is allowed to support older registries + tlsc = &tls.Config{} } - t, ok := httpClient.Transport.(*http.Transport) - if ok { - var tlsc *tls.Config - if t.TLSClientConfig != nil { - tlsc = t.TLSClientConfig.Clone() + if h.config.TLS == config.TLSInsecure { + tlsc.InsecureSkipVerify = true + } else { + rootPool, err := makeRootPool(c.rootCAPool, c.rootCADirs, h.config.Hostname, h.config.RegCert) + if err != nil { + c.log.WithFields(logrus.Fields{ + "err": err, + }).Warn("failed to setup CA pool") } else { - //#nosec G402 the default TLS 1.2 minimum version is allowed to support older registries - tlsc = &tls.Config{} + tlsc.RootCAs = rootPool } - if h.config.TLS == config.TLSInsecure { - tlsc.InsecureSkipVerify = true + } + if h.config.ClientCert != "" && h.config.ClientKey != "" { + cert, err := tls.X509KeyPair([]byte(h.config.ClientCert), []byte(h.config.ClientKey)) + if err != nil { + c.log.WithFields(logrus.Fields{ + "err": err, + }).Warn("failed to configure client certs") } else { - rootPool, err := makeRootPool(c.rootCAPool, c.rootCADirs, h.config.Hostname, h.config.RegCert) - if err != nil { - c.log.WithFields(logrus.Fields{ - "err": err, - }).Warn("failed to setup CA pool") - } else { - tlsc.RootCAs = rootPool - } - } - if h.config.ClientCert != "" && h.config.ClientKey != "" { - cert, err := tls.X509KeyPair([]byte(h.config.ClientCert), []byte(h.config.ClientKey)) - if err != nil { - c.log.WithFields(logrus.Fields{ - "err": err, - }).Warn("failed to configure client certs") - } else { - tlsc.Certificates = []tls.Certificate{cert} - } + tlsc.Certificates = []tls.Certificate{cert} } - t.TLSClientConfig = tlsc - httpClient.Transport = t } - h.httpClient = &httpClient + t.TLSClientConfig = tlsc + h.httpClient.Transport = t } } + // wrap the transport for logging and to handle warning headers + h.httpClient.Transport = &wrapTransport{c: c, orig: h.httpClient.Transport} - if h.newAuth == nil { - h.newAuth = func() auth.Auth { - return auth.NewAuth( - auth.WithLog(c.log), - auth.WithHTTPClient(h.httpClient), - auth.WithCreds(h.AuthCreds()), - auth.WithClientID(c.userAgent), - ) - } + c.host[conf.Name] = h + if conf.Name != host { + // save another reference for faster lookups + c.host[host] = h } - - h.initialized = true - c.host[host] = h return h } -// getAuth returns an auth, which may be repository specific -func (ch *clientHost) getAuth(repo string) auth.Auth { +// getHTTPClient returns a client specific to the repo being queried. +// Repository specific authentication needs a dedicated CheckRedirect handler. +func (ch *clientHost) getHTTPClient(repo string) *http.Client { + hc := *ch.httpClient + hc.CheckRedirect = ch.checkRedirect(repo, hc.CheckRedirect) + return &hc +} + +// checkRedirect wraps http.CheckRedirect to inject auth headers to specific hosts in the redirect chain +func (ch *clientHost) checkRedirect(repo string, orig func(req *http.Request, via []*http.Request) error) func(req *http.Request, via []*http.Request) error { + return func(req *http.Request, via []*http.Request) error { + // fail on too many redirects + if len(via) >= 10 { + return errors.New("stopped after 10 redirects") + } + // add auth headers if appropriate for the target host + hAuth := ch.getAuth(repo) + err := hAuth.UpdateRequest(req) + if err != nil { + return err + } + // wrap original redirect check + if orig != nil { + return orig(req, via) + } + return nil + } +} + +// getAuth returns an auth, which may be repository specific. +func (ch *clientHost) getAuth(repo string) *auth.Auth { ch.mu.Lock() defer ch.mu.Unlock() if !ch.config.RepoAuth { repo = "" // without RepoAuth, unset the provided repo } if _, ok := ch.auth[repo]; !ok { - ch.auth[repo] = ch.newAuth() + ch.auth[repo] = auth.NewAuth( + auth.WithLog(ch.log), + auth.WithHTTPClient(ch.httpClient), + auth.WithCreds(ch.AuthCreds()), + auth.WithClientID(ch.userAgent), + ) } return ch.auth[repo] } @@ -831,7 +860,45 @@ func (ch *clientHost) AuthCreds() func(h string) auth.Cred { } } -// HTTPError returns an error based on the status code +type wrapTransport struct { + c *Client + orig http.RoundTripper +} + +func (wt *wrapTransport) RoundTrip(req *http.Request) (*http.Response, error) { + resp, err := wt.orig.RoundTrip(req) + // copy headers to censor auth field + reqHead := req.Header.Clone() + if reqHead.Get("Authorization") != "" { + reqHead.Set("Authorization", "[censored]") + } + if err != nil { + wt.c.log.WithFields(logrus.Fields{ + "req-method": req.Method, + "req-url": req.URL.String(), + "req-headers": reqHead, + "err": err, + }).Debug("reg http request") + } else { + // extract any warnings + for _, wh := range resp.Header.Values("Warning") { + if match := warnRegexp.FindStringSubmatch(wh); len(match) == 2 { + // TODO(bmitch): pass other fields (registry hostname) with structured logging + warning.Handle(req.Context(), wt.c.log, match[1]) + } + } + wt.c.log.WithFields(logrus.Fields{ + "req-method": req.Method, + "req-url": req.URL.String(), + "req-headers": reqHead, + "resp-status": resp.Status, + "resp-headers": resp.Header, + }).Trace("reg http request") + } + return resp, err +} + +// HTTPError returns an error based on the status code. func HTTPError(statusCode int) error { switch statusCode { case 401: @@ -898,13 +965,13 @@ func makeRootPool(rootCAPool [][]byte, rootCADirs []string, hostname string, hos return pool, nil } -// sortHostCmp to sort host list of mirrors +// sortHostCmp to sort host list of mirrors. func sortHostsCmp(hosts []*clientHost, upstream string) func(i, j int) bool { now := time.Now() // sort by backoff first, then priority decending, then upstream name last return func(i, j int) bool { - if now.Before(hosts[i].backoffUntil) || now.Before(hosts[j].backoffUntil) { - return hosts[i].backoffUntil.Before(hosts[j].backoffUntil) + if now.Before(hosts[i].backoffLast) || now.Before(hosts[j].backoffLast) { + return hosts[i].backoffLast.Before(hosts[j].backoffLast) } if hosts[i].config.Priority != hosts[j].config.Priority { return hosts[i].config.Priority < hosts[j].config.Priority diff --git a/vendor/github.com/regclient/regclient/internal/reqmeta/data.go b/vendor/github.com/regclient/regclient/internal/reqmeta/data.go new file mode 100644 index 000000000..2a7b94a90 --- /dev/null +++ b/vendor/github.com/regclient/regclient/internal/reqmeta/data.go @@ -0,0 +1,88 @@ +// Package reqmeta provides metadata on requests for prioritizing with a pqueue. +package reqmeta + +type Data struct { + Kind Kind + Size int64 +} + +type Kind int + +const ( + Unknown Kind = iota + Head + Manifest + Query + Blob +) + +const ( + smallLimit = 4194304 // 4MiB + largePct = 0.9 // anything above 90% of largest queued entry size is large +) + +func DataNext(queued, active []*Data) int { + if len(queued) == 0 { + return -1 + } + // After removing one small entry, split remaining requests 50/50 between large and old (truncated int division always rounds down). + // If len active = 2, this function returns the 3rd entry (+1), minus 1 for the small, divide by 2 to split with old = goal of 1. + largeGoal := len(active) / 2 + largeI := 0 + var largeSize int64 + if largeGoal > 0 { + // find the largest queued blob requests + for i, cur := range queued { + if cur.Kind == Blob && cur.Size > largeSize { + largeI = i + largeSize = cur.Size + } + } + } + largeCutoff := int64(float64(largeSize) * 0.9) + // count active requests by type + small := 0 + large := 0 + old := 0 + for _, cur := range active { + if cur.Kind != Blob && cur.Size <= smallLimit { + small++ + } else if cur.Kind == Blob && largeSize > 0 && cur.Size >= largeCutoff { + large++ + } else { + old++ + } + } + // if there is at least one active, and none are small, return the best small entry if available. + if len(active) > 0 && small == 0 { + var sizeI int64 + bestI := -1 + kindI := Unknown + for i, cur := range queued { + // the small search skips blobs and large requests + if cur.Kind == Blob || cur.Size > smallLimit { + continue + } + // the best small entry is the: + // - first one found if no other matches + // - one with a better Kind (Head > Manifest > Query) + // - one with the same kind but smaller request + if bestI < 0 || + (cur.Kind != Unknown && (kindI == Unknown || cur.Kind < kindI)) || + (cur.Kind == kindI && cur.Size > 0 && (cur.Size < sizeI || sizeI <= 0)) { + bestI = i + kindI = cur.Kind + sizeI = cur.Size + } + } + if bestI >= 0 { + return bestI + } + } + // Prefer the biggest of these blobs to minimize the size of the last running blob. + if largeGoal > 0 && large < largeGoal && largeSize > 0 { + return largeI + } + // enough small and large, or none available, so return the oldest queued entry to avoid starvation. + return 0 +} diff --git a/vendor/github.com/regclient/regclient/internal/throttle/throttle.go b/vendor/github.com/regclient/regclient/internal/throttle/throttle.go deleted file mode 100644 index 9c6b0f7c8..000000000 --- a/vendor/github.com/regclient/regclient/internal/throttle/throttle.go +++ /dev/null @@ -1,192 +0,0 @@ -// Package throttle is used to limit concurrent activities -package throttle - -import ( - "context" - "fmt" -) - -type token struct{} -type Throttle struct { - ch chan token -} -type key int -type valMany struct { - tList []*Throttle -} - -var keyMany key - -func New(count int) *Throttle { - ch := make(chan token, count) - return &Throttle{ch: ch} -} - -func (t *Throttle) checkContext(ctx context.Context) (bool, error) { - tCtx := ctx.Value(keyMany) - if tCtx == nil { - return false, nil - } - tCtxVal, ok := tCtx.(*valMany) - if !ok { - return true, fmt.Errorf("context value is not a throttle list") - } - if tCtxVal.tList == nil { - return false, nil - } - for _, cur := range tCtxVal.tList { - if cur == t { - // instance already locked - return true, nil - } - } - return true, fmt.Errorf("cannot acquire new locks during a transaction") -} - -func (t *Throttle) Acquire(ctx context.Context) error { - if t == nil { - return nil - } - // check if already acquired in context - if found, err := t.checkContext(ctx); found { - return err - } - select { - case <-ctx.Done(): - return ctx.Err() - case t.ch <- token{}: - return nil - } -} - -func (t *Throttle) Release(ctx context.Context) error { - if t == nil { - return nil - } - // check if already acquired in context - if found, err := t.checkContext(ctx); found { - return err - } - select { - case <-t.ch: - return nil - default: - return fmt.Errorf("failed to release throttle") - } -} - -func (t *Throttle) TryAcquire(ctx context.Context) (bool, error) { - if t == nil { - return true, nil - } - // check if already acquired in context - if found, err := t.checkContext(ctx); found { - return err == nil, err - } - select { - case t.ch <- token{}: - return true, nil - default: - return false, nil - } -} - -func AcquireMulti(ctx context.Context, tList []*Throttle) (context.Context, error) { - // verify context not already holding locks - tCtx := ctx.Value(keyMany) - if tCtx != nil { - if tCtxVal, ok := tCtx.(*valMany); ok && tCtxVal.tList != nil { - return ctx, fmt.Errorf("throttle cannot manage concurrent transactions") - } - } - if len(tList) <= 0 { - // noop? - return ctx, nil - } - // dedup entries from the list - for i := len(tList) - 2; i >= 0; i-- { - for j := len(tList) - 1; j > i; j-- { - if tList[i] == tList[j] { - // delete j from the list - tList[j] = tList[len(tList)-1] - tList = tList[:len(tList)-1] - } - } - } - lockI := 0 - for { - err := tList[lockI].Acquire(ctx) - if err != nil { - return ctx, err - } - acquired := true - i := 0 - for i < len(tList) { - if i != lockI { - acquired, err = tList[i].TryAcquire(ctx) - if err != nil || !acquired { - break - } - } - i++ - } - if err == nil && acquired { - break - } - // TODO: errors on Release should be included using errors.Join once 1.20 is the minimum version - // cleanup on failed attempt - if lockI > i { - _ = tList[lockI].Release(ctx) - } - // track blocking index - lockI = i - for i > 0 { - i-- - _ = tList[i].Release(ctx) - } - // abort on errors - if err != nil { - return ctx, err - } - } - // success, update context - newCtx := context.WithValue(ctx, keyMany, &valMany{tList: tList}) - return newCtx, nil -} - -func ReleaseMulti(ctx context.Context, tList []*Throttle) error { - // verify context shows locked values - tCtx := ctx.Value(keyMany) - if tCtx == nil { - return fmt.Errorf("no transaction found to release") - } - tCtxVal, ok := tCtx.(*valMany) - if !ok || tCtxVal.tList == nil { - return fmt.Errorf("no transaction found to release") - } - // dedup entries from the list - for i := len(tList) - 2; i >= 0; i-- { - for j := len(tList) - 1; j > i; j-- { - if tList[i] == tList[j] { - // delete j from the list - tList[j] = tList[len(tList)-1] - tList = tList[:len(tList)-1] - } - } - } - // TODO: release from tList, tCtx, or compare and error if diff? - for _, t := range tList { - if t == nil { - continue - } - // cannot call t.Release since context has value defined - select { - case <-t.ch: - default: - return fmt.Errorf("failed to release throttle") - } - } - // modify context value to track release - tCtxVal.tList = nil - return nil -} diff --git a/vendor/github.com/regclient/regclient/manifest.go b/vendor/github.com/regclient/regclient/manifest.go index 12d6eac1e..c5cc84e90 100644 --- a/vendor/github.com/regclient/regclient/manifest.go +++ b/vendor/github.com/regclient/regclient/manifest.go @@ -10,6 +10,7 @@ import ( "github.com/regclient/regclient/types/manifest" "github.com/regclient/regclient/types/platform" "github.com/regclient/regclient/types/ref" + "github.com/regclient/regclient/types/warning" ) type manifestOpt struct { @@ -107,6 +108,10 @@ func (rc *RegClient) ManifestGet(ctx context.Context, r ref.Ref, opts ...Manifes ) } } + // dedup warnings + if w := warning.FromContext(ctx); w == nil { + ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()}) + } schemeAPI, err := rc.schemeGet(r.Scheme) if err != nil { return nil, err @@ -142,6 +147,10 @@ func (rc *RegClient) ManifestHead(ctx context.Context, r ref.Ref, opts ...Manife for _, fn := range opts { fn(&opt) } + // dedup warnings + if w := warning.FromContext(ctx); w == nil { + ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()}) + } schemeAPI, err := rc.schemeGet(r.Scheme) if err != nil { return nil, err diff --git a/vendor/github.com/regclient/regclient/pkg/archive/tar.go b/vendor/github.com/regclient/regclient/pkg/archive/tar.go index 705e9bac4..b6a18ce3b 100644 --- a/vendor/github.com/regclient/regclient/pkg/archive/tar.go +++ b/vendor/github.com/regclient/regclient/pkg/archive/tar.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "io/fs" + "math" "os" "path/filepath" "time" @@ -136,6 +137,9 @@ func Extract(ctx context.Context, path string, r io.Reader, opts ...TarOpts) err fn := filepath.Join(path, filepath.Clean("/"+hdr.Name)) switch hdr.Typeflag { case tar.TypeDir: + if hdr.Mode < 0 || hdr.Mode > math.MaxUint32 { + return fmt.Errorf("integer conversion overflow/underflow (file mode = %d)", hdr.Mode) + } err = os.MkdirAll(fn, fs.FileMode(hdr.Mode)) if err != nil { return err diff --git a/vendor/github.com/regclient/regclient/regclient.go b/vendor/github.com/regclient/regclient/regclient.go index dd76f29d0..0ac4be851 100644 --- a/vendor/github.com/regclient/regclient/regclient.go +++ b/vendor/github.com/regclient/regclient/regclient.go @@ -31,12 +31,12 @@ const ( // RegClient is used to access OCI distribution-spec registries. type RegClient struct { - hosts map[string]*config.Host - log *logrus.Logger - // mu sync.Mutex - regOpts []reg.Opts - schemes map[string]scheme.API - userAgent string + hosts map[string]*config.Host + hostDefault *config.Host + log *logrus.Logger + regOpts []reg.Opts + schemes map[string]scheme.API + userAgent string } // Opt functions are used by [New] to create a [*RegClient]. @@ -47,10 +47,9 @@ func New(opts ...Opt) *RegClient { var rc = RegClient{ hosts: map[string]*config.Host{}, userAgent: DefaultUserAgent, - // logging is disabled by default - log: &logrus.Logger{Out: io.Discard}, - regOpts: []reg.Opts{}, - schemes: map[string]scheme.API{}, + log: &logrus.Logger{Out: io.Discard}, + regOpts: []reg.Opts{}, + schemes: map[string]scheme.API{}, } info := version.GetInfo() @@ -74,6 +73,7 @@ func New(opts ...Opt) *RegClient { } rc.regOpts = append(rc.regOpts, reg.WithConfigHosts(hostList), + reg.WithConfigHostDefault(rc.hostDefault), reg.WithLog(rc.log), reg.WithUserAgent(rc.userAgent), ) @@ -126,6 +126,13 @@ func WithConfigHost(configHost ...config.Host) Opt { } } +// WithConfigHostDefault adds default settings for new hosts. +func WithConfigHostDefault(configHost config.Host) Opt { + return func(rc *RegClient) { + rc.hostDefault = &configHost + } +} + // WithConfigHosts adds a list of config host settings. // // Deprecated: replace with [WithConfigHost]. @@ -232,7 +239,6 @@ func (rc *RegClient) hostLoad(src string, hosts []config.Host) { "tls": string(tls), "pathPrefix": configHost.PathPrefix, "mirrors": configHost.Mirrors, - "api": configHost.API, "blobMax": configHost.BlobMax, "blobChunk": configHost.BlobChunk, }).Debugf("Loading %s config", src) @@ -250,12 +256,9 @@ func (rc *RegClient) hostLoad(src string, hosts []config.Host) { func (rc *RegClient) hostSet(newHost config.Host) error { name := newHost.Name var err error - // hostSet should only run on New, which single threaded - // rc.mu.Lock() - // defer rc.mu.Unlock() if _, ok := rc.hosts[name]; !ok { // merge newHost with default host settings - rc.hosts[name] = config.HostNewName(name) + rc.hosts[name] = config.HostNewDefName(rc.hostDefault, name) err = rc.hosts[name].Merge(newHost, nil) } else { // merge newHost with existing settings diff --git a/vendor/github.com/regclient/regclient/release.md b/vendor/github.com/regclient/regclient/release.md index bd0a553ad..82b6dafaf 100644 --- a/vendor/github.com/regclient/regclient/release.md +++ b/vendor/github.com/regclient/regclient/release.md @@ -1,37 +1,74 @@ -# Release v0.7.1 +# Release v0.7.2 -[PR 798][pr-798] fixes an issue where a malicious registry could return a pinned manifest different from the request. -Commands like `regctl manifest get $image@$digest` will now verify the digest of the returned manifest matches the request rather than the registry headers. +Breaking Changes: -Security updates: +The breaking changes are to internal methods and undocumented features that should not be encountered by users. -- Validate the digest of the ref when provided. ([PR 798][pr-798]) +- Update scheme to use pqueue instead of throttle. ([PR 803][pr-803]) +- Removes an undocumented API for deleting images from Hub. ([PR 803][pr-803]) +- `config.Host.Throttle()` has been removed. Use `scheme.Throttler` instead. ([PR 813][pr-813]) Features: -- Add a `WithDockerCredsFile() regclient.Opt`. ([PR 784][pr-784]) -- Add `regctl artifact get --config` option to only return the config. ([PR 795][pr-795]) +- Significant refactor of http APIs to speed up image copies. ([PR 803][pr-803]) +- Add a priority queue for network requests. ([PR 803][pr-803]) +- Move logging into transport and rework backoff. ([PR 803][pr-803]) +- Remove default rate limit. ([PR 803][pr-803]) +- Add priority queue algorithm and reorder image copy steps. ([PR 803][pr-803]) +- Consolidate warnings. ([PR 810][pr-810]) +- Limit number of retries for a request. ([PR 812][pr-812]) +- Add default host config. ([PR 821][pr-821]) Fixes: -- Detect `amd64` variants for `--platform local`. ([PR 782][pr-782]) -- Mod tracking of changed manifests. ([PR 783][pr-783]) -- Tar path separator should always be a `/`. ([PR 788][pr-788]) +- Update GHA output generating steps. ([PR 800][pr-800]) +- Lookup referrers when registry does not give digest with head. ([PR 801][pr-801]) +- Support auth on redirect. ([PR 805][pr-805]) +- Prevent data race when reading blob and seeking. ([PR 814][pr-814]) +- Detect integer overflows on type conversion. ([PR 830][pr-830]) +- Add a warning if syft is not installed. ([PR 841][pr-841]) +- Race condition in the pqueue tests. ([PR 843][pr-843]) +- Dedup warnings on image mod. ([PR 846][pr-846]) -Other Changes: +Chores: -- Remove docker build cache in GHA. ([PR 780][pr-780]) +- Update staticcheck and fix linter warnings for Go 1.23. ([PR 804][pr-804]) +- Remove digest calculation from reghttp. ([PR 803][pr-803]) +- Remove `ReqPerSec` in tests. ([PR 806][pr-806]) +- Move throttle from `config` to `reghttp`. ([PR 813][pr-813]) +- Refactoring to remove globals in regsync. ([PR 815][pr-815]) +- Refactor to remove globals in regbot. ([PR 816][pr-816]) +- Remove throttle package. ([PR 817][pr-817]) +- Update version-bump config for processors. ([PR 828][pr-828]) +- Update config to use yaml anchors and aliases ([PR 829][pr-829]) +- Do not automatically assign myself to GitHub issues. ([PR 831][pr-831]) +- Remove OpenSSF scorecard and best practices. ([PR 832][pr-832]) +- Update docker image base filesystem. ([PR 837][pr-837]) Contributors: -- @mmonaco -- @stormyyd - @sudo-bmitch -[pr-780]: https://github.com/regclient/regclient/pull/780 -[pr-782]: https://github.com/regclient/regclient/pull/782 -[pr-783]: https://github.com/regclient/regclient/pull/783 -[pr-784]: https://github.com/regclient/regclient/pull/784 -[pr-788]: https://github.com/regclient/regclient/pull/788 -[pr-795]: https://github.com/regclient/regclient/pull/795 -[pr-798]: https://github.com/regclient/regclient/pull/798 +[pr-800]: https://github.com/regclient/regclient/pull/800 +[pr-801]: https://github.com/regclient/regclient/pull/801 +[pr-804]: https://github.com/regclient/regclient/pull/804 +[pr-803]: https://github.com/regclient/regclient/pull/803 +[pr-805]: https://github.com/regclient/regclient/pull/805 +[pr-806]: https://github.com/regclient/regclient/pull/806 +[pr-810]: https://github.com/regclient/regclient/pull/810 +[pr-812]: https://github.com/regclient/regclient/pull/812 +[pr-813]: https://github.com/regclient/regclient/pull/813 +[pr-814]: https://github.com/regclient/regclient/pull/814 +[pr-815]: https://github.com/regclient/regclient/pull/815 +[pr-816]: https://github.com/regclient/regclient/pull/816 +[pr-817]: https://github.com/regclient/regclient/pull/817 +[pr-821]: https://github.com/regclient/regclient/pull/821 +[pr-828]: https://github.com/regclient/regclient/pull/828 +[pr-829]: https://github.com/regclient/regclient/pull/829 +[pr-830]: https://github.com/regclient/regclient/pull/830 +[pr-831]: https://github.com/regclient/regclient/pull/831 +[pr-832]: https://github.com/regclient/regclient/pull/832 +[pr-837]: https://github.com/regclient/regclient/pull/837 +[pr-841]: https://github.com/regclient/regclient/pull/841 +[pr-843]: https://github.com/regclient/regclient/pull/843 +[pr-846]: https://github.com/regclient/regclient/pull/846 diff --git a/vendor/github.com/regclient/regclient/scheme/ocidir/blob.go b/vendor/github.com/regclient/regclient/scheme/ocidir/blob.go index 3b316e779..d23fd507f 100644 --- a/vendor/github.com/regclient/regclient/scheme/ocidir/blob.go +++ b/vendor/github.com/regclient/regclient/scheme/ocidir/blob.go @@ -11,6 +11,7 @@ import ( "github.com/sirupsen/logrus" + "github.com/regclient/regclient/internal/reqmeta" "github.com/regclient/regclient/types/blob" "github.com/regclient/regclient/types/descriptor" "github.com/regclient/regclient/types/errs" @@ -96,11 +97,11 @@ func (o *OCIDir) BlobMount(ctx context.Context, refSrc ref.Ref, refTgt ref.Ref, // BlobPut sends a blob to the repository, returns the digest and size when successful func (o *OCIDir) BlobPut(ctx context.Context, r ref.Ref, d descriptor.Descriptor, rdr io.Reader) (descriptor.Descriptor, error) { t := o.throttleGet(r, false) - err := t.Acquire(ctx) + done, err := t.Acquire(ctx, reqmeta.Data{Kind: reqmeta.Blob, Size: d.Size}) if err != nil { return d, err } - defer t.Release(ctx) + defer done() err = o.initIndex(r, false) if err != nil { diff --git a/vendor/github.com/regclient/regclient/scheme/ocidir/ocidir.go b/vendor/github.com/regclient/regclient/scheme/ocidir/ocidir.go index 906194443..802dec9a2 100644 --- a/vendor/github.com/regclient/regclient/scheme/ocidir/ocidir.go +++ b/vendor/github.com/regclient/regclient/scheme/ocidir/ocidir.go @@ -14,7 +14,8 @@ import ( "github.com/sirupsen/logrus" - "github.com/regclient/regclient/internal/throttle" + "github.com/regclient/regclient/internal/pqueue" + "github.com/regclient/regclient/internal/reqmeta" "github.com/regclient/regclient/types/descriptor" "github.com/regclient/regclient/types/errs" "github.com/regclient/regclient/types/mediatype" @@ -34,7 +35,7 @@ type OCIDir struct { log *logrus.Logger gc bool modRefs map[string]*ociGC - throttle map[string]*throttle.Throttle + throttle map[string]*pqueue.Queue[reqmeta.Data] throttleDef int mu sync.Mutex } @@ -67,7 +68,7 @@ func New(opts ...Opts) *OCIDir { log: conf.log, gc: conf.gc, modRefs: map[string]*ociGC{}, - throttle: map[string]*throttle.Throttle{}, + throttle: map[string]*pqueue.Queue[reqmeta.Data]{}, throttleDef: conf.throttle, } } @@ -116,16 +117,16 @@ func (o *OCIDir) GCUnlock(r ref.Ref) { } // Throttle is used to limit concurrency -func (o *OCIDir) Throttle(r ref.Ref, put bool) []*throttle.Throttle { - tList := []*throttle.Throttle{} +func (o *OCIDir) Throttle(r ref.Ref, put bool) []*pqueue.Queue[reqmeta.Data] { + tList := []*pqueue.Queue[reqmeta.Data]{} // throttle only applies to put requests if !put || o.throttleDef <= 0 { return tList } - return []*throttle.Throttle{o.throttleGet(r, false)} + return []*pqueue.Queue[reqmeta.Data]{o.throttleGet(r, false)} } -func (o *OCIDir) throttleGet(r ref.Ref, locked bool) *throttle.Throttle { +func (o *OCIDir) throttleGet(r ref.Ref, locked bool) *pqueue.Queue[reqmeta.Data] { if !locked { o.mu.Lock() defer o.mu.Unlock() @@ -134,7 +135,7 @@ func (o *OCIDir) throttleGet(r ref.Ref, locked bool) *throttle.Throttle { return t } // init a new throttle - o.throttle[r.Path] = throttle.New(o.throttleDef) + o.throttle[r.Path] = pqueue.New(pqueue.Opts[reqmeta.Data]{Max: o.throttleDef}) return o.throttle[r.Path] } diff --git a/vendor/github.com/regclient/regclient/scheme/reg/blob.go b/vendor/github.com/regclient/regclient/scheme/reg/blob.go index ce05f7e3b..b0a8e5143 100644 --- a/vendor/github.com/regclient/regclient/scheme/reg/blob.go +++ b/vendor/github.com/regclient/regclient/scheme/reg/blob.go @@ -19,10 +19,12 @@ import ( "github.com/sirupsen/logrus" "github.com/regclient/regclient/internal/reghttp" + "github.com/regclient/regclient/internal/reqmeta" "github.com/regclient/regclient/types/blob" "github.com/regclient/regclient/types/descriptor" "github.com/regclient/regclient/types/errs" "github.com/regclient/regclient/types/ref" + "github.com/regclient/regclient/types/warning" ) var ( @@ -32,14 +34,11 @@ var ( // BlobDelete removes a blob from the repository func (reg *Reg) BlobDelete(ctx context.Context, r ref.Ref, d descriptor.Descriptor) error { req := ®http.Req{ - Host: r.Registry, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "DELETE", - Repository: r.Repository, - Path: "blobs/" + d.Digest.String(), - }, - }, + MetaKind: reqmeta.Query, + Host: r.Registry, + Method: "DELETE", + Repository: r.Repository, + Path: "blobs/" + d.Digest.String(), } resp, err := reg.reghttp.Do(ctx, req) if err != nil { @@ -55,14 +54,12 @@ func (reg *Reg) BlobDelete(ctx context.Context, r ref.Ref, d descriptor.Descript func (reg *Reg) BlobGet(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) { // build/send request req := ®http.Req{ - Host: r.Registry, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "GET", - Repository: r.Repository, - Path: "blobs/" + d.Digest.String(), - }, - }, + MetaKind: reqmeta.Blob, + Host: r.Registry, + Method: "GET", + Repository: r.Repository, + Path: "blobs/" + d.Digest.String(), + ExpectLen: d.Size, } resp, err := reg.reghttp.Do(ctx, req) if err != nil && len(d.URLs) > 0 { @@ -74,15 +71,13 @@ func (reg *Reg) BlobGet(ctx context.Context, r ref.Ref, d descriptor.Descriptor) return nil, fmt.Errorf("failed to parse external url \"%s\": %w", curURL, err) } req = ®http.Req{ - Host: r.Registry, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "GET", - Repository: r.Repository, - DirectURL: u, - }, - }, - NoMirrors: true, + MetaKind: reqmeta.Blob, + Host: r.Registry, + Method: "GET", + Repository: r.Repository, + DirectURL: u, + NoMirrors: true, + ExpectLen: d.Size, } resp, err = reg.reghttp.Do(ctx, req) if err == nil { @@ -110,14 +105,11 @@ func (reg *Reg) BlobGet(ctx context.Context, r ref.Ref, d descriptor.Descriptor) func (reg *Reg) BlobHead(ctx context.Context, r ref.Ref, d descriptor.Descriptor) (blob.Reader, error) { // build/send request req := ®http.Req{ - Host: r.Registry, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "HEAD", - Repository: r.Repository, - Path: "blobs/" + d.Digest.String(), - }, - }, + MetaKind: reqmeta.Head, + Host: r.Registry, + Method: "HEAD", + Repository: r.Repository, + Path: "blobs/" + d.Digest.String(), } resp, err := reg.reghttp.Do(ctx, req) if err != nil && len(d.URLs) > 0 { @@ -129,15 +121,12 @@ func (reg *Reg) BlobHead(ctx context.Context, r ref.Ref, d descriptor.Descriptor return nil, fmt.Errorf("failed to parse external url \"%s\": %w", curURL, err) } req = ®http.Req{ - Host: r.Registry, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "HEAD", - Repository: r.Repository, - DirectURL: u, - }, - }, - NoMirrors: true, + MetaKind: reqmeta.Head, + Host: r.Registry, + Method: "HEAD", + Repository: r.Repository, + DirectURL: u, + NoMirrors: true, } resp, err = reg.reghttp.Do(ctx, req) if err == nil { @@ -182,6 +171,10 @@ func (reg *Reg) BlobPut(ctx context.Context, r ref.Ref, d descriptor.Descriptor, var putURL *url.URL var err error validDesc := (d.Size > 0 && d.Digest.Validate() == nil) || (d.Size == 0 && d.Digest == zeroDig) + // dedup warnings + if w := warning.FromContext(ctx); w == nil { + ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()}) + } // attempt an anonymous blob mount if validDesc { @@ -245,16 +238,14 @@ func (reg *Reg) blobGetUploadURL(ctx context.Context, r ref.Ref, d descriptor.De } // request an upload location req := ®http.Req{ - Host: r.Registry, - NoMirrors: true, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "POST", - Repository: r.Repository, - Path: "blobs/uploads/", - Query: q, - }, - }, + MetaKind: reqmeta.Blob, + Host: r.Registry, + NoMirrors: true, + Method: "POST", + Repository: r.Repository, + Path: "blobs/uploads/", + Query: q, + TransactLen: d.Size, } resp, err := reg.reghttp.Do(ctx, req) if err != nil { @@ -321,17 +312,15 @@ func (reg *Reg) blobMount(ctx context.Context, rTgt ref.Ref, d descriptor.Descri } req := ®http.Req{ - Host: rTgt.Registry, - NoMirrors: true, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "POST", - Repository: rTgt.Repository, - Path: "blobs/uploads/", - Query: query, - IgnoreErr: ignoreErr, - }, - }, + MetaKind: reqmeta.Blob, + Host: rTgt.Registry, + NoMirrors: true, + Method: "POST", + Repository: rTgt.Repository, + Path: "blobs/uploads/", + Query: query, + IgnoreErr: ignoreErr, + TransactLen: d.Size, } resp, err := reg.reghttp.Do(ctx, req) if err != nil { @@ -424,18 +413,15 @@ func (reg *Reg) blobPutUploadFull(ctx context.Context, r ref.Ref, d descriptor.D "Content-Type": {"application/octet-stream"}, } req := ®http.Req{ - Host: r.Registry, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "PUT", - Repository: r.Repository, - DirectURL: putURL, - BodyFunc: bodyFunc, - BodyLen: d.Size, - Headers: header, - }, - }, - NoMirrors: true, + MetaKind: reqmeta.Blob, + Host: r.Registry, + Method: "PUT", + Repository: r.Repository, + DirectURL: putURL, + BodyFunc: bodyFunc, + BodyLen: d.Size, + Headers: header, + NoMirrors: true, } resp, err := reg.reghttp.Do(ctx, req) if err != nil { @@ -524,18 +510,16 @@ func (reg *Reg) blobPutUploadChunked(ctx context.Context, r ref.Ref, d descripto "Content-Range": {fmt.Sprintf("%d-%d", chunkStart, chunkStart+int64(chunkSize)-1)}, } req := ®http.Req{ - Host: r.Registry, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "PATCH", - Repository: r.Repository, - DirectURL: &chunkURL, - BodyFunc: bodyFunc, - BodyLen: int64(chunkSize), - Headers: header, - }, - }, - NoMirrors: true, + MetaKind: reqmeta.Blob, + Host: r.Registry, + Method: "PATCH", + Repository: r.Repository, + DirectURL: &chunkURL, + BodyFunc: bodyFunc, + BodyLen: int64(chunkSize), + Headers: header, + NoMirrors: true, + TransactLen: d.Size - int64(chunkSize), } resp, err := reg.reghttp.Do(ctx, req) if err != nil && !errors.Is(err, errs.ErrHTTPStatus) && !errors.Is(err, errs.ErrNotFound) { @@ -620,17 +604,14 @@ func (reg *Reg) blobPutUploadChunked(ctx context.Context, r ref.Ref, d descripto "Content-Type": {"application/octet-stream"}, } req := ®http.Req{ - Host: r.Registry, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "PUT", - Repository: r.Repository, - DirectURL: &chunkURL, - BodyLen: int64(0), - Headers: header, - }, - }, - NoMirrors: true, + MetaKind: reqmeta.Query, + Host: r.Registry, + Method: "PUT", + Repository: r.Repository, + DirectURL: &chunkURL, + BodyLen: int64(0), + Headers: header, + NoMirrors: true, } resp, err := reg.reghttp.Do(ctx, req) if err != nil { @@ -651,15 +632,12 @@ func (reg *Reg) blobUploadCancel(ctx context.Context, r ref.Ref, putURL *url.URL return fmt.Errorf("failed to cancel upload %s: url undefined", r.CommonName()) } req := ®http.Req{ - Host: r.Registry, - NoMirrors: true, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "DELETE", - Repository: r.Repository, - DirectURL: putURL, - }, - }, + MetaKind: reqmeta.Query, + Host: r.Registry, + NoMirrors: true, + Method: "DELETE", + Repository: r.Repository, + DirectURL: putURL, } resp, err := reg.reghttp.Do(ctx, req) if err != nil { @@ -675,15 +653,12 @@ func (reg *Reg) blobUploadCancel(ctx context.Context, r ref.Ref, putURL *url.URL // blobUploadStatus provides a response with headers indicating the progress of an upload func (reg *Reg) blobUploadStatus(ctx context.Context, r ref.Ref, putURL *url.URL) (*http.Response, error) { req := ®http.Req{ - Host: r.Registry, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "GET", - Repository: r.Repository, - DirectURL: putURL, - }, - }, - NoMirrors: true, + MetaKind: reqmeta.Query, + Host: r.Registry, + Method: "GET", + Repository: r.Repository, + DirectURL: putURL, + NoMirrors: true, } resp, err := reg.reghttp.Do(ctx, req) if err != nil { diff --git a/vendor/github.com/regclient/regclient/scheme/reg/manifest.go b/vendor/github.com/regclient/regclient/scheme/reg/manifest.go index 9016dde4d..e9efb1b32 100644 --- a/vendor/github.com/regclient/regclient/scheme/reg/manifest.go +++ b/vendor/github.com/regclient/regclient/scheme/reg/manifest.go @@ -14,16 +14,22 @@ import ( "github.com/regclient/regclient/internal/limitread" "github.com/regclient/regclient/internal/reghttp" + "github.com/regclient/regclient/internal/reqmeta" "github.com/regclient/regclient/scheme" "github.com/regclient/regclient/types/errs" "github.com/regclient/regclient/types/manifest" "github.com/regclient/regclient/types/mediatype" "github.com/regclient/regclient/types/ref" + "github.com/regclient/regclient/types/warning" ) // ManifestDelete removes a manifest by reference (digest) from a registry. // This will implicitly delete all tags pointing to that manifest. func (reg *Reg) ManifestDelete(ctx context.Context, r ref.Ref, opts ...scheme.ManifestOpts) error { + // dedup warnings + if w := warning.FromContext(ctx); w == nil { + ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()}) + } if r.Digest == "" { return fmt.Errorf("digest required to delete manifest, reference %s%.0w", r.CommonName(), errs.ErrMissingDigest) } @@ -57,15 +63,12 @@ func (reg *Reg) ManifestDelete(ctx context.Context, r ref.Ref, opts ...scheme.Ma // build/send request req := ®http.Req{ - Host: r.Registry, - NoMirrors: true, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "DELETE", - Repository: r.Repository, - Path: "manifests/" + r.Digest, - }, - }, + MetaKind: reqmeta.Query, + Host: r.Registry, + NoMirrors: true, + Method: "DELETE", + Repository: r.Repository, + Path: "manifests/" + r.Digest, } resp, err := reg.reghttp.Do(ctx, req) if err != nil { @@ -107,15 +110,12 @@ func (reg *Reg) ManifestGet(ctx context.Context, r ref.Ref) (manifest.Manifest, }, } req := ®http.Req{ - Host: r.Registry, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "GET", - Repository: r.Repository, - Path: "manifests/" + tagOrDigest, - Headers: headers, - }, - }, + MetaKind: reqmeta.Manifest, + Host: r.Registry, + Method: "GET", + Repository: r.Repository, + Path: "manifests/" + tagOrDigest, + Headers: headers, } resp, err := reg.reghttp.Do(ctx, req) if err != nil { @@ -184,15 +184,12 @@ func (reg *Reg) ManifestHead(ctx context.Context, r ref.Ref) (manifest.Manifest, }, } req := ®http.Req{ - Host: r.Registry, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "HEAD", - Repository: r.Repository, - Path: "manifests/" + tagOrDigest, - Headers: headers, - }, - }, + MetaKind: reqmeta.Head, + Host: r.Registry, + Method: "HEAD", + Repository: r.Repository, + Path: "manifests/" + tagOrDigest, + Headers: headers, } resp, err := reg.reghttp.Do(ctx, req) if err != nil { @@ -222,6 +219,10 @@ func (reg *Reg) ManifestPut(ctx context.Context, r ref.Ref, m manifest.Manifest, }).Warn("Manifest put requires a tag") return errs.ErrMissingTag } + // dedup warnings + if w := warning.FromContext(ctx); w == nil { + ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()}) + } // create the request body mj, err := m.MarshalJSON() @@ -248,19 +249,16 @@ func (reg *Reg) ManifestPut(ctx context.Context, r ref.Ref, m manifest.Manifest, q.Add(paramManifestDigest, m.GetDescriptor().Digest.String()) } req := ®http.Req{ - Host: r.Registry, - NoMirrors: true, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "PUT", - Repository: r.Repository, - Path: "manifests/" + tagOrDigest, - Query: q, - Headers: headers, - BodyLen: int64(len(mj)), - BodyBytes: mj, - }, - }, + MetaKind: reqmeta.Manifest, + Host: r.Registry, + NoMirrors: true, + Method: "PUT", + Repository: r.Repository, + Path: "manifests/" + tagOrDigest, + Query: q, + Headers: headers, + BodyLen: int64(len(mj)), + BodyBytes: mj, } resp, err := reg.reghttp.Do(ctx, req) if err != nil { diff --git a/vendor/github.com/regclient/regclient/scheme/reg/ping.go b/vendor/github.com/regclient/regclient/scheme/reg/ping.go index 4a89548c9..e4fc0f022 100644 --- a/vendor/github.com/regclient/regclient/scheme/reg/ping.go +++ b/vendor/github.com/regclient/regclient/scheme/reg/ping.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/regclient/regclient/internal/reghttp" + "github.com/regclient/regclient/internal/reqmeta" "github.com/regclient/regclient/types/ping" "github.com/regclient/regclient/types/ref" ) @@ -13,14 +14,11 @@ import ( func (reg *Reg) Ping(ctx context.Context, r ref.Ref) (ping.Result, error) { ret := ping.Result{} req := ®http.Req{ + MetaKind: reqmeta.Query, Host: r.Registry, NoMirrors: true, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "GET", - Path: "", - }, - }, + Method: "GET", + Path: "", } resp, err := reg.reghttp.Do(ctx, req) diff --git a/vendor/github.com/regclient/regclient/scheme/reg/referrer.go b/vendor/github.com/regclient/regclient/scheme/reg/referrer.go index f59fc20cf..664e75066 100644 --- a/vendor/github.com/regclient/regclient/scheme/reg/referrer.go +++ b/vendor/github.com/regclient/regclient/scheme/reg/referrer.go @@ -9,6 +9,7 @@ import ( "github.com/regclient/regclient/internal/httplink" "github.com/regclient/regclient/internal/reghttp" + "github.com/regclient/regclient/internal/reqmeta" "github.com/regclient/regclient/scheme" "github.com/regclient/regclient/types/errs" "github.com/regclient/regclient/types/manifest" @@ -17,6 +18,7 @@ import ( "github.com/regclient/regclient/types/platform" "github.com/regclient/regclient/types/ref" "github.com/regclient/regclient/types/referrer" + "github.com/regclient/regclient/types/warning" ) const OCISubjectHeader = "OCI-Subject" @@ -30,6 +32,10 @@ func (reg *Reg) ReferrerList(ctx context.Context, r ref.Ref, opts ...scheme.Refe rl := referrer.ReferrerList{ Tags: []string{}, } + // dedup warnings + if w := warning.FromContext(ctx); w == nil { + ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()}) + } // select a platform from a manifest list if config.Platform != "" { p, err := platform.Parse(config.Platform) @@ -74,6 +80,15 @@ func (reg *Reg) ReferrerList(ctx context.Context, r ref.Ref, opts ...scheme.Refe if err != nil { return rl, err } + if m.GetDescriptor().Digest == "" { + m, err = reg.ManifestGet(ctx, r) + if err != nil { + return rl, err + } + } + if m.GetDescriptor().Digest == "" { + return rl, fmt.Errorf("unable to resolve digest for ref %s", r.CommonName()) + } r = r.SetDigest(m.GetDescriptor().Digest.String()) } rl.Subject = r @@ -126,10 +141,9 @@ func (reg *Reg) referrerListByAPI(ctx context.Context, r ref.Ref, config scheme. Tags: []string{}, } var link *url.URL - var resp reghttp.Resp // loop for paging for { - rlAdd, respNext, err := reg.referrerListByAPIPage(ctx, r, config, link) + rlAdd, linkNext, err := reg.referrerListByAPIPage(ctx, r, config, link) if err != nil { return rl, err } @@ -138,33 +152,15 @@ func (reg *Reg) referrerListByAPI(ctx context.Context, r ref.Ref, config scheme. } else { rl.Descriptors = append(rl.Descriptors, rlAdd.Descriptors...) } - resp = respNext - if resp.HTTPResponse() == nil { - return rl, fmt.Errorf("missing http response") - } - respHead := resp.HTTPResponse().Header - links, err := httplink.Parse((respHead.Values("Link"))) - if err != nil { - return rl, err - } - next, err := links.Get("rel", "next") - if err != nil { - // no next link + if linkNext == nil { break } - link = resp.HTTPResponse().Request.URL - if link == nil { - return rl, fmt.Errorf("referrers list failed to get URL of previous request") - } - link, err = link.Parse(next.URI) - if err != nil { - return rl, fmt.Errorf("referrers list failed to parse Link: %w", err) - } + link = linkNext } return rl, nil } -func (reg *Reg) referrerListByAPIPage(ctx context.Context, r ref.Ref, config scheme.ReferrerConfig, link *url.URL) (referrer.ReferrerList, reghttp.Resp, error) { +func (reg *Reg) referrerListByAPIPage(ctx context.Context, r ref.Ref, config scheme.ReferrerConfig, link *url.URL) (referrer.ReferrerList, *url.URL, error) { rl := referrer.ReferrerList{ Subject: r, Tags: []string{}, @@ -174,24 +170,18 @@ func (reg *Reg) referrerListByAPIPage(ctx context.Context, r ref.Ref, config sch query.Set("artifactType", config.MatchOpt.ArtifactType) } req := ®http.Req{ - Host: r.Registry, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "GET", - Repository: r.Repository, - Path: "referrers/" + r.Digest, - Query: query, - IgnoreErr: true, - }, - }, - } - // replace the API if a link is provided + MetaKind: reqmeta.Query, + Host: r.Registry, + Method: "GET", + Repository: r.Repository, + } + if link == nil { + req.Path = "referrers/" + r.Digest + req.Query = query + req.IgnoreErr = true + } if link != nil { - req.APIs[""] = reghttp.ReqAPI{ - Method: "GET", - DirectURL: link, - Repository: r.Repository, - } + req.DirectURL = link } resp, err := reg.reghttp.Do(ctx, req) if err != nil { @@ -224,7 +214,28 @@ func (reg *Reg) referrerListByAPIPage(ctx context.Context, r ref.Ref, config sch rl.Descriptors = ociML.Manifests rl.Annotations = ociML.Annotations - return rl, resp, nil + // lookup next link + respHead := resp.HTTPResponse().Header + links, err := httplink.Parse((respHead.Values("Link"))) + if err != nil { + return rl, nil, err + } + next, err := links.Get("rel", "next") + if err != nil { + // no next link + link = nil + } else { + link = resp.HTTPResponse().Request.URL + if link == nil { + return rl, nil, fmt.Errorf("referrers list failed to get URL of previous request") + } + link, err = link.Parse(next.URI) + if err != nil { + return rl, nil, fmt.Errorf("referrers list failed to parse Link: %w", err) + } + } + + return rl, link, nil } func (reg *Reg) referrerListByTag(ctx context.Context, r ref.Ref) (referrer.ReferrerList, error) { @@ -265,6 +276,10 @@ func (reg *Reg) referrerListByTag(ctx context.Context, r ref.Ref) (referrer.Refe // referrerDelete deletes a referrer associated with a manifest func (reg *Reg) referrerDelete(ctx context.Context, r ref.Ref, m manifest.Manifest) error { + // dedup warnings + if w := warning.FromContext(ctx); w == nil { + ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()}) + } // get subject field mSubject, ok := m.(manifest.Subjecter) if !ok { @@ -314,6 +329,10 @@ func (reg *Reg) referrerDelete(ctx context.Context, r ref.Ref, m manifest.Manife // referrerPut pushes a new referrer associated with a manifest func (reg *Reg) referrerPut(ctx context.Context, r ref.Ref, m manifest.Manifest) error { + // dedup warnings + if w := warning.FromContext(ctx); w == nil { + ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()}) + } // get subject field mSubject, ok := m.(manifest.Subjecter) if !ok { @@ -373,14 +392,11 @@ func (reg *Reg) referrerPing(ctx context.Context, r ref.Ref) bool { return referrerEnabled } req := ®http.Req{ - Host: r.Registry, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "GET", - Repository: r.Repository, - Path: "referrers/" + r.Digest, - }, - }, + MetaKind: reqmeta.Query, + Host: r.Registry, + Method: "GET", + Repository: r.Repository, + Path: "referrers/" + r.Digest, } resp, err := reg.reghttp.Do(ctx, req) if err != nil { diff --git a/vendor/github.com/regclient/regclient/scheme/reg/reg.go b/vendor/github.com/regclient/regclient/scheme/reg/reg.go index 772e008e4..5672ba66e 100644 --- a/vendor/github.com/regclient/regclient/scheme/reg/reg.go +++ b/vendor/github.com/regclient/regclient/scheme/reg/reg.go @@ -10,8 +10,9 @@ import ( "github.com/regclient/regclient/config" "github.com/regclient/regclient/internal/cache" + "github.com/regclient/regclient/internal/pqueue" "github.com/regclient/regclient/internal/reghttp" - "github.com/regclient/regclient/internal/throttle" + "github.com/regclient/regclient/internal/reqmeta" "github.com/regclient/regclient/types/manifest" "github.com/regclient/regclient/types/ref" "github.com/regclient/regclient/types/referrer" @@ -44,6 +45,7 @@ type Reg struct { reghttpOpts []reghttp.Opts log *logrus.Logger hosts map[string]*config.Host + hostDefault *config.Host features map[featureKey]*featureVal blobChunkSize int64 blobChunkLimit int64 @@ -83,7 +85,7 @@ func New(opts ...Opts) *Reg { hosts: map[string]*config.Host{}, features: map[featureKey]*featureVal{}, } - r.reghttpOpts = append(r.reghttpOpts, reghttp.WithConfigHost(r.hostGet)) + r.reghttpOpts = append(r.reghttpOpts, reghttp.WithConfigHostFn(r.hostGet)) for _, opt := range opts { opt(&r) } @@ -92,16 +94,16 @@ func New(opts ...Opts) *Reg { } // Throttle is used to limit concurrency -func (reg *Reg) Throttle(r ref.Ref, put bool) []*throttle.Throttle { - tList := []*throttle.Throttle{} +func (reg *Reg) Throttle(r ref.Ref, put bool) []*pqueue.Queue[reqmeta.Data] { + tList := []*pqueue.Queue[reqmeta.Data]{} host := reg.hostGet(r.Registry) - t := host.Throttle() + t := reg.reghttp.GetThrottle(r.Registry) if t != nil { tList = append(tList, t) } if !put { for _, mirror := range host.Mirrors { - t := reg.hostGet(mirror).Throttle() + t := reg.reghttp.GetThrottle(mirror) if t != nil { tList = append(tList, t) } @@ -114,7 +116,7 @@ func (reg *Reg) hostGet(hostname string) *config.Host { reg.muHost.Lock() defer reg.muHost.Unlock() if _, ok := reg.hosts[hostname]; !ok { - newHost := config.HostNewName(hostname) + newHost := config.HostNewDefName(reg.hostDefault, hostname) // check for normalized hostname if newHost.Name != hostname { hostname = newHost.Name @@ -200,6 +202,13 @@ func WithCertFiles(files []string) Opts { } } +// WithConfigHostDefault provides default settings for hosts. +func WithConfigHostDefault(ch *config.Host) Opts { + return func(r *Reg) { + r.hostDefault = ch + } +} + // WithConfigHosts adds host configs for credentials func WithConfigHosts(configHosts []*config.Host) Opts { return func(r *Reg) { diff --git a/vendor/github.com/regclient/regclient/scheme/reg/repo.go b/vendor/github.com/regclient/regclient/scheme/reg/repo.go index cea8f0ac1..779493dcb 100644 --- a/vendor/github.com/regclient/regclient/scheme/reg/repo.go +++ b/vendor/github.com/regclient/regclient/scheme/reg/repo.go @@ -11,6 +11,7 @@ import ( "github.com/sirupsen/logrus" "github.com/regclient/regclient/internal/reghttp" + "github.com/regclient/regclient/internal/reqmeta" "github.com/regclient/regclient/scheme" "github.com/regclient/regclient/types/mediatype" "github.com/regclient/regclient/types/repo" @@ -36,17 +37,14 @@ func (reg *Reg) RepoList(ctx context.Context, hostname string, opts ...scheme.Re "Accept": []string{"application/json"}, } req := ®http.Req{ + MetaKind: reqmeta.Query, Host: hostname, NoMirrors: true, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "GET", - Path: "_catalog", - NoPrefix: true, - Query: query, - Headers: headers, - }, - }, + Method: "GET", + Path: "_catalog", + NoPrefix: true, + Query: query, + Headers: headers, } resp, err := reg.reghttp.Do(ctx, req) if err != nil { diff --git a/vendor/github.com/regclient/regclient/scheme/reg/tag.go b/vendor/github.com/regclient/regclient/scheme/reg/tag.go index fe9ac5a66..66952aa7a 100644 --- a/vendor/github.com/regclient/regclient/scheme/reg/tag.go +++ b/vendor/github.com/regclient/regclient/scheme/reg/tag.go @@ -21,6 +21,7 @@ import ( "github.com/regclient/regclient/internal/httplink" "github.com/regclient/regclient/internal/reghttp" + "github.com/regclient/regclient/internal/reqmeta" "github.com/regclient/regclient/scheme" "github.com/regclient/regclient/types/descriptor" "github.com/regclient/regclient/types/docker/schema2" @@ -31,6 +32,7 @@ import ( "github.com/regclient/regclient/types/platform" "github.com/regclient/regclient/types/ref" "github.com/regclient/regclient/types/tag" + "github.com/regclient/regclient/types/warning" ) // TagDelete removes a tag from a repository. @@ -41,30 +43,26 @@ func (reg *Reg) TagDelete(ctx context.Context, r ref.Ref) error { if r.Tag == "" { return errs.ErrMissingTag } + // dedup warnings + if w := warning.FromContext(ctx); w == nil { + ctx = warning.NewContext(ctx, &warning.Warning{Hook: warning.DefaultHook()}) + } // attempt to delete the tag directly, available in OCI distribution-spec, and Hub API req := ®http.Req{ - Host: r.Registry, - NoMirrors: true, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "DELETE", - Repository: r.Repository, - Path: "manifests/" + r.Tag, - IgnoreErr: true, // do not trigger backoffs if this fails - }, - "hub": { - Method: "DELETE", - Path: "repositories/" + r.Repository + "/tags/" + r.Tag + "/", - }, - }, + MetaKind: reqmeta.Query, + Host: r.Registry, + NoMirrors: true, + Method: "DELETE", + Repository: r.Repository, + Path: "manifests/" + r.Tag, + IgnoreErr: true, // do not trigger backoffs if this fails } resp, err := reg.reghttp.Do(ctx, req) if resp != nil { defer resp.Close() } - // TODO: Hub may return a different status if err == nil && resp != nil && resp.HTTPResponse().StatusCode == 202 { return nil } @@ -265,16 +263,13 @@ func (reg *Reg) tagListOCI(ctx context.Context, r ref.Ref, config scheme.TagConf "Accept": []string{"application/json"}, } req := ®http.Req{ - Host: r.Registry, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "GET", - Repository: r.Repository, - Path: "tags/list", - Query: query, - Headers: headers, - }, - }, + MetaKind: reqmeta.Query, + Host: r.Registry, + Method: "GET", + Repository: r.Repository, + Path: "tags/list", + Query: query, + Headers: headers, } resp, err := reg.reghttp.Do(ctx, req) if err != nil { @@ -314,15 +309,12 @@ func (reg *Reg) tagListLink(ctx context.Context, r ref.Ref, _ scheme.TagConfig, "Accept": []string{"application/json"}, } req := ®http.Req{ - Host: r.Registry, - APIs: map[string]reghttp.ReqAPI{ - "": { - Method: "GET", - DirectURL: link, - Repository: r.Repository, - Headers: headers, - }, - }, + MetaKind: reqmeta.Query, + Host: r.Registry, + Method: "GET", + DirectURL: link, + Repository: r.Repository, + Headers: headers, } resp, err := reg.reghttp.Do(ctx, req) if err != nil { diff --git a/vendor/github.com/regclient/regclient/scheme/scheme.go b/vendor/github.com/regclient/regclient/scheme/scheme.go index f55c72786..0ba063fe6 100644 --- a/vendor/github.com/regclient/regclient/scheme/scheme.go +++ b/vendor/github.com/regclient/regclient/scheme/scheme.go @@ -5,7 +5,8 @@ import ( "context" "io" - "github.com/regclient/regclient/internal/throttle" + "github.com/regclient/regclient/internal/pqueue" + "github.com/regclient/regclient/internal/reqmeta" "github.com/regclient/regclient/types/blob" "github.com/regclient/regclient/types/descriptor" "github.com/regclient/regclient/types/manifest" @@ -65,7 +66,7 @@ type GCLocker interface { // Throttler is used to indicate the scheme implements Throttle. type Throttler interface { - Throttle(r ref.Ref, put bool) []*throttle.Throttle + Throttle(r ref.Ref, put bool) []*pqueue.Queue[reqmeta.Data] } // ManifestConfig is used by schemes to import [ManifestOpts]. diff --git a/vendor/github.com/regclient/regclient/types/blob/reader.go b/vendor/github.com/regclient/regclient/types/blob/reader.go index f3512ec1f..632fac20d 100644 --- a/vendor/github.com/regclient/regclient/types/blob/reader.go +++ b/vendor/github.com/regclient/regclient/types/blob/reader.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "strconv" + "sync" // crypto libraries included for go-digest _ "crypto/sha256" @@ -26,6 +27,7 @@ type BReader struct { reader io.Reader origRdr io.Reader digester digest.Digester + mu sync.Mutex } // NewReader creates a new BReader. @@ -103,6 +105,8 @@ func (r *BReader) Read(p []byte) (int, error) { if r == nil || r.reader == nil { return 0, fmt.Errorf("blob has no reader: %w", io.ErrUnexpectedEOF) } + r.mu.Lock() + defer r.mu.Unlock() size, err := r.reader.Read(p) r.readBytes = r.readBytes + int64(size) if err == io.EOF { @@ -126,6 +130,11 @@ func (r *BReader) Read(p []byte) (int, error) { // Seek passes through the seek operation, reseting or invalidating the digest func (r *BReader) Seek(offset int64, whence int) (int64, error) { + if r == nil || r.origRdr == nil { + return 0, fmt.Errorf("blob has no reader") + } + r.mu.Lock() + defer r.mu.Unlock() if offset == 0 && whence == io.SeekCurrent { return r.readBytes, nil } @@ -133,9 +142,6 @@ func (r *BReader) Seek(offset int64, whence int) (int64, error) { if offset != 0 || whence != io.SeekStart { return r.readBytes, fmt.Errorf("unable to seek to arbitrary position") } - if r == nil || r.origRdr == nil { - return 0, fmt.Errorf("blob has no reader") - } rdrSeek, ok := r.origRdr.(io.Seeker) if !ok { return r.readBytes, fmt.Errorf("Seek unsupported") @@ -189,6 +195,8 @@ func (r *BReader) ToTarReader() (*BTarReader, error) { if r == nil || !r.blobSet { return nil, fmt.Errorf("blob is not defined") } + r.mu.Lock() + defer r.mu.Unlock() if r.readBytes != 0 { return nil, fmt.Errorf("unable to convert after read has been performed") } diff --git a/vendor/github.com/regclient/regclient/types/descriptor/descriptor.go b/vendor/github.com/regclient/regclient/types/descriptor/descriptor.go index 097021bf3..142269334 100644 --- a/vendor/github.com/regclient/regclient/types/descriptor/descriptor.go +++ b/vendor/github.com/regclient/regclient/types/descriptor/descriptor.go @@ -225,7 +225,7 @@ func (d Descriptor) Match(opt MatchOpt) bool { if opt.ArtifactType != "" && d.ArtifactType != opt.ArtifactType { return false } - if opt.Annotations != nil && len(opt.Annotations) > 0 { + if len(opt.Annotations) > 0 { if d.Annotations == nil { return false } diff --git a/vendor/github.com/regclient/regclient/types/errs/error.go b/vendor/github.com/regclient/regclient/types/errs/error.go index fc3e1dd10..03f1842fe 100644 --- a/vendor/github.com/regclient/regclient/types/errs/error.go +++ b/vendor/github.com/regclient/regclient/types/errs/error.go @@ -62,6 +62,8 @@ var ( ErrParsingFailed = errors.New("parsing failed") // ErrRetryNeeded indicates a request needs to be retried ErrRetryNeeded = errors.New("retry needed") + // ErrRetryLimitExceeded indicates too many retries have occurred + ErrRetryLimitExceeded = errors.New("retry limit exceeded") // ErrShortRead if contents are less than expected the size ErrShortRead = errors.New("short read") // ErrSizeLimitExceeded if contents exceed the size limit diff --git a/vendor/github.com/regclient/regclient/types/manifest/docker2.go b/vendor/github.com/regclient/regclient/types/manifest/docker2.go index e884fd26b..7d04bb726 100644 --- a/vendor/github.com/regclient/regclient/types/manifest/docker2.go +++ b/vendor/github.com/regclient/regclient/types/manifest/docker2.go @@ -165,7 +165,7 @@ func (m *docker2Manifest) MarshalPretty() ([]byte, error) { } fmt.Fprintf(tw, "MediaType:\t%s\n", m.desc.MediaType) fmt.Fprintf(tw, "Digest:\t%s\n", m.desc.Digest.String()) - if m.Annotations != nil && len(m.Annotations) > 0 { + if len(m.Annotations) > 0 { fmt.Fprintf(tw, "Annotations:\t\n") keys := make([]string, 0, len(m.Annotations)) for k := range m.Annotations { @@ -211,7 +211,7 @@ func (m *docker2ManifestList) MarshalPretty() ([]byte, error) { } fmt.Fprintf(tw, "MediaType:\t%s\n", m.desc.MediaType) fmt.Fprintf(tw, "Digest:\t%s\n", m.desc.Digest.String()) - if m.Annotations != nil && len(m.Annotations) > 0 { + if len(m.Annotations) > 0 { fmt.Fprintf(tw, "Annotations:\t\n") keys := make([]string, 0, len(m.Annotations)) for k := range m.Annotations { diff --git a/vendor/github.com/regclient/regclient/types/manifest/oci1.go b/vendor/github.com/regclient/regclient/types/manifest/oci1.go index 2fd20eb9c..594a765aa 100644 --- a/vendor/github.com/regclient/regclient/types/manifest/oci1.go +++ b/vendor/github.com/regclient/regclient/types/manifest/oci1.go @@ -226,7 +226,7 @@ func (m *oci1Manifest) MarshalPretty() ([]byte, error) { fmt.Fprintf(tw, "ArtifactType:\t%s\n", m.ArtifactType) } fmt.Fprintf(tw, "Digest:\t%s\n", m.desc.Digest.String()) - if m.Annotations != nil && len(m.Annotations) > 0 { + if len(m.Annotations) > 0 { fmt.Fprintf(tw, "Annotations:\t\n") keys := make([]string, 0, len(m.Annotations)) for k := range m.Annotations { @@ -283,7 +283,7 @@ func (m *oci1Index) MarshalPretty() ([]byte, error) { fmt.Fprintf(tw, "ArtifactType:\t%s\n", m.ArtifactType) } fmt.Fprintf(tw, "Digest:\t%s\n", m.desc.Digest.String()) - if m.Annotations != nil && len(m.Annotations) > 0 { + if len(m.Annotations) > 0 { fmt.Fprintf(tw, "Annotations:\t\n") keys := make([]string, 0, len(m.Annotations)) for k := range m.Annotations { @@ -332,7 +332,7 @@ func (m *oci1Artifact) MarshalPretty() ([]byte, error) { fmt.Fprintf(tw, "MediaType:\t%s\n", m.desc.MediaType) fmt.Fprintf(tw, "ArtifactType:\t%s\n", m.ArtifactType) fmt.Fprintf(tw, "Digest:\t%s\n", m.desc.Digest.String()) - if m.Annotations != nil && len(m.Annotations) > 0 { + if len(m.Annotations) > 0 { fmt.Fprintf(tw, "Annotations:\t\n") keys := make([]string, 0, len(m.Annotations)) for k := range m.Annotations { diff --git a/vendor/github.com/regclient/regclient/types/referrer/referrer.go b/vendor/github.com/regclient/regclient/types/referrer/referrer.go index 2cbb727f1..6ffe92c6d 100644 --- a/vendor/github.com/regclient/regclient/types/referrer/referrer.go +++ b/vendor/github.com/regclient/regclient/types/referrer/referrer.go @@ -128,7 +128,7 @@ func (rl ReferrerList) MarshalPretty() ([]byte, error) { return []byte{}, err } } - if rl.Annotations != nil && len(rl.Annotations) > 0 { + if len(rl.Annotations) > 0 { fmt.Fprintf(tw, "Annotations:\t\n") keys := make([]string, 0, len(rl.Annotations)) for k := range rl.Annotations { diff --git a/vendor/modules.txt b/vendor/modules.txt index 4e674f3df..794bd91f4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -299,8 +299,8 @@ github.com/josharian/intern # github.com/json-iterator/go v1.1.12 ## explicit; go 1.12 github.com/json-iterator/go -# github.com/klauspost/compress v1.17.9 -## explicit; go 1.20 +# github.com/klauspost/compress v1.17.11 +## explicit; go 1.21 github.com/klauspost/compress github.com/klauspost/compress/fse github.com/klauspost/compress/huff0 @@ -473,8 +473,8 @@ github.com/prometheus/common/model github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util -# github.com/regclient/regclient v0.7.1 -## explicit; go 1.20 +# github.com/regclient/regclient v0.7.2 +## explicit; go 1.21 github.com/regclient/regclient github.com/regclient/regclient/config github.com/regclient/regclient/internal/auth @@ -482,9 +482,10 @@ github.com/regclient/regclient/internal/cache github.com/regclient/regclient/internal/conffile github.com/regclient/regclient/internal/httplink github.com/regclient/regclient/internal/limitread +github.com/regclient/regclient/internal/pqueue github.com/regclient/regclient/internal/reghttp +github.com/regclient/regclient/internal/reqmeta github.com/regclient/regclient/internal/strparse -github.com/regclient/regclient/internal/throttle github.com/regclient/regclient/internal/timejson github.com/regclient/regclient/internal/units github.com/regclient/regclient/internal/version