resolve conflicts merging mmr,massifs and mmrtesting re: AB#9467

datatrails · May 15, 2024 · 8a1834b · 8a1834b
2 parents 7de2d45 + ddc7067
commit 8a1834b
Show file tree

Hide file tree

Showing 93 changed files with 11,795 additions and 0 deletions.
diff --git a/go.work.sum b/go.work.sum
@@ -0,0 +1,70 @@
+cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
+cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
+code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8=
+github.com/IBM/sarama v1.40.1/go.mod h1:+5OFwA5Du9I6QrznhaMHsuwWdWZNMjaBSIxEWEgKOYE=
+github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
+github.com/alicebob/miniredis/v2 v2.32.1/go.mod h1:AqkLNAfUm0K07J28hnAyyQKf/x0YkCY/g5DCtuL01Mw=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
+github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho=
+github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
+github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
+github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g=
+github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
+github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
+github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
+github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
+github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo=
+github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/CyVWsJEsJO0=
+github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
+github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y=
+github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
+github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
+github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4=
+github.com/lestrrat-go/jwx v1.2.29/go.mod h1:hU8k2l6WF0ncx20uQdOmik/Gjg6E3/wIRtXSNFeZuB8=
+github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/microsoft/ApplicationInsights-Go v0.4.4/go.mod h1:fKRUseBqkw6bDiXTs3ESTiU/4YTIHsQS4W3fP2ieF4U=
+github.com/nuts-foundation/go-did v0.6.4/go.mod h1:Jb3IgnO2Zeed970JMIlfjr4g1kvikmgWUJA0EfeDEFE=
+github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
+github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
+github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/rabbitmq/amqp091-go v1.8.1/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc=
+github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/shengdoushi/base58 v1.0.0/go.mod h1:m5uIILfzcKMw6238iWAhP4l3s5+uXyF3+bJKUNhAL9I=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f h1:Vn+VyHU5guc9KjB5KrjI2q0wCOWEOIh0OEsleqakHJg=
+google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f/go.mod h1:nWSwAFPb+qfNJXsoeO3Io7zf4tMSfN8EA8RlDA04GhY=
diff --git a/massifs/bloblast.go b/massifs/bloblast.go
@@ -0,0 +1,168 @@
+package massifs
+
+import (
+	"context"
+	"errors"
+
+	azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/datatrails/go-datatrails-common/azblob"
+)
+
+var (
+	ErrBlobNotFound = errors.New("the blob was not found")
+)
+
+// LastPrefixedBlob returns the details of last blob found under the prefix path
+// And the total number of blobs under the path.
+func LastPrefixedBlob(
+	ctx context.Context, store logBlobReader, blobPrefixPath string,
+	opts ...azblob.Option,
+) (LogBlobContext, uint64, error) {
+
+	bc := LogBlobContext{}
+
+	var foundCount uint64
+
+	opts = append([]azblob.Option{azblob.WithListPrefix(blobPrefixPath)}, opts...)
+
+	var marker azblob.ListMarker
+	for {
+		r, err := store.List(ctx, append(opts, azblob.WithListMarker(marker))...)
+		if err != nil {
+			return bc, foundCount, err
+		}
+		if len(r.Items) == 0 {
+			return bc, foundCount, nil
+		}
+
+		foundCount += uint64(len(r.Items))
+
+		// we want the _last_ listed, so we just keep over-writing
+		i := r.Items[len(r.Items)-1]
+		bc.ETag = *i.Properties.Etag
+		bc.LastModified = *i.Properties.LastModified
+		if i.Properties.ContentLength != nil {
+			bc.ContentLength = *i.Properties.ContentLength
+		}
+		bc.BlobPath = *i.Name
+		marker = r.Marker
+		if marker == nil || *marker == "" {
+			bc.Tags = listResponseTags(i.BlobTags)
+			break
+		}
+	}
+
+	return bc, foundCount, nil
+}
+
+// FirstPrefixedBlob returns the first blob found under the prefix path
+func FirstPrefixedBlob(
+	ctx context.Context, store logBlobReader, blobPrefixPath string,
+	opts ...azblob.Option,
+) (LogBlobContext, error) {
+
+	bc := LogBlobContext{}
+
+	opts = append([]azblob.Option{
+		azblob.WithListPrefix(blobPrefixPath),
+		azblob.WithListMaxResults(1),
+	}, opts...)
+
+	r, err := store.List(ctx, opts...)
+	if err != nil {
+		return bc, err
+	}
+	if len(r.Items) == 0 {
+		return bc, ErrBlobNotFound
+	}
+
+	i := r.Items[0]
+	bc.ETag = *i.Properties.Etag
+	bc.LastModified = *i.Properties.LastModified
+	if i.Properties.ContentLength != nil {
+		bc.ContentLength = *i.Properties.ContentLength
+	}
+	bc.BlobPath = *i.Name
+	bc.Tags = listResponseTags(r.Items[0].BlobTags)
+
+	return bc, nil
+}
+
+// PrefixedBlobLastN returns contexts for the last n blobs under the provided prefix.
+//
+// The number of items in the returned tail is always min(massifCount, n)
+// Un filled items are zero valued.
+func PrefixedBlobLastN(
+	ctx context.Context,
+	store logBlobReader,
+	blobPrefixPath string,
+	n int,
+	opts ...azblob.Option,
+) ([]LogBlobContext, uint64, error) {
+
+	opts = append([]azblob.Option{azblob.WithListPrefix(blobPrefixPath)}, opts...)
+
+	tail := make([]LogBlobContext, n)
+
+	var foundCount uint64
+
+	var marker azblob.ListMarker
+	for {
+		r, err := store.List(ctx, append(opts, azblob.WithListMarker(marker))...)
+		if err != nil {
+			return tail, foundCount, err
+		}
+		if len(r.Items) == 0 {
+			return tail, foundCount, nil
+		}
+
+		foundCount += uint64(len(r.Items))
+
+		// The stale items are those from the previous round that can be
+		// replaced by the current. Typically, len(r.Items) will be greater than
+		// n and so it will be n. Note that stale is > 0 here due to the len 0
+		// check above.
+		stale := min(len(r.Items), n)
+
+		// copy the items *after* the stale items to the front.
+		if stale != n {
+			copy(tail, tail[n-stale-1:])
+		}
+
+		for i := 0; i < stale; i++ {
+
+			// stale is also the count of items we are taking from items.
+
+			it := r.Items[len(r.Items)-stale+i]
+			tail[n-stale+i].ETag = *it.Properties.Etag
+			tail[n-stale+i].LastModified = *it.Properties.LastModified
+			if it.Properties.ContentLength != nil {
+				tail[n-stale+i].ContentLength = *it.Properties.ContentLength
+			}
+			tail[n-stale+i].BlobPath = *it.Name
+			tail[n-stale+i].Tags = listResponseTags(it.BlobTags)
+		}
+
+		marker = r.Marker
+		if marker == nil || *marker == "" {
+			break
+		}
+	}
+
+	// Note massifIndex will be zero, the id of the first massif blob
+	return tail, foundCount, nil
+}
+
+func listResponseTags(blobTags *azStorageBlob.BlobTags) map[string]string {
+	if blobTags == nil {
+		return nil
+	}
+	tags := map[string]string{}
+	for _, tag := range blobTags.BlobTagSet {
+		if tag.Key == nil || tag.Value == nil {
+			continue
+		}
+		tags[*tag.Key] = *tag.Value
+	}
+	return tags
+}