diff --git a/api/gen/proto/go/metastore/v1/metadata_query.pb.go b/api/gen/proto/go/metastore/v1/metadata_query.pb.go deleted file mode 100644 index 6bfa14d2c7..0000000000 --- a/api/gen/proto/go/metastore/v1/metadata_query.pb.go +++ /dev/null @@ -1,263 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.34.2 -// protoc (unknown) -// source: metastore/v1/metadata_query.proto - -package metastorev1 - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type QueryMetadataRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - TenantId []string `protobuf:"bytes,1,rep,name=tenant_id,json=tenantId,proto3" json:"tenant_id,omitempty"` - StartTime int64 `protobuf:"varint,2,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` - EndTime int64 `protobuf:"varint,3,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"` - Query string `protobuf:"bytes,4,opt,name=query,proto3" json:"query,omitempty"` -} - -func (x *QueryMetadataRequest) Reset() { - *x = QueryMetadataRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_metastore_v1_metadata_query_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *QueryMetadataRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*QueryMetadataRequest) ProtoMessage() {} - -func (x *QueryMetadataRequest) ProtoReflect() protoreflect.Message { - mi := &file_metastore_v1_metadata_query_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use QueryMetadataRequest.ProtoReflect.Descriptor instead. -func (*QueryMetadataRequest) Descriptor() ([]byte, []int) { - return file_metastore_v1_metadata_query_proto_rawDescGZIP(), []int{0} -} - -func (x *QueryMetadataRequest) GetTenantId() []string { - if x != nil { - return x.TenantId - } - return nil -} - -func (x *QueryMetadataRequest) GetStartTime() int64 { - if x != nil { - return x.StartTime - } - return 0 -} - -func (x *QueryMetadataRequest) GetEndTime() int64 { - if x != nil { - return x.EndTime - } - return 0 -} - -func (x *QueryMetadataRequest) GetQuery() string { - if x != nil { - return x.Query - } - return "" -} - -type QueryMetadataResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Blocks []*BlockMeta `protobuf:"bytes,1,rep,name=blocks,proto3" json:"blocks,omitempty"` -} - -func (x *QueryMetadataResponse) Reset() { - *x = QueryMetadataResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_metastore_v1_metadata_query_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *QueryMetadataResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*QueryMetadataResponse) ProtoMessage() {} - -func (x *QueryMetadataResponse) ProtoReflect() protoreflect.Message { - mi := &file_metastore_v1_metadata_query_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use QueryMetadataResponse.ProtoReflect.Descriptor instead. -func (*QueryMetadataResponse) Descriptor() ([]byte, []int) { - return file_metastore_v1_metadata_query_proto_rawDescGZIP(), []int{1} -} - -func (x *QueryMetadataResponse) GetBlocks() []*BlockMeta { - if x != nil { - return x.Blocks - } - return nil -} - -var File_metastore_v1_metadata_query_proto protoreflect.FileDescriptor - -var file_metastore_v1_metadata_query_proto_rawDesc = []byte{ - 0x0a, 0x21, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, - 0x31, 0x1a, 0x18, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x31, 0x2f, - 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x83, 0x01, 0x0a, 0x14, - 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x49, - 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, - 0x12, 0x19, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x71, - 0x75, 0x65, 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, - 0x79, 0x22, 0x48, 0x0a, 0x15, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x06, 0x62, 0x6c, - 0x6f, 0x63, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x65, 0x74, - 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4d, - 0x65, 0x74, 0x61, 0x52, 0x06, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x32, 0x72, 0x0a, 0x14, 0x4d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x51, 0x75, 0x65, 0x72, 0x79, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x12, 0x5a, 0x0a, 0x0d, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x73, - 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x65, 0x74, - 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, - 0xbf, 0x01, 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x2e, 0x76, 0x31, 0x42, 0x12, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x51, 0x75, - 0x65, 0x72, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x46, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x72, 0x61, 0x66, 0x61, 0x6e, 0x61, 0x2f, 0x70, - 0x79, 0x72, 0x6f, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x67, 0x65, 0x6e, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x31, 0x3b, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4d, 0x58, 0x58, 0xaa, 0x02, 0x0c, 0x4d, 0x65, 0x74, 0x61, 0x73, - 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0c, 0x4d, 0x65, 0x74, 0x61, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x18, 0x4d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, - 0x72, 0x65, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, - 0x61, 0xea, 0x02, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x3a, 0x3a, 0x56, - 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_metastore_v1_metadata_query_proto_rawDescOnce sync.Once - file_metastore_v1_metadata_query_proto_rawDescData = file_metastore_v1_metadata_query_proto_rawDesc -) - -func file_metastore_v1_metadata_query_proto_rawDescGZIP() []byte { - file_metastore_v1_metadata_query_proto_rawDescOnce.Do(func() { - file_metastore_v1_metadata_query_proto_rawDescData = protoimpl.X.CompressGZIP(file_metastore_v1_metadata_query_proto_rawDescData) - }) - return file_metastore_v1_metadata_query_proto_rawDescData -} - -var file_metastore_v1_metadata_query_proto_msgTypes = make([]protoimpl.MessageInfo, 2) -var file_metastore_v1_metadata_query_proto_goTypes = []any{ - (*QueryMetadataRequest)(nil), // 0: metastore.v1.QueryMetadataRequest - (*QueryMetadataResponse)(nil), // 1: metastore.v1.QueryMetadataResponse - (*BlockMeta)(nil), // 2: metastore.v1.BlockMeta -} -var file_metastore_v1_metadata_query_proto_depIdxs = []int32{ - 2, // 0: metastore.v1.QueryMetadataResponse.blocks:type_name -> metastore.v1.BlockMeta - 0, // 1: metastore.v1.MetadataQueryService.QueryMetadata:input_type -> metastore.v1.QueryMetadataRequest - 1, // 2: metastore.v1.MetadataQueryService.QueryMetadata:output_type -> metastore.v1.QueryMetadataResponse - 2, // [2:3] is the sub-list for method output_type - 1, // [1:2] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name -} - -func init() { file_metastore_v1_metadata_query_proto_init() } -func file_metastore_v1_metadata_query_proto_init() { - if File_metastore_v1_metadata_query_proto != nil { - return - } - file_metastore_v1_types_proto_init() - if !protoimpl.UnsafeEnabled { - file_metastore_v1_metadata_query_proto_msgTypes[0].Exporter = func(v any, i int) any { - switch v := v.(*QueryMetadataRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_metastore_v1_metadata_query_proto_msgTypes[1].Exporter = func(v any, i int) any { - switch v := v.(*QueryMetadataResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_metastore_v1_metadata_query_proto_rawDesc, - NumEnums: 0, - NumMessages: 2, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_metastore_v1_metadata_query_proto_goTypes, - DependencyIndexes: file_metastore_v1_metadata_query_proto_depIdxs, - MessageInfos: file_metastore_v1_metadata_query_proto_msgTypes, - }.Build() - File_metastore_v1_metadata_query_proto = out.File - file_metastore_v1_metadata_query_proto_rawDesc = nil - file_metastore_v1_metadata_query_proto_goTypes = nil - file_metastore_v1_metadata_query_proto_depIdxs = nil -} diff --git a/api/gen/proto/go/metastore/v1/metastorev1connect/metadata_query.connect.go b/api/gen/proto/go/metastore/v1/metastorev1connect/query.connect.go similarity index 66% rename from api/gen/proto/go/metastore/v1/metastorev1connect/metadata_query.connect.go rename to api/gen/proto/go/metastore/v1/metastorev1connect/query.connect.go index 98646a38e2..454dc9a97b 100644 --- a/api/gen/proto/go/metastore/v1/metastorev1connect/metadata_query.connect.go +++ b/api/gen/proto/go/metastore/v1/metastorev1connect/query.connect.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-connect-go. DO NOT EDIT. // -// Source: metastore/v1/metadata_query.proto +// Source: metastore/v1/query.proto package metastorev1connect @@ -36,17 +36,22 @@ const ( // MetadataQueryServiceQueryMetadataProcedure is the fully-qualified name of the // MetadataQueryService's QueryMetadata RPC. MetadataQueryServiceQueryMetadataProcedure = "/metastore.v1.MetadataQueryService/QueryMetadata" + // MetadataQueryServiceQueryMetadataLabelsProcedure is the fully-qualified name of the + // MetadataQueryService's QueryMetadataLabels RPC. + MetadataQueryServiceQueryMetadataLabelsProcedure = "/metastore.v1.MetadataQueryService/QueryMetadataLabels" ) // These variables are the protoreflect.Descriptor objects for the RPCs defined in this package. var ( - metadataQueryServiceServiceDescriptor = v1.File_metastore_v1_metadata_query_proto.Services().ByName("MetadataQueryService") - metadataQueryServiceQueryMetadataMethodDescriptor = metadataQueryServiceServiceDescriptor.Methods().ByName("QueryMetadata") + metadataQueryServiceServiceDescriptor = v1.File_metastore_v1_query_proto.Services().ByName("MetadataQueryService") + metadataQueryServiceQueryMetadataMethodDescriptor = metadataQueryServiceServiceDescriptor.Methods().ByName("QueryMetadata") + metadataQueryServiceQueryMetadataLabelsMethodDescriptor = metadataQueryServiceServiceDescriptor.Methods().ByName("QueryMetadataLabels") ) // MetadataQueryServiceClient is a client for the metastore.v1.MetadataQueryService service. type MetadataQueryServiceClient interface { QueryMetadata(context.Context, *connect.Request[v1.QueryMetadataRequest]) (*connect.Response[v1.QueryMetadataResponse], error) + QueryMetadataLabels(context.Context, *connect.Request[v1.QueryMetadataLabelsRequest]) (*connect.Response[v1.QueryMetadataLabelsResponse], error) } // NewMetadataQueryServiceClient constructs a client for the metastore.v1.MetadataQueryService @@ -65,12 +70,19 @@ func NewMetadataQueryServiceClient(httpClient connect.HTTPClient, baseURL string connect.WithSchema(metadataQueryServiceQueryMetadataMethodDescriptor), connect.WithClientOptions(opts...), ), + queryMetadataLabels: connect.NewClient[v1.QueryMetadataLabelsRequest, v1.QueryMetadataLabelsResponse]( + httpClient, + baseURL+MetadataQueryServiceQueryMetadataLabelsProcedure, + connect.WithSchema(metadataQueryServiceQueryMetadataLabelsMethodDescriptor), + connect.WithClientOptions(opts...), + ), } } // metadataQueryServiceClient implements MetadataQueryServiceClient. type metadataQueryServiceClient struct { - queryMetadata *connect.Client[v1.QueryMetadataRequest, v1.QueryMetadataResponse] + queryMetadata *connect.Client[v1.QueryMetadataRequest, v1.QueryMetadataResponse] + queryMetadataLabels *connect.Client[v1.QueryMetadataLabelsRequest, v1.QueryMetadataLabelsResponse] } // QueryMetadata calls metastore.v1.MetadataQueryService.QueryMetadata. @@ -78,10 +90,16 @@ func (c *metadataQueryServiceClient) QueryMetadata(ctx context.Context, req *con return c.queryMetadata.CallUnary(ctx, req) } +// QueryMetadataLabels calls metastore.v1.MetadataQueryService.QueryMetadataLabels. +func (c *metadataQueryServiceClient) QueryMetadataLabels(ctx context.Context, req *connect.Request[v1.QueryMetadataLabelsRequest]) (*connect.Response[v1.QueryMetadataLabelsResponse], error) { + return c.queryMetadataLabels.CallUnary(ctx, req) +} + // MetadataQueryServiceHandler is an implementation of the metastore.v1.MetadataQueryService // service. type MetadataQueryServiceHandler interface { QueryMetadata(context.Context, *connect.Request[v1.QueryMetadataRequest]) (*connect.Response[v1.QueryMetadataResponse], error) + QueryMetadataLabels(context.Context, *connect.Request[v1.QueryMetadataLabelsRequest]) (*connect.Response[v1.QueryMetadataLabelsResponse], error) } // NewMetadataQueryServiceHandler builds an HTTP handler from the service implementation. It returns @@ -96,10 +114,18 @@ func NewMetadataQueryServiceHandler(svc MetadataQueryServiceHandler, opts ...con connect.WithSchema(metadataQueryServiceQueryMetadataMethodDescriptor), connect.WithHandlerOptions(opts...), ) + metadataQueryServiceQueryMetadataLabelsHandler := connect.NewUnaryHandler( + MetadataQueryServiceQueryMetadataLabelsProcedure, + svc.QueryMetadataLabels, + connect.WithSchema(metadataQueryServiceQueryMetadataLabelsMethodDescriptor), + connect.WithHandlerOptions(opts...), + ) return "/metastore.v1.MetadataQueryService/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case MetadataQueryServiceQueryMetadataProcedure: metadataQueryServiceQueryMetadataHandler.ServeHTTP(w, r) + case MetadataQueryServiceQueryMetadataLabelsProcedure: + metadataQueryServiceQueryMetadataLabelsHandler.ServeHTTP(w, r) default: http.NotFound(w, r) } @@ -112,3 +138,7 @@ type UnimplementedMetadataQueryServiceHandler struct{} func (UnimplementedMetadataQueryServiceHandler) QueryMetadata(context.Context, *connect.Request[v1.QueryMetadataRequest]) (*connect.Response[v1.QueryMetadataResponse], error) { return nil, connect.NewError(connect.CodeUnimplemented, errors.New("metastore.v1.MetadataQueryService.QueryMetadata is not implemented")) } + +func (UnimplementedMetadataQueryServiceHandler) QueryMetadataLabels(context.Context, *connect.Request[v1.QueryMetadataLabelsRequest]) (*connect.Response[v1.QueryMetadataLabelsResponse], error) { + return nil, connect.NewError(connect.CodeUnimplemented, errors.New("metastore.v1.MetadataQueryService.QueryMetadataLabels is not implemented")) +} diff --git a/api/gen/proto/go/metastore/v1/metastorev1connect/metadata_query.connect.mux.go b/api/gen/proto/go/metastore/v1/metastorev1connect/query.connect.mux.go similarity index 82% rename from api/gen/proto/go/metastore/v1/metastorev1connect/metadata_query.connect.mux.go rename to api/gen/proto/go/metastore/v1/metastorev1connect/query.connect.mux.go index 8b62d1b865..9e7d230a1d 100644 --- a/api/gen/proto/go/metastore/v1/metastorev1connect/metadata_query.connect.mux.go +++ b/api/gen/proto/go/metastore/v1/metastorev1connect/query.connect.mux.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-connect-go-mux. DO NOT EDIT. // -// Source: metastore/v1/metadata_query.proto +// Source: metastore/v1/query.proto package metastorev1connect @@ -24,4 +24,9 @@ func RegisterMetadataQueryServiceHandler(mux *mux.Router, svc MetadataQueryServi svc.QueryMetadata, opts..., )) + mux.Handle("/metastore.v1.MetadataQueryService/QueryMetadataLabels", connect.NewUnaryHandler( + "/metastore.v1.MetadataQueryService/QueryMetadataLabels", + svc.QueryMetadataLabels, + opts..., + )) } diff --git a/api/gen/proto/go/metastore/v1/query.pb.go b/api/gen/proto/go/metastore/v1/query.pb.go new file mode 100644 index 0000000000..9c93e8c8f6 --- /dev/null +++ b/api/gen/proto/go/metastore/v1/query.pb.go @@ -0,0 +1,442 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.34.2 +// protoc (unknown) +// source: metastore/v1/query.proto + +package metastorev1 + +import ( + v1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type QueryMetadataRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TenantId []string `protobuf:"bytes,1,rep,name=tenant_id,json=tenantId,proto3" json:"tenant_id,omitempty"` + StartTime int64 `protobuf:"varint,2,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` + EndTime int64 `protobuf:"varint,3,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"` + Query string `protobuf:"bytes,4,opt,name=query,proto3" json:"query,omitempty"` +} + +func (x *QueryMetadataRequest) Reset() { + *x = QueryMetadataRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_metastore_v1_query_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *QueryMetadataRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*QueryMetadataRequest) ProtoMessage() {} + +func (x *QueryMetadataRequest) ProtoReflect() protoreflect.Message { + mi := &file_metastore_v1_query_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use QueryMetadataRequest.ProtoReflect.Descriptor instead. +func (*QueryMetadataRequest) Descriptor() ([]byte, []int) { + return file_metastore_v1_query_proto_rawDescGZIP(), []int{0} +} + +func (x *QueryMetadataRequest) GetTenantId() []string { + if x != nil { + return x.TenantId + } + return nil +} + +func (x *QueryMetadataRequest) GetStartTime() int64 { + if x != nil { + return x.StartTime + } + return 0 +} + +func (x *QueryMetadataRequest) GetEndTime() int64 { + if x != nil { + return x.EndTime + } + return 0 +} + +func (x *QueryMetadataRequest) GetQuery() string { + if x != nil { + return x.Query + } + return "" +} + +type QueryMetadataResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Blocks []*BlockMeta `protobuf:"bytes,1,rep,name=blocks,proto3" json:"blocks,omitempty"` +} + +func (x *QueryMetadataResponse) Reset() { + *x = QueryMetadataResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_metastore_v1_query_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *QueryMetadataResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*QueryMetadataResponse) ProtoMessage() {} + +func (x *QueryMetadataResponse) ProtoReflect() protoreflect.Message { + mi := &file_metastore_v1_query_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use QueryMetadataResponse.ProtoReflect.Descriptor instead. +func (*QueryMetadataResponse) Descriptor() ([]byte, []int) { + return file_metastore_v1_query_proto_rawDescGZIP(), []int{1} +} + +func (x *QueryMetadataResponse) GetBlocks() []*BlockMeta { + if x != nil { + return x.Blocks + } + return nil +} + +type QueryMetadataLabelsRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TenantId []string `protobuf:"bytes,1,rep,name=tenant_id,json=tenantId,proto3" json:"tenant_id,omitempty"` + StartTime int64 `protobuf:"varint,2,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` + EndTime int64 `protobuf:"varint,3,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"` + Query string `protobuf:"bytes,4,opt,name=query,proto3" json:"query,omitempty"` + Labels []string `protobuf:"bytes,5,rep,name=labels,proto3" json:"labels,omitempty"` +} + +func (x *QueryMetadataLabelsRequest) Reset() { + *x = QueryMetadataLabelsRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_metastore_v1_query_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *QueryMetadataLabelsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*QueryMetadataLabelsRequest) ProtoMessage() {} + +func (x *QueryMetadataLabelsRequest) ProtoReflect() protoreflect.Message { + mi := &file_metastore_v1_query_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use QueryMetadataLabelsRequest.ProtoReflect.Descriptor instead. +func (*QueryMetadataLabelsRequest) Descriptor() ([]byte, []int) { + return file_metastore_v1_query_proto_rawDescGZIP(), []int{2} +} + +func (x *QueryMetadataLabelsRequest) GetTenantId() []string { + if x != nil { + return x.TenantId + } + return nil +} + +func (x *QueryMetadataLabelsRequest) GetStartTime() int64 { + if x != nil { + return x.StartTime + } + return 0 +} + +func (x *QueryMetadataLabelsRequest) GetEndTime() int64 { + if x != nil { + return x.EndTime + } + return 0 +} + +func (x *QueryMetadataLabelsRequest) GetQuery() string { + if x != nil { + return x.Query + } + return "" +} + +func (x *QueryMetadataLabelsRequest) GetLabels() []string { + if x != nil { + return x.Labels + } + return nil +} + +type QueryMetadataLabelsResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Labels []*v1.Labels `protobuf:"bytes,1,rep,name=labels,proto3" json:"labels,omitempty"` +} + +func (x *QueryMetadataLabelsResponse) Reset() { + *x = QueryMetadataLabelsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_metastore_v1_query_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *QueryMetadataLabelsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*QueryMetadataLabelsResponse) ProtoMessage() {} + +func (x *QueryMetadataLabelsResponse) ProtoReflect() protoreflect.Message { + mi := &file_metastore_v1_query_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use QueryMetadataLabelsResponse.ProtoReflect.Descriptor instead. +func (*QueryMetadataLabelsResponse) Descriptor() ([]byte, []int) { + return file_metastore_v1_query_proto_rawDescGZIP(), []int{3} +} + +func (x *QueryMetadataLabelsResponse) GetLabels() []*v1.Labels { + if x != nil { + return x.Labels + } + return nil +} + +var File_metastore_v1_query_proto protoreflect.FileDescriptor + +var file_metastore_v1_query_proto_rawDesc = []byte{ + 0x0a, 0x18, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x71, + 0x75, 0x65, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x6d, 0x65, 0x74, 0x61, + 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x18, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, + 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x1a, 0x14, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2f, 0x76, 0x31, 0x2f, 0x74, 0x79, 0x70, + 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x83, 0x01, 0x0a, 0x14, 0x51, 0x75, 0x65, + 0x72, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x1d, + 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x19, 0x0a, + 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, + 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x22, 0x48, + 0x0a, 0x15, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x06, 0x62, 0x6c, 0x6f, 0x63, 0x6b, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, + 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4d, 0x65, 0x74, 0x61, + 0x52, 0x06, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x22, 0xa1, 0x01, 0x0a, 0x1a, 0x51, 0x75, 0x65, + 0x72, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x65, 0x6e, 0x61, 0x6e, + 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x74, 0x65, 0x6e, 0x61, + 0x6e, 0x74, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, + 0x69, 0x6d, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x14, + 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x71, + 0x75, 0x65, 0x72, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x05, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x22, 0x47, 0x0a, 0x1b, + 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x4c, 0x61, 0x62, + 0x65, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x28, 0x0a, 0x06, 0x6c, + 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x74, 0x79, + 0x70, 0x65, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x52, 0x06, 0x6c, + 0x61, 0x62, 0x65, 0x6c, 0x73, 0x32, 0xe0, 0x01, 0x0a, 0x14, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x51, 0x75, 0x65, 0x72, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x5a, + 0x0a, 0x0d, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, + 0x22, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x51, + 0x75, 0x65, 0x72, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, + 0x76, 0x31, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6c, 0x0a, 0x13, 0x51, 0x75, + 0x65, 0x72, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x4c, 0x61, 0x62, 0x65, 0x6c, + 0x73, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, + 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x4c, 0x61, + 0x62, 0x65, 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x65, + 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, + 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0xb7, 0x01, 0x0a, 0x10, 0x63, 0x6f, 0x6d, + 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x42, 0x0a, 0x51, + 0x75, 0x65, 0x72, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x46, 0x67, 0x69, 0x74, + 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x72, 0x61, 0x66, 0x61, 0x6e, 0x61, 0x2f, + 0x70, 0x79, 0x72, 0x6f, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x67, 0x65, + 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x73, + 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x31, 0x3b, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, + 0x65, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4d, 0x58, 0x58, 0xaa, 0x02, 0x0c, 0x4d, 0x65, 0x74, 0x61, + 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0c, 0x4d, 0x65, 0x74, 0x61, 0x73, + 0x74, 0x6f, 0x72, 0x65, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x18, 0x4d, 0x65, 0x74, 0x61, 0x73, 0x74, + 0x6f, 0x72, 0x65, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0xea, 0x02, 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x3a, 0x3a, + 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_metastore_v1_query_proto_rawDescOnce sync.Once + file_metastore_v1_query_proto_rawDescData = file_metastore_v1_query_proto_rawDesc +) + +func file_metastore_v1_query_proto_rawDescGZIP() []byte { + file_metastore_v1_query_proto_rawDescOnce.Do(func() { + file_metastore_v1_query_proto_rawDescData = protoimpl.X.CompressGZIP(file_metastore_v1_query_proto_rawDescData) + }) + return file_metastore_v1_query_proto_rawDescData +} + +var file_metastore_v1_query_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_metastore_v1_query_proto_goTypes = []any{ + (*QueryMetadataRequest)(nil), // 0: metastore.v1.QueryMetadataRequest + (*QueryMetadataResponse)(nil), // 1: metastore.v1.QueryMetadataResponse + (*QueryMetadataLabelsRequest)(nil), // 2: metastore.v1.QueryMetadataLabelsRequest + (*QueryMetadataLabelsResponse)(nil), // 3: metastore.v1.QueryMetadataLabelsResponse + (*BlockMeta)(nil), // 4: metastore.v1.BlockMeta + (*v1.Labels)(nil), // 5: types.v1.Labels +} +var file_metastore_v1_query_proto_depIdxs = []int32{ + 4, // 0: metastore.v1.QueryMetadataResponse.blocks:type_name -> metastore.v1.BlockMeta + 5, // 1: metastore.v1.QueryMetadataLabelsResponse.labels:type_name -> types.v1.Labels + 0, // 2: metastore.v1.MetadataQueryService.QueryMetadata:input_type -> metastore.v1.QueryMetadataRequest + 2, // 3: metastore.v1.MetadataQueryService.QueryMetadataLabels:input_type -> metastore.v1.QueryMetadataLabelsRequest + 1, // 4: metastore.v1.MetadataQueryService.QueryMetadata:output_type -> metastore.v1.QueryMetadataResponse + 3, // 5: metastore.v1.MetadataQueryService.QueryMetadataLabels:output_type -> metastore.v1.QueryMetadataLabelsResponse + 4, // [4:6] is the sub-list for method output_type + 2, // [2:4] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_metastore_v1_query_proto_init() } +func file_metastore_v1_query_proto_init() { + if File_metastore_v1_query_proto != nil { + return + } + file_metastore_v1_types_proto_init() + if !protoimpl.UnsafeEnabled { + file_metastore_v1_query_proto_msgTypes[0].Exporter = func(v any, i int) any { + switch v := v.(*QueryMetadataRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_metastore_v1_query_proto_msgTypes[1].Exporter = func(v any, i int) any { + switch v := v.(*QueryMetadataResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_metastore_v1_query_proto_msgTypes[2].Exporter = func(v any, i int) any { + switch v := v.(*QueryMetadataLabelsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_metastore_v1_query_proto_msgTypes[3].Exporter = func(v any, i int) any { + switch v := v.(*QueryMetadataLabelsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_metastore_v1_query_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_metastore_v1_query_proto_goTypes, + DependencyIndexes: file_metastore_v1_query_proto_depIdxs, + MessageInfos: file_metastore_v1_query_proto_msgTypes, + }.Build() + File_metastore_v1_query_proto = out.File + file_metastore_v1_query_proto_rawDesc = nil + file_metastore_v1_query_proto_goTypes = nil + file_metastore_v1_query_proto_depIdxs = nil +} diff --git a/api/gen/proto/go/metastore/v1/metadata_query_vtproto.pb.go b/api/gen/proto/go/metastore/v1/query_vtproto.pb.go similarity index 50% rename from api/gen/proto/go/metastore/v1/metadata_query_vtproto.pb.go rename to api/gen/proto/go/metastore/v1/query_vtproto.pb.go index 2c3cebcc68..127133631d 100644 --- a/api/gen/proto/go/metastore/v1/metadata_query_vtproto.pb.go +++ b/api/gen/proto/go/metastore/v1/query_vtproto.pb.go @@ -1,12 +1,13 @@ // Code generated by protoc-gen-go-vtproto. DO NOT EDIT. // protoc-gen-go-vtproto version: v0.6.0 -// source: metastore/v1/metadata_query.proto +// source: metastore/v1/query.proto package metastorev1 import ( context "context" fmt "fmt" + v1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" protohelpers "github.com/planetscale/vtprotobuf/protohelpers" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" @@ -70,6 +71,62 @@ func (m *QueryMetadataResponse) CloneMessageVT() proto.Message { return m.CloneVT() } +func (m *QueryMetadataLabelsRequest) CloneVT() *QueryMetadataLabelsRequest { + if m == nil { + return (*QueryMetadataLabelsRequest)(nil) + } + r := new(QueryMetadataLabelsRequest) + r.StartTime = m.StartTime + r.EndTime = m.EndTime + r.Query = m.Query + if rhs := m.TenantId; rhs != nil { + tmpContainer := make([]string, len(rhs)) + copy(tmpContainer, rhs) + r.TenantId = tmpContainer + } + if rhs := m.Labels; rhs != nil { + tmpContainer := make([]string, len(rhs)) + copy(tmpContainer, rhs) + r.Labels = tmpContainer + } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *QueryMetadataLabelsRequest) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *QueryMetadataLabelsResponse) CloneVT() *QueryMetadataLabelsResponse { + if m == nil { + return (*QueryMetadataLabelsResponse)(nil) + } + r := new(QueryMetadataLabelsResponse) + if rhs := m.Labels; rhs != nil { + tmpContainer := make([]*v1.Labels, len(rhs)) + for k, v := range rhs { + if vtpb, ok := interface{}(v).(interface{ CloneVT() *v1.Labels }); ok { + tmpContainer[k] = vtpb.CloneVT() + } else { + tmpContainer[k] = proto.Clone(v).(*v1.Labels) + } + } + r.Labels = tmpContainer + } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *QueryMetadataLabelsResponse) CloneMessageVT() proto.Message { + return m.CloneVT() +} + func (this *QueryMetadataRequest) EqualVT(that *QueryMetadataRequest) bool { if this == that { return true @@ -137,6 +194,86 @@ func (this *QueryMetadataResponse) EqualMessageVT(thatMsg proto.Message) bool { } return this.EqualVT(that) } +func (this *QueryMetadataLabelsRequest) EqualVT(that *QueryMetadataLabelsRequest) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if len(this.TenantId) != len(that.TenantId) { + return false + } + for i, vx := range this.TenantId { + vy := that.TenantId[i] + if vx != vy { + return false + } + } + if this.StartTime != that.StartTime { + return false + } + if this.EndTime != that.EndTime { + return false + } + if this.Query != that.Query { + return false + } + if len(this.Labels) != len(that.Labels) { + return false + } + for i, vx := range this.Labels { + vy := that.Labels[i] + if vx != vy { + return false + } + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *QueryMetadataLabelsRequest) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*QueryMetadataLabelsRequest) + if !ok { + return false + } + return this.EqualVT(that) +} +func (this *QueryMetadataLabelsResponse) EqualVT(that *QueryMetadataLabelsResponse) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if len(this.Labels) != len(that.Labels) { + return false + } + for i, vx := range this.Labels { + vy := that.Labels[i] + if p, q := vx, vy; p != q { + if p == nil { + p = &v1.Labels{} + } + if q == nil { + q = &v1.Labels{} + } + if equal, ok := interface{}(p).(interface{ EqualVT(*v1.Labels) bool }); ok { + if !equal.EqualVT(q) { + return false + } + } else if !proto.Equal(p, q) { + return false + } + } + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *QueryMetadataLabelsResponse) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*QueryMetadataLabelsResponse) + if !ok { + return false + } + return this.EqualVT(that) +} // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. @@ -148,6 +285,7 @@ const _ = grpc.SupportPackageIsVersion7 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type MetadataQueryServiceClient interface { QueryMetadata(ctx context.Context, in *QueryMetadataRequest, opts ...grpc.CallOption) (*QueryMetadataResponse, error) + QueryMetadataLabels(ctx context.Context, in *QueryMetadataLabelsRequest, opts ...grpc.CallOption) (*QueryMetadataLabelsResponse, error) } type metadataQueryServiceClient struct { @@ -167,11 +305,21 @@ func (c *metadataQueryServiceClient) QueryMetadata(ctx context.Context, in *Quer return out, nil } +func (c *metadataQueryServiceClient) QueryMetadataLabels(ctx context.Context, in *QueryMetadataLabelsRequest, opts ...grpc.CallOption) (*QueryMetadataLabelsResponse, error) { + out := new(QueryMetadataLabelsResponse) + err := c.cc.Invoke(ctx, "/metastore.v1.MetadataQueryService/QueryMetadataLabels", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // MetadataQueryServiceServer is the server API for MetadataQueryService service. // All implementations must embed UnimplementedMetadataQueryServiceServer // for forward compatibility type MetadataQueryServiceServer interface { QueryMetadata(context.Context, *QueryMetadataRequest) (*QueryMetadataResponse, error) + QueryMetadataLabels(context.Context, *QueryMetadataLabelsRequest) (*QueryMetadataLabelsResponse, error) mustEmbedUnimplementedMetadataQueryServiceServer() } @@ -182,6 +330,9 @@ type UnimplementedMetadataQueryServiceServer struct { func (UnimplementedMetadataQueryServiceServer) QueryMetadata(context.Context, *QueryMetadataRequest) (*QueryMetadataResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method QueryMetadata not implemented") } +func (UnimplementedMetadataQueryServiceServer) QueryMetadataLabels(context.Context, *QueryMetadataLabelsRequest) (*QueryMetadataLabelsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method QueryMetadataLabels not implemented") +} func (UnimplementedMetadataQueryServiceServer) mustEmbedUnimplementedMetadataQueryServiceServer() {} // UnsafeMetadataQueryServiceServer may be embedded to opt out of forward compatibility for this service. @@ -213,6 +364,24 @@ func _MetadataQueryService_QueryMetadata_Handler(srv interface{}, ctx context.Co return interceptor(ctx, in, info, handler) } +func _MetadataQueryService_QueryMetadataLabels_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(QueryMetadataLabelsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MetadataQueryServiceServer).QueryMetadataLabels(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/metastore.v1.MetadataQueryService/QueryMetadataLabels", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetadataQueryServiceServer).QueryMetadataLabels(ctx, req.(*QueryMetadataLabelsRequest)) + } + return interceptor(ctx, in, info, handler) +} + // MetadataQueryService_ServiceDesc is the grpc.ServiceDesc for MetadataQueryService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -224,9 +393,13 @@ var MetadataQueryService_ServiceDesc = grpc.ServiceDesc{ MethodName: "QueryMetadata", Handler: _MetadataQueryService_QueryMetadata_Handler, }, + { + MethodName: "QueryMetadataLabels", + Handler: _MetadataQueryService_QueryMetadataLabels_Handler, + }, }, Streams: []grpc.StreamDesc{}, - Metadata: "metastore/v1/metadata_query.proto", + Metadata: "metastore/v1/query.proto", } func (m *QueryMetadataRequest) MarshalVT() (dAtA []byte, err error) { @@ -333,6 +506,131 @@ func (m *QueryMetadataResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) return len(dAtA) - i, nil } +func (m *QueryMetadataLabelsRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *QueryMetadataLabelsRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *QueryMetadataLabelsRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Labels) > 0 { + for iNdEx := len(m.Labels) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Labels[iNdEx]) + copy(dAtA[i:], m.Labels[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Labels[iNdEx]))) + i-- + dAtA[i] = 0x2a + } + } + if len(m.Query) > 0 { + i -= len(m.Query) + copy(dAtA[i:], m.Query) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Query))) + i-- + dAtA[i] = 0x22 + } + if m.EndTime != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.EndTime)) + i-- + dAtA[i] = 0x18 + } + if m.StartTime != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.StartTime)) + i-- + dAtA[i] = 0x10 + } + if len(m.TenantId) > 0 { + for iNdEx := len(m.TenantId) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.TenantId[iNdEx]) + copy(dAtA[i:], m.TenantId[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.TenantId[iNdEx]))) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *QueryMetadataLabelsResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *QueryMetadataLabelsResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *QueryMetadataLabelsResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Labels) > 0 { + for iNdEx := len(m.Labels) - 1; iNdEx >= 0; iNdEx-- { + if vtmsg, ok := interface{}(m.Labels[iNdEx]).(interface { + MarshalToSizedBufferVT([]byte) (int, error) + }); ok { + size, err := vtmsg.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + } else { + encoded, err := proto.Marshal(m.Labels[iNdEx]) + if err != nil { + return 0, err + } + i -= len(encoded) + copy(dAtA[i:], encoded) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(encoded))) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func (m *QueryMetadataRequest) SizeVT() (n int) { if m == nil { return 0 @@ -375,35 +673,89 @@ func (m *QueryMetadataResponse) SizeVT() (n int) { return n } -func (m *QueryMetadataRequest) UnmarshalVT(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: QueryMetadataRequest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: QueryMetadataRequest: illegal tag %d (wire type %d)", fieldNum, wire) +func (m *QueryMetadataLabelsRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.TenantId) > 0 { + for _, s := range m.TenantId { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } - switch fieldNum { + } + if m.StartTime != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.StartTime)) + } + if m.EndTime != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.EndTime)) + } + l = len(m.Query) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.Labels) > 0 { + for _, s := range m.Labels { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *QueryMetadataLabelsResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Labels) > 0 { + for _, e := range m.Labels { + if size, ok := interface{}(e).(interface { + SizeVT() int + }); ok { + l = size.SizeVT() + } else { + l = proto.Size(e) + } + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *QueryMetadataRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: QueryMetadataRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: QueryMetadataRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { case 1: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field TenantId", wireType) @@ -613,3 +965,281 @@ func (m *QueryMetadataResponse) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *QueryMetadataLabelsRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: QueryMetadataLabelsRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: QueryMetadataLabelsRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TenantId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TenantId = append(m.TenantId, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StartTime", wireType) + } + m.StartTime = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StartTime |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field EndTime", wireType) + } + m.EndTime = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.EndTime |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Query", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Query = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Labels = append(m.Labels, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *QueryMetadataLabelsResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: QueryMetadataLabelsResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: QueryMetadataLabelsResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Labels = append(m.Labels, &v1.Labels{}) + if unmarshal, ok := interface{}(m.Labels[len(m.Labels)-1]).(interface { + UnmarshalVT([]byte) error + }); ok { + if err := unmarshal.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + } else { + if err := proto.Unmarshal(dAtA[iNdEx:postIndex], m.Labels[len(m.Labels)-1]); err != nil { + return err + } + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/api/gen/proto/go/metastore/v1/types.pb.go b/api/gen/proto/go/metastore/v1/types.pb.go index 6df3108ee6..be113e760b 100644 --- a/api/gen/proto/go/metastore/v1/types.pb.go +++ b/api/gen/proto/go/metastore/v1/types.pb.go @@ -20,69 +20,14 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -type BlockList struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Tenant string `protobuf:"bytes,1,opt,name=tenant,proto3" json:"tenant,omitempty"` - Shard uint32 `protobuf:"varint,2,opt,name=shard,proto3" json:"shard,omitempty"` - Blocks []string `protobuf:"bytes,3,rep,name=blocks,proto3" json:"blocks,omitempty"` -} - -func (x *BlockList) Reset() { - *x = BlockList{} - if protoimpl.UnsafeEnabled { - mi := &file_metastore_v1_types_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BlockList) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BlockList) ProtoMessage() {} - -func (x *BlockList) ProtoReflect() protoreflect.Message { - mi := &file_metastore_v1_types_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BlockList.ProtoReflect.Descriptor instead. -func (*BlockList) Descriptor() ([]byte, []int) { - return file_metastore_v1_types_proto_rawDescGZIP(), []int{0} -} - -func (x *BlockList) GetTenant() string { - if x != nil { - return x.Tenant - } - return "" -} - -func (x *BlockList) GetShard() uint32 { - if x != nil { - return x.Shard - } - return 0 -} - -func (x *BlockList) GetBlocks() []string { - if x != nil { - return x.Blocks - } - return nil -} - +// BlockMeta is a metadata entry that describes the block's contents. A block +// is a collection of datasets that share certain properties, such as shard ID, +// compaction level, tenant ID, time range, creation time, and more. +// +// The block content's format denotes the binary format of the datasets and the +// metadata entry (to address logical dependencies). Each dataset has its own +// table of contents that lists the sections within the dataset. Each dataset +// has its own set of attributes (labels) that describe its specific contents. type BlockMeta struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -93,7 +38,7 @@ type BlockMeta struct { // This is the only field that is not included into // the string table. Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` - // Optional. Empty if compaction level is 0. + // If empty, datasets belong to distinct tenants. Tenant int32 `protobuf:"varint,3,opt,name=tenant,proto3" json:"tenant,omitempty"` Shard uint32 `protobuf:"varint,4,opt,name=shard,proto3" json:"shard,omitempty"` CompactionLevel uint32 `protobuf:"varint,5,opt,name=compaction_level,json=compactionLevel,proto3" json:"compaction_level,omitempty"` @@ -110,7 +55,7 @@ type BlockMeta struct { func (x *BlockMeta) Reset() { *x = BlockMeta{} if protoimpl.UnsafeEnabled { - mi := &file_metastore_v1_types_proto_msgTypes[1] + mi := &file_metastore_v1_types_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -123,7 +68,7 @@ func (x *BlockMeta) String() string { func (*BlockMeta) ProtoMessage() {} func (x *BlockMeta) ProtoReflect() protoreflect.Message { - mi := &file_metastore_v1_types_proto_msgTypes[1] + mi := &file_metastore_v1_types_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -136,7 +81,7 @@ func (x *BlockMeta) ProtoReflect() protoreflect.Message { // Deprecated: Use BlockMeta.ProtoReflect.Descriptor instead. func (*BlockMeta) Descriptor() ([]byte, []int) { - return file_metastore_v1_types_proto_rawDescGZIP(), []int{1} + return file_metastore_v1_types_proto_rawDescGZIP(), []int{0} } func (x *BlockMeta) GetFormatVersion() uint32 { @@ -234,19 +179,66 @@ type Dataset struct { // - 1: index.tsdb // - 2: symbols.symdb TableOfContents []uint64 `protobuf:"varint,5,rep,packed,name=table_of_contents,json=tableOfContents,proto3" json:"table_of_contents,omitempty"` - // Size of the section in bytes. + // Size of the dataset in bytes. Size uint64 `protobuf:"varint,6,opt,name=size,proto3" json:"size,omitempty"` - // TODO: Delete. Use labels instead. - // Profile types present in the tenant service data. - ProfileTypes []int32 `protobuf:"varint,7,rep,packed,name=profile_types,json=profileTypes,proto3" json:"profile_types,omitempty"` // Length prefixed label key-value pairs. + // + // Multiple label sets can be associated with a dataset to denote relationships + // across multiple dimensions. For example, each dataset currently stores data + // for multiple profile types: + // - service_name=A, profile_type=cpu + // - service_name=A, profile_type=memory + // + // Labels are primarily used to filter datasets based on their attributes. + // For instance, labels can be used to select datasets containing a specific + // service. + // + // The set of attributes is extensible and can grow over time. For example, a + // namespace attribute could be added to datasets: + // - service_name=A, profile_type=cpu + // - service_name=A, profile_type=memory + // - service_name=B, namespace=N, profile_type=cpu + // - service_name=B, namespace=N, profile_type=memory + // - service_name=C, namespace=N, profile_type=cpu + // - service_name=C, namespace=N, profile_type=memory + // + // This organization enables querying datasets by namespace without accessing + // the block contents, which significantly improves performance. + // + // Metadata labels are not required to be included in the block's TSDB index + // and may be orthogonal to the data dimensions. Generally, attributes serve + // two primary purposes: + // - To create data scopes that span multiple service, reducing the need to + // scan the entire set of block satisfying the query expression, i.e., + // the time range and tenant ID. + // - To provide additional information about datasets without altering the + // storage schema or access methods. + // + // For example, this approach can support cost attribution or similar breakdown + // analyses. It can also handle data dependencies (e.g., links to external data) + // using labels. + // + // The cardinality of the labels is expected to remain relatively low (fewer + // than a million unique combinations globally). However, this depends on the + // metadata storage system. + // + // Metadata labels are represented as a slice of `int32` values that refer to + // strings in the metadata entry's string table. The slice is a sequence of + // length-prefixed key-value (KV) pairs: + // + // len(2) | k1 | v1 | k2 | v2 | len(3) | k1 | v3 | k2 | v4 | k3 | v5 + // + // The order of KV pairs is not defined. The format is optimized for indexing + // rather than querying, and it is not intended to be the most space-efficient + // representation. Since entries are supposed to be indexed, the redundancy of + // denormalized relationships is not a concern. Labels []int32 `protobuf:"varint,8,rep,packed,name=labels,proto3" json:"labels,omitempty"` } func (x *Dataset) Reset() { *x = Dataset{} if protoimpl.UnsafeEnabled { - mi := &file_metastore_v1_types_proto_msgTypes[2] + mi := &file_metastore_v1_types_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -259,7 +251,7 @@ func (x *Dataset) String() string { func (*Dataset) ProtoMessage() {} func (x *Dataset) ProtoReflect() protoreflect.Message { - mi := &file_metastore_v1_types_proto_msgTypes[2] + mi := &file_metastore_v1_types_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -272,7 +264,7 @@ func (x *Dataset) ProtoReflect() protoreflect.Message { // Deprecated: Use Dataset.ProtoReflect.Descriptor instead. func (*Dataset) Descriptor() ([]byte, []int) { - return file_metastore_v1_types_proto_rawDescGZIP(), []int{2} + return file_metastore_v1_types_proto_rawDescGZIP(), []int{1} } func (x *Dataset) GetTenant() int32 { @@ -317,16 +309,72 @@ func (x *Dataset) GetSize() uint64 { return 0 } -func (x *Dataset) GetProfileTypes() []int32 { +func (x *Dataset) GetLabels() []int32 { if x != nil { - return x.ProfileTypes + return x.Labels } return nil } -func (x *Dataset) GetLabels() []int32 { +type BlockList struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Tenant string `protobuf:"bytes,1,opt,name=tenant,proto3" json:"tenant,omitempty"` + Shard uint32 `protobuf:"varint,2,opt,name=shard,proto3" json:"shard,omitempty"` + Blocks []string `protobuf:"bytes,3,rep,name=blocks,proto3" json:"blocks,omitempty"` +} + +func (x *BlockList) Reset() { + *x = BlockList{} + if protoimpl.UnsafeEnabled { + mi := &file_metastore_v1_types_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BlockList) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BlockList) ProtoMessage() {} + +func (x *BlockList) ProtoReflect() protoreflect.Message { + mi := &file_metastore_v1_types_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BlockList.ProtoReflect.Descriptor instead. +func (*BlockList) Descriptor() ([]byte, []int) { + return file_metastore_v1_types_proto_rawDescGZIP(), []int{2} +} + +func (x *BlockList) GetTenant() string { if x != nil { - return x.Labels + return x.Tenant + } + return "" +} + +func (x *BlockList) GetShard() uint32 { + if x != nil { + return x.Shard + } + return 0 +} + +func (x *BlockList) GetBlocks() []string { + if x != nil { + return x.Blocks } return nil } @@ -336,61 +384,59 @@ var File_metastore_v1_types_proto protoreflect.FileDescriptor var file_metastore_v1_types_proto_rawDesc = []byte{ 0x0a, 0x18, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x6d, 0x65, 0x74, 0x61, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x22, 0x51, 0x0a, 0x09, 0x42, 0x6c, 0x6f, 0x63, - 0x6b, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x12, 0x14, 0x0a, - 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x73, 0x68, - 0x61, 0x72, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x06, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x22, 0xda, 0x02, 0x0a, 0x09, - 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x6f, 0x72, - 0x6d, 0x61, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x0d, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, - 0x12, 0x16, 0x0a, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, - 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x29, - 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6c, 0x65, 0x76, - 0x65, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x69, 0x6e, - 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x6d, 0x69, 0x6e, - 0x54, 0x69, 0x6d, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x61, 0x78, 0x5f, 0x74, 0x69, 0x6d, 0x65, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x6d, 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x12, - 0x1d, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x62, 0x79, 0x18, 0x08, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x79, 0x12, 0x12, - 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, 0x69, - 0x7a, 0x65, 0x12, 0x31, 0x0a, 0x08, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x73, 0x18, 0x0a, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x52, 0x08, 0x64, 0x61, 0x74, - 0x61, 0x73, 0x65, 0x74, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, - 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x74, 0x72, - 0x69, 0x6e, 0x67, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x22, 0xe8, 0x01, 0x0a, 0x07, 0x44, 0x61, 0x74, - 0x61, 0x73, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x69, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x07, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6d, - 0x61, 0x78, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x6d, - 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, - 0x6f, 0x66, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, - 0x04, 0x52, 0x0f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4f, 0x66, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, - 0x74, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, - 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x05, 0x52, 0x0c, 0x70, - 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6c, - 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x05, 0x52, 0x06, 0x6c, 0x61, 0x62, - 0x65, 0x6c, 0x73, 0x42, 0xb7, 0x01, 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x65, 0x74, 0x61, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x42, 0x0a, 0x54, 0x79, 0x70, 0x65, 0x73, 0x50, - 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x46, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x67, 0x72, 0x61, 0x66, 0x61, 0x6e, 0x61, 0x2f, 0x70, 0x79, 0x72, 0x6f, 0x73, - 0x63, 0x6f, 0x70, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, - 0x76, 0x31, 0x3b, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0xa2, 0x02, - 0x03, 0x4d, 0x58, 0x58, 0xaa, 0x02, 0x0c, 0x4d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0c, 0x4d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5c, - 0x56, 0x31, 0xe2, 0x02, 0x18, 0x4d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5c, 0x56, - 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0d, - 0x4d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x22, 0xda, 0x02, 0x0a, 0x09, 0x42, 0x6c, 0x6f, + 0x63, 0x6b, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, + 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, + 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, + 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x74, + 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x63, + 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x69, 0x6e, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, + 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x61, 0x78, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x07, 0x6d, 0x61, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, + 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x62, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x73, + 0x69, 0x7a, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, + 0x31, 0x0a, 0x08, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, + 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x52, 0x08, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, + 0x74, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x74, 0x61, 0x62, + 0x6c, 0x65, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, + 0x54, 0x61, 0x62, 0x6c, 0x65, 0x22, 0xc9, 0x01, 0x0a, 0x07, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, + 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x19, 0x0a, + 0x08, 0x6d, 0x69, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x07, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x61, 0x78, 0x5f, + 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x6d, 0x61, 0x78, 0x54, + 0x69, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6f, 0x66, 0x5f, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x04, 0x52, 0x0f, + 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4f, 0x66, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x12, + 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x73, + 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x08, 0x20, + 0x03, 0x28, 0x05, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x4a, 0x04, 0x08, 0x07, 0x10, + 0x08, 0x22, 0x51, 0x0a, 0x09, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x16, + 0x0a, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x16, 0x0a, 0x06, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x62, 0x6c, + 0x6f, 0x63, 0x6b, 0x73, 0x42, 0xb7, 0x01, 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x65, 0x74, + 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x42, 0x0a, 0x54, 0x79, 0x70, 0x65, 0x73, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x46, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x72, 0x61, 0x66, 0x61, 0x6e, 0x61, 0x2f, 0x70, 0x79, 0x72, 0x6f, + 0x73, 0x63, 0x6f, 0x70, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, + 0x2f, 0x76, 0x31, 0x3b, 0x6d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0xa2, + 0x02, 0x03, 0x4d, 0x58, 0x58, 0xaa, 0x02, 0x0c, 0x4d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, + 0x65, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0c, 0x4d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, + 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x18, 0x4d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5c, + 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, + 0x0d, 0x4d, 0x65, 0x74, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -407,12 +453,12 @@ func file_metastore_v1_types_proto_rawDescGZIP() []byte { var file_metastore_v1_types_proto_msgTypes = make([]protoimpl.MessageInfo, 3) var file_metastore_v1_types_proto_goTypes = []any{ - (*BlockList)(nil), // 0: metastore.v1.BlockList - (*BlockMeta)(nil), // 1: metastore.v1.BlockMeta - (*Dataset)(nil), // 2: metastore.v1.Dataset + (*BlockMeta)(nil), // 0: metastore.v1.BlockMeta + (*Dataset)(nil), // 1: metastore.v1.Dataset + (*BlockList)(nil), // 2: metastore.v1.BlockList } var file_metastore_v1_types_proto_depIdxs = []int32{ - 2, // 0: metastore.v1.BlockMeta.datasets:type_name -> metastore.v1.Dataset + 1, // 0: metastore.v1.BlockMeta.datasets:type_name -> metastore.v1.Dataset 1, // [1:1] is the sub-list for method output_type 1, // [1:1] is the sub-list for method input_type 1, // [1:1] is the sub-list for extension type_name @@ -427,7 +473,7 @@ func file_metastore_v1_types_proto_init() { } if !protoimpl.UnsafeEnabled { file_metastore_v1_types_proto_msgTypes[0].Exporter = func(v any, i int) any { - switch v := v.(*BlockList); i { + switch v := v.(*BlockMeta); i { case 0: return &v.state case 1: @@ -439,7 +485,7 @@ func file_metastore_v1_types_proto_init() { } } file_metastore_v1_types_proto_msgTypes[1].Exporter = func(v any, i int) any { - switch v := v.(*BlockMeta); i { + switch v := v.(*Dataset); i { case 0: return &v.state case 1: @@ -451,7 +497,7 @@ func file_metastore_v1_types_proto_init() { } } file_metastore_v1_types_proto_msgTypes[2].Exporter = func(v any, i int) any { - switch v := v.(*Dataset); i { + switch v := v.(*BlockList); i { case 0: return &v.state case 1: diff --git a/api/gen/proto/go/metastore/v1/types_vtproto.pb.go b/api/gen/proto/go/metastore/v1/types_vtproto.pb.go index 349ea3dd4f..f563ae2d8a 100644 --- a/api/gen/proto/go/metastore/v1/types_vtproto.pb.go +++ b/api/gen/proto/go/metastore/v1/types_vtproto.pb.go @@ -19,29 +19,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -func (m *BlockList) CloneVT() *BlockList { - if m == nil { - return (*BlockList)(nil) - } - r := new(BlockList) - r.Tenant = m.Tenant - r.Shard = m.Shard - if rhs := m.Blocks; rhs != nil { - tmpContainer := make([]string, len(rhs)) - copy(tmpContainer, rhs) - r.Blocks = tmpContainer - } - if len(m.unknownFields) > 0 { - r.unknownFields = make([]byte, len(m.unknownFields)) - copy(r.unknownFields, m.unknownFields) - } - return r -} - -func (m *BlockList) CloneMessageVT() proto.Message { - return m.CloneVT() -} - func (m *BlockMeta) CloneVT() *BlockMeta { if m == nil { return (*BlockMeta)(nil) @@ -94,11 +71,6 @@ func (m *Dataset) CloneVT() *Dataset { copy(tmpContainer, rhs) r.TableOfContents = tmpContainer } - if rhs := m.ProfileTypes; rhs != nil { - tmpContainer := make([]int32, len(rhs)) - copy(tmpContainer, rhs) - r.ProfileTypes = tmpContainer - } if rhs := m.Labels; rhs != nil { tmpContainer := make([]int32, len(rhs)) copy(tmpContainer, rhs) @@ -115,37 +87,29 @@ func (m *Dataset) CloneMessageVT() proto.Message { return m.CloneVT() } -func (this *BlockList) EqualVT(that *BlockList) bool { - if this == that { - return true - } else if this == nil || that == nil { - return false - } - if this.Tenant != that.Tenant { - return false - } - if this.Shard != that.Shard { - return false +func (m *BlockList) CloneVT() *BlockList { + if m == nil { + return (*BlockList)(nil) } - if len(this.Blocks) != len(that.Blocks) { - return false + r := new(BlockList) + r.Tenant = m.Tenant + r.Shard = m.Shard + if rhs := m.Blocks; rhs != nil { + tmpContainer := make([]string, len(rhs)) + copy(tmpContainer, rhs) + r.Blocks = tmpContainer } - for i, vx := range this.Blocks { - vy := that.Blocks[i] - if vx != vy { - return false - } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) } - return string(this.unknownFields) == string(that.unknownFields) + return r } -func (this *BlockList) EqualMessageVT(thatMsg proto.Message) bool { - that, ok := thatMsg.(*BlockList) - if !ok { - return false - } - return this.EqualVT(that) +func (m *BlockList) CloneMessageVT() proto.Message { + return m.CloneVT() } + func (this *BlockMeta) EqualVT(that *BlockMeta) bool { if this == that { return true @@ -245,15 +209,6 @@ func (this *Dataset) EqualVT(that *Dataset) bool { if this.Size != that.Size { return false } - if len(this.ProfileTypes) != len(that.ProfileTypes) { - return false - } - for i, vx := range this.ProfileTypes { - vy := that.ProfileTypes[i] - if vx != vy { - return false - } - } if len(this.Labels) != len(that.Labels) { return false } @@ -273,60 +228,37 @@ func (this *Dataset) EqualMessageVT(thatMsg proto.Message) bool { } return this.EqualVT(that) } -func (m *BlockList) MarshalVT() (dAtA []byte, err error) { - if m == nil { - return nil, nil +func (this *BlockList) EqualVT(that *BlockList) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false } - size := m.SizeVT() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBufferVT(dAtA[:size]) - if err != nil { - return nil, err + if this.Tenant != that.Tenant { + return false } - return dAtA[:n], nil -} - -func (m *BlockList) MarshalToVT(dAtA []byte) (int, error) { - size := m.SizeVT() - return m.MarshalToSizedBufferVT(dAtA[:size]) -} - -func (m *BlockList) MarshalToSizedBufferVT(dAtA []byte) (int, error) { - if m == nil { - return 0, nil + if this.Shard != that.Shard { + return false } - i := len(dAtA) - _ = i - var l int - _ = l - if m.unknownFields != nil { - i -= len(m.unknownFields) - copy(dAtA[i:], m.unknownFields) + if len(this.Blocks) != len(that.Blocks) { + return false } - if len(m.Blocks) > 0 { - for iNdEx := len(m.Blocks) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.Blocks[iNdEx]) - copy(dAtA[i:], m.Blocks[iNdEx]) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Blocks[iNdEx]))) - i-- - dAtA[i] = 0x1a + for i, vx := range this.Blocks { + vy := that.Blocks[i] + if vx != vy { + return false } } - if m.Shard != 0 { - i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Shard)) - i-- - dAtA[i] = 0x10 - } - if len(m.Tenant) > 0 { - i -= len(m.Tenant) - copy(dAtA[i:], m.Tenant) - i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Tenant))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil + return string(this.unknownFields) == string(that.unknownFields) } +func (this *BlockList) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*BlockList) + if !ok { + return false + } + return this.EqualVT(that) +} func (m *BlockMeta) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -479,15 +411,19 @@ func (m *Dataset) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i-- dAtA[i] = 0x42 } - if len(m.ProfileTypes) > 0 { + if m.Size != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Size)) + i-- + dAtA[i] = 0x30 + } + if len(m.TableOfContents) > 0 { var pksize4 int - for _, num := range m.ProfileTypes { + for _, num := range m.TableOfContents { pksize4 += protohelpers.SizeOfVarint(uint64(num)) } i -= pksize4 j3 := i - for _, num1 := range m.ProfileTypes { - num := uint64(num1) + for _, num := range m.TableOfContents { for num >= 1<<7 { dAtA[j3] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -498,31 +434,6 @@ func (m *Dataset) MarshalToSizedBufferVT(dAtA []byte) (int, error) { } i = protohelpers.EncodeVarint(dAtA, i, uint64(pksize4)) i-- - dAtA[i] = 0x3a - } - if m.Size != 0 { - i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Size)) - i-- - dAtA[i] = 0x30 - } - if len(m.TableOfContents) > 0 { - var pksize6 int - for _, num := range m.TableOfContents { - pksize6 += protohelpers.SizeOfVarint(uint64(num)) - } - i -= pksize6 - j5 := i - for _, num := range m.TableOfContents { - for num >= 1<<7 { - dAtA[j5] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j5++ - } - dAtA[j5] = uint8(num) - j5++ - } - i = protohelpers.EncodeVarint(dAtA, i, uint64(pksize6)) - i-- dAtA[i] = 0x2a } if m.MaxTime != 0 { @@ -548,27 +459,58 @@ func (m *Dataset) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *BlockList) SizeVT() (n int) { +func (m *BlockList) MarshalVT() (dAtA []byte, err error) { if m == nil { - return 0 + return nil, nil } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *BlockList) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *BlockList) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i var l int _ = l - l = len(m.Tenant) - if l > 0 { - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) - } - if m.Shard != 0 { - n += 1 + protohelpers.SizeOfVarint(uint64(m.Shard)) + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) } if len(m.Blocks) > 0 { - for _, s := range m.Blocks { - l = len(s) - n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + for iNdEx := len(m.Blocks) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Blocks[iNdEx]) + copy(dAtA[i:], m.Blocks[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Blocks[iNdEx]))) + i-- + dAtA[i] = 0x1a } } - n += len(m.unknownFields) - return n + if m.Shard != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Shard)) + i-- + dAtA[i] = 0x10 + } + if len(m.Tenant) > 0 { + i -= len(m.Tenant) + copy(dAtA[i:], m.Tenant) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Tenant))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *BlockMeta) SizeVT() (n int) { @@ -622,185 +564,67 @@ func (m *BlockMeta) SizeVT() (n int) { } func (m *Dataset) SizeVT() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Tenant != 0 { - n += 1 + protohelpers.SizeOfVarint(uint64(m.Tenant)) - } - if m.Name != 0 { - n += 1 + protohelpers.SizeOfVarint(uint64(m.Name)) - } - if m.MinTime != 0 { - n += 1 + protohelpers.SizeOfVarint(uint64(m.MinTime)) - } - if m.MaxTime != 0 { - n += 1 + protohelpers.SizeOfVarint(uint64(m.MaxTime)) - } - if len(m.TableOfContents) > 0 { - l = 0 - for _, e := range m.TableOfContents { - l += protohelpers.SizeOfVarint(uint64(e)) - } - n += 1 + protohelpers.SizeOfVarint(uint64(l)) + l - } - if m.Size != 0 { - n += 1 + protohelpers.SizeOfVarint(uint64(m.Size)) - } - if len(m.ProfileTypes) > 0 { - l = 0 - for _, e := range m.ProfileTypes { - l += protohelpers.SizeOfVarint(uint64(e)) - } - n += 1 + protohelpers.SizeOfVarint(uint64(l)) + l - } - if len(m.Labels) > 0 { - l = 0 - for _, e := range m.Labels { - l += protohelpers.SizeOfVarint(uint64(e)) - } - n += 1 + protohelpers.SizeOfVarint(uint64(l)) + l - } - n += len(m.unknownFields) - return n -} - -func (m *BlockList) UnmarshalVT(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: BlockList: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: BlockList: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Tenant", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Tenant = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) - } - m.Shard = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Shard |= uint32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Blocks", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Blocks = append(m.Blocks, string(dAtA[iNdEx:postIndex])) - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := protohelpers.Skip(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return protohelpers.ErrInvalidLength - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) - iNdEx += skippy + if m == nil { + return 0 + } + var l int + _ = l + if m.Tenant != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Tenant)) + } + if m.Name != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Name)) + } + if m.MinTime != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.MinTime)) + } + if m.MaxTime != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.MaxTime)) + } + if len(m.TableOfContents) > 0 { + l = 0 + for _, e := range m.TableOfContents { + l += protohelpers.SizeOfVarint(uint64(e)) + } + n += 1 + protohelpers.SizeOfVarint(uint64(l)) + l + } + if m.Size != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Size)) + } + if len(m.Labels) > 0 { + l = 0 + for _, e := range m.Labels { + l += protohelpers.SizeOfVarint(uint64(e)) } + n += 1 + protohelpers.SizeOfVarint(uint64(l)) + l } + n += len(m.unknownFields) + return n +} - if iNdEx > l { - return io.ErrUnexpectedEOF +func (m *BlockList) SizeVT() (n int) { + if m == nil { + return 0 } - return nil + var l int + _ = l + l = len(m.Tenant) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Shard != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Shard)) + } + if len(m.Blocks) > 0 { + for _, s := range m.Blocks { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n } + func (m *BlockMeta) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -1302,7 +1126,7 @@ func (m *Dataset) UnmarshalVT(dAtA []byte) error { break } } - case 7: + case 8: if wireType == 0 { var v int32 for shift := uint(0); ; shift += 7 { @@ -1319,7 +1143,7 @@ func (m *Dataset) UnmarshalVT(dAtA []byte) error { break } } - m.ProfileTypes = append(m.ProfileTypes, v) + m.Labels = append(m.Labels, v) } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { @@ -1354,8 +1178,8 @@ func (m *Dataset) UnmarshalVT(dAtA []byte) error { } } elementCount = count - if elementCount != 0 && len(m.ProfileTypes) == 0 { - m.ProfileTypes = make([]int32, 0, elementCount) + if elementCount != 0 && len(m.Labels) == 0 { + m.Labels = make([]int32, 0, elementCount) } for iNdEx < postIndex { var v int32 @@ -1373,87 +1197,145 @@ func (m *Dataset) UnmarshalVT(dAtA []byte) error { break } } - m.ProfileTypes = append(m.ProfileTypes, v) + m.Labels = append(m.Labels, v) } } else { - return fmt.Errorf("proto: wrong wireType = %d for field ProfileTypes", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType) } - case 8: - if wireType == 0 { - var v int32 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= int32(b&0x7F) << shift - if b < 0x80 { - break - } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *BlockList) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BlockList: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BlockList: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Tenant", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow } - m.Labels = append(m.Labels, v) - } else if wireType == 2 { - var packedLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - packedLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } + if iNdEx >= l { + return io.ErrUnexpectedEOF } - if packedLen < 0 { - return protohelpers.ErrInvalidLength + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break } - postIndex := iNdEx + packedLen - if postIndex < 0 { - return protohelpers.ErrInvalidLength + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Tenant = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Shard", wireType) + } + m.Shard = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow } - if postIndex > l { + if iNdEx >= l { return io.ErrUnexpectedEOF } - var elementCount int - var count int - for _, integer := range dAtA[iNdEx:postIndex] { - if integer < 128 { - count++ - } + b := dAtA[iNdEx] + iNdEx++ + m.Shard |= uint32(b&0x7F) << shift + if b < 0x80 { + break } - elementCount = count - if elementCount != 0 && len(m.Labels) == 0 { - m.Labels = make([]int32, 0, elementCount) + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Blocks", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow } - for iNdEx < postIndex { - var v int32 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= int32(b&0x7F) << shift - if b < 0x80 { - break - } - } - m.Labels = append(m.Labels, v) + if iNdEx >= l { + return io.ErrUnexpectedEOF } - } else { - return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType) + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF } + m.Blocks = append(m.Blocks, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/api/metastore/v1/metadata_query.proto b/api/metastore/v1/query.proto similarity index 50% rename from api/metastore/v1/metadata_query.proto rename to api/metastore/v1/query.proto index 3f0c3bde68..7f18eea7b6 100644 --- a/api/metastore/v1/metadata_query.proto +++ b/api/metastore/v1/query.proto @@ -3,9 +3,11 @@ syntax = "proto3"; package metastore.v1; import "metastore/v1/types.proto"; +import "types/v1/types.proto"; service MetadataQueryService { rpc QueryMetadata(QueryMetadataRequest) returns (QueryMetadataResponse) {} + rpc QueryMetadataLabels(QueryMetadataLabelsRequest) returns (QueryMetadataLabelsResponse) {} } message QueryMetadataRequest { @@ -18,3 +20,15 @@ message QueryMetadataRequest { message QueryMetadataResponse { repeated BlockMeta blocks = 1; } + +message QueryMetadataLabelsRequest { + repeated string tenant_id = 1; + int64 start_time = 2; + int64 end_time = 3; + string query = 4; + repeated string labels = 5; +} + +message QueryMetadataLabelsResponse { + repeated types.v1.Labels labels = 1; +} diff --git a/api/metastore/v1/types.proto b/api/metastore/v1/types.proto index 18ac943dba..f6150def4e 100644 --- a/api/metastore/v1/types.proto +++ b/api/metastore/v1/types.proto @@ -2,19 +2,23 @@ syntax = "proto3"; package metastore.v1; -message BlockList { - string tenant = 1; - uint32 shard = 2; - repeated string blocks = 3; -} - +// BlockMeta is a metadata entry that describes the block's contents. A block +// is a collection of datasets that share certain properties, such as shard ID, +// compaction level, tenant ID, time range, creation time, and more. +// +// The block content's format denotes the binary format of the datasets and the +// metadata entry (to address logical dependencies). Each dataset has its own +// table of contents that lists the sections within the dataset. Each dataset +// has its own set of attributes (labels) that describe its specific contents. message BlockMeta { uint32 format_version = 1; + // Block ID is a unique identifier for the block. // This is the only field that is not included into // the string table. string id = 2; - // Optional. Empty if compaction level is 0. + + // If empty, datasets belong to distinct tenants. int32 tenant = 3; uint32 shard = 4; uint32 compaction_level = 5; @@ -23,6 +27,7 @@ message BlockMeta { int32 created_by = 8; uint64 size = 9; repeated Dataset datasets = 10; + // String table contains strings of the block. // By convention, the first string is always an empty string. repeated string string_table = 11; @@ -43,12 +48,67 @@ message Dataset { // - 1: index.tsdb // - 2: symbols.symdb repeated uint64 table_of_contents = 5; - // Size of the section in bytes. + + // Size of the dataset in bytes. uint64 size = 6; + reserved 7; - // TODO: Delete. Use labels instead. - // Profile types present in the tenant service data. - repeated int32 profile_types = 7; // Length prefixed label key-value pairs. + // + // Multiple label sets can be associated with a dataset to denote relationships + // across multiple dimensions. For example, each dataset currently stores data + // for multiple profile types: + // - service_name=A, profile_type=cpu + // - service_name=A, profile_type=memory + // + // Labels are primarily used to filter datasets based on their attributes. + // For instance, labels can be used to select datasets containing a specific + // service. + // + // The set of attributes is extensible and can grow over time. For example, a + // namespace attribute could be added to datasets: + // - service_name=A, profile_type=cpu + // - service_name=A, profile_type=memory + // - service_name=B, namespace=N, profile_type=cpu + // - service_name=B, namespace=N, profile_type=memory + // - service_name=C, namespace=N, profile_type=cpu + // - service_name=C, namespace=N, profile_type=memory + // + // This organization enables querying datasets by namespace without accessing + // the block contents, which significantly improves performance. + // + // Metadata labels are not required to be included in the block's TSDB index + // and may be orthogonal to the data dimensions. Generally, attributes serve + // two primary purposes: + // - To create data scopes that span multiple service, reducing the need to + // scan the entire set of block satisfying the query expression, i.e., + // the time range and tenant ID. + // - To provide additional information about datasets without altering the + // storage schema or access methods. + // + // For example, this approach can support cost attribution or similar breakdown + // analyses. It can also handle data dependencies (e.g., links to external data) + // using labels. + // + // The cardinality of the labels is expected to remain relatively low (fewer + // than a million unique combinations globally). However, this depends on the + // metadata storage system. + // + // Metadata labels are represented as a slice of `int32` values that refer to + // strings in the metadata entry's string table. The slice is a sequence of + // length-prefixed key-value (KV) pairs: + // + // len(2) | k1 | v1 | k2 | v2 | len(3) | k1 | v3 | k2 | v4 | k3 | v5 + // + // The order of KV pairs is not defined. The format is optimized for indexing + // rather than querying, and it is not intended to be the most space-efficient + // representation. Since entries are supposed to be indexed, the redundancy of + // denormalized relationships is not a concern. repeated int32 labels = 8; } + +message BlockList { + string tenant = 1; + uint32 shard = 2; + repeated string blocks = 3; +} diff --git a/api/openapiv2/gen/phlare.swagger.json b/api/openapiv2/gen/phlare.swagger.json index dc2d8ae1dd..75f094d205 100644 --- a/api/openapiv2/gen/phlare.swagger.json +++ b/api/openapiv2/gen/phlare.swagger.json @@ -612,7 +612,7 @@ "tenant": { "type": "integer", "format": "int32", - "description": "Optional. Empty if compaction level is 0." + "description": "If empty, datasets belong to distinct tenants." }, "shard": { "type": "integer", @@ -652,7 +652,8 @@ }, "description": "String table contains strings of the block.\nBy convention, the first string is always an empty string." } - } + }, + "description": "BlockMeta is a metadata entry that describes the block's contents. A block\nis a collection of datasets that share certain properties, such as shard ID,\ncompaction level, tenant ID, time range, creation time, and more.\n\nThe block content's format denotes the binary format of the datasets and the\nmetadata entry (to address logical dependencies). Each dataset has its own\ntable of contents that lists the sections within the dataset. Each dataset\nhas its own set of attributes (labels) that describe its specific contents." }, "v1BlockMetadataResponse": { "type": "object", @@ -861,15 +862,7 @@ "size": { "type": "string", "format": "uint64", - "description": "Size of the section in bytes." - }, - "profileTypes": { - "type": "array", - "items": { - "type": "integer", - "format": "int32" - }, - "description": "TODO: Delete. Use labels instead.\nProfile types present in the tenant service data." + "description": "Size of the dataset in bytes." }, "labels": { "type": "array", @@ -877,7 +870,7 @@ "type": "integer", "format": "int32" }, - "description": "Length prefixed label key-value pairs." + "description": "Length prefixed label key-value pairs.\n\nMultiple label sets can be associated with a dataset to denote relationships\nacross multiple dimensions. For example, each dataset currently stores data\nfor multiple profile types:\n - service_name=A, profile_type=cpu\n - service_name=A, profile_type=memory\n\nLabels are primarily used to filter datasets based on their attributes.\nFor instance, labels can be used to select datasets containing a specific\nservice.\n\nThe set of attributes is extensible and can grow over time. For example, a\nnamespace attribute could be added to datasets:\n - service_name=A, profile_type=cpu\n - service_name=A, profile_type=memory\n - service_name=B, namespace=N, profile_type=cpu\n - service_name=B, namespace=N, profile_type=memory\n - service_name=C, namespace=N, profile_type=cpu\n - service_name=C, namespace=N, profile_type=memory\n\nThis organization enables querying datasets by namespace without accessing\nthe block contents, which significantly improves performance.\n\nMetadata labels are not required to be included in the block's TSDB index\nand may be orthogonal to the data dimensions. Generally, attributes serve\ntwo primary purposes:\n - To create data scopes that span multiple service, reducing the need to\n scan the entire set of block satisfying the query expression, i.e.,\n the time range and tenant ID.\n - To provide additional information about datasets without altering the\n storage schema or access methods.\n\nFor example, this approach can support cost attribution or similar breakdown\nanalyses. It can also handle data dependencies (e.g., links to external data)\nusing labels.\n\nThe cardinality of the labels is expected to remain relatively low (fewer\nthan a million unique combinations globally). However, this depends on the\nmetadata storage system.\n\nMetadata labels are represented as a slice of `int32` values that refer to\nstrings in the metadata entry's string table. The slice is a sequence of\nlength-prefixed key-value (KV) pairs:\n\nlen(2) | k1 | v1 | k2 | v2 | len(3) | k1 | v3 | k2 | v4 | k3 | v5\n\nThe order of KV pairs is not defined. The format is optimized for indexing\nrather than querying, and it is not intended to be the most space-efficient\nrepresentation. Since entries are supposed to be indexed, the redundancy of\ndenormalized relationships is not a concern." } } }, @@ -1651,6 +1644,18 @@ } } }, + "v1QueryMetadataLabelsResponse": { + "type": "object", + "properties": { + "labels": { + "type": "array", + "items": { + "type": "object", + "$ref": "#/definitions/v1Labels" + } + } + } + }, "v1QueryMetadataResponse": { "type": "object", "properties": { diff --git a/pkg/experiment/block/compaction.go b/pkg/experiment/block/compaction.go index a7b535f06e..d5d8d5d069 100644 --- a/pkg/experiment/block/compaction.go +++ b/pkg/experiment/block/compaction.go @@ -17,6 +17,7 @@ import ( "golang.org/x/sync/errgroup" metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" + "github.com/grafana/pyroscope/pkg/experiment/block/metadata" phlaremodel "github.com/grafana/pyroscope/pkg/model" "github.com/grafana/pyroscope/pkg/objstore" "github.com/grafana/pyroscope/pkg/phlaredb/block" @@ -119,7 +120,12 @@ func PlanCompaction(objects Objects) ([]*CompactionPlan, error) { for _, s := range obj.meta.Datasets { tm, ok := m[obj.meta.StringTable[s.Tenant]] if !ok { - tm = newBlockCompaction(g.ULID().String(), obj.meta.StringTable[s.Tenant], r.meta.Shard, level) + tm = newBlockCompaction( + g.ULID().String(), + obj.meta.StringTable[s.Tenant], + r.meta.Shard, + level, + ) m[obj.meta.StringTable[s.Tenant]] = tm } // Bind objects to datasets. @@ -148,7 +154,7 @@ type CompactionPlan struct { datasetMap map[int32]*datasetCompaction datasets []*datasetCompaction meta *metastorev1.BlockMeta - strings *MetadataStrings + strings *metadata.StringTable } func newBlockCompaction( @@ -160,7 +166,7 @@ func newBlockCompaction( p := &CompactionPlan{ tenant: tenant, datasetMap: make(map[int32]*datasetCompaction), - strings: NewMetadataStringTable(), + strings: metadata.NewStringTable(), } p.path = BuildObjectPath(tenant, shard, compactionLevel, id) p.meta = &metastorev1.BlockMeta{ @@ -215,9 +221,8 @@ type datasetCompaction struct { // Dataset name. name string parent *CompactionPlan - meta *metastorev1.Dataset - ptypes map[int32]struct{} + labels *metadata.LabelBuilder path string // Set at open. datasets []*Dataset @@ -235,9 +240,9 @@ type datasetCompaction struct { func (b *CompactionPlan) newDatasetCompaction(tenant, name int32) *datasetCompaction { return &datasetCompaction{ - name: b.strings.Strings[name], parent: b, - ptypes: make(map[int32]struct{}, 10), + name: b.strings.Strings[name], + labels: metadata.NewLabelBuilder(b.strings), meta: &metastorev1.Dataset{ Tenant: tenant, Name: name, @@ -247,7 +252,7 @@ func (b *CompactionPlan) newDatasetCompaction(tenant, name int32) *datasetCompac // Updated at writeTo. TableOfContents: nil, Size: 0, - ProfileTypes: nil, + Labels: nil, }, } } @@ -260,10 +265,7 @@ func (m *datasetCompaction) append(s *Dataset) { if s.meta.MaxTime > m.meta.MaxTime { m.meta.MaxTime = s.meta.MaxTime } - for _, pt := range s.meta.ProfileTypes { - ptn := m.parent.strings.Put(s.obj.meta.StringTable[pt]) - m.ptypes[ptn] = struct{}{} - } + m.labels.Put(s.meta.Labels, s.obj.meta.StringTable) } func (m *datasetCompaction) compact(ctx context.Context, w *Writer) (err error) { @@ -400,10 +402,7 @@ func (m *datasetCompaction) writeTo(w *Writer) (err error) { return err } m.meta.Size = w.Offset() - off - m.meta.ProfileTypes = make([]int32, 0, len(m.ptypes)) - for pt := range m.ptypes { - m.meta.ProfileTypes = append(m.meta.ProfileTypes, pt) - } + m.meta.Labels = m.labels.Build() return nil } diff --git a/pkg/experiment/block/metadata.go b/pkg/experiment/block/metadata/metadata.go similarity index 50% rename from pkg/experiment/block/metadata.go rename to pkg/experiment/block/metadata/metadata.go index 9693172604..59225e68f8 100644 --- a/pkg/experiment/block/metadata.go +++ b/pkg/experiment/block/metadata/metadata.go @@ -1,12 +1,9 @@ -package block +package metadata import ( - "io" - "math/rand" "sync" "time" - "github.com/cespare/xxhash/v2" "github.com/oklog/ulid" metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" @@ -24,44 +21,44 @@ func Timestamp(md *metastorev1.BlockMeta) time.Time { return time.UnixMilli(int64(ulid.MustParse(md.Id).Time())) } -func SanitizeMetadata(md *metastorev1.BlockMeta) error { +func Sanitize(md *metastorev1.BlockMeta) error { // TODO(kolesnikovae): Implement. _, err := ulid.Parse(md.Id) return err } var stringTablePool = sync.Pool{ - New: func() any { return NewMetadataStringTable() }, + New: func() any { return NewStringTable() }, } -type MetadataStrings struct { +type StringTable struct { Dict map[string]int32 Strings []string } -func NewMetadataStringTable() *MetadataStrings { +func NewStringTable() *StringTable { var empty string - return &MetadataStrings{ + return &StringTable{ Dict: map[string]int32{empty: 0}, Strings: []string{empty}, } } -func (t *MetadataStrings) IsEmpty() bool { +func (t *StringTable) IsEmpty() bool { if len(t.Strings) == 0 { return true } return len(t.Strings) == 1 && t.Strings[0] == "" } -func (t *MetadataStrings) Reset() { +func (t *StringTable) Reset() { clear(t.Dict) t.Dict[""] = 0 t.Strings[0] = "" t.Strings = t.Strings[:1] } -func (t *MetadataStrings) Put(s string) int32 { +func (t *StringTable) Put(s string) int32 { if i, ok := t.Dict[s]; ok { return i } @@ -71,15 +68,22 @@ func (t *MetadataStrings) Put(s string) int32 { return i } -func (t *MetadataStrings) Lookup(i int32) string { +func (t *StringTable) Lookup(i int32) string { if i < 0 || int(i) >= len(t.Strings) { return "" } return t.Strings[i] } +func (t *StringTable) LookupString(s string) int32 { + if i, ok := t.Dict[s]; ok { + return i + } + return -1 +} + // Import strings from the metadata entry and update the references. -func (t *MetadataStrings) Import(src *metastorev1.BlockMeta) { +func (t *StringTable) Import(src *metastorev1.BlockMeta) { if len(src.StringTable) < 2 { return } @@ -94,22 +98,32 @@ func (t *MetadataStrings) Import(src *metastorev1.BlockMeta) { for _, ds := range src.Datasets { ds.Tenant = lut[ds.Tenant] ds.Name = lut[ds.Name] - for i, p := range ds.ProfileTypes { - ds.ProfileTypes[i] = lut[p] + var skip int + for i, v := range ds.Labels { + if i == skip { + skip += int(v)*2 + 1 + continue + } + ds.Labels[i] = lut[v] } } } -func (t *MetadataStrings) Export(dst *metastorev1.BlockMeta) { - n := stringTablePool.Get().(*MetadataStrings) +func (t *StringTable) Export(dst *metastorev1.BlockMeta) { + n := stringTablePool.Get().(*StringTable) defer stringTablePool.Put(n) dst.Tenant = n.Put(t.Lookup(dst.Tenant)) dst.CreatedBy = n.Put(t.Lookup(dst.CreatedBy)) for _, ds := range dst.Datasets { ds.Tenant = n.Put(t.Lookup(ds.Tenant)) ds.Name = n.Put(t.Lookup(ds.Name)) - for i := range ds.ProfileTypes { - ds.ProfileTypes[i] = n.Put(t.Lookup(ds.ProfileTypes[i])) + var skip int + for i, v := range ds.Labels { + if i == skip { + skip += int(v)*2 + 1 + continue + } + ds.Labels[i] = n.Put(t.Lookup(ds.Labels[i])) } } dst.StringTable = make([]string, len(n.Strings)) @@ -117,45 +131,9 @@ func (t *MetadataStrings) Export(dst *metastorev1.BlockMeta) { n.Reset() } -func (t *MetadataStrings) Load(x iter.Iterator[string]) error { +func (t *StringTable) Load(x iter.Iterator[string]) error { for x.Next() { t.Put(x.At()) } return x.Err() } - -// ULIDGenerator generates deterministic ULIDs for blocks in an -// idempotent way: for the same set of objects, the generator -// will always produce the same set of ULIDs. -// -// We require block identifiers to be deterministic to ensure -// deduplication of the blocks. -type ULIDGenerator struct { - timestamp uint64 // Unix millis. - entropy io.Reader -} - -func NewULIDGenerator(objects Objects) *ULIDGenerator { - if len(objects) == 0 { - return &ULIDGenerator{ - timestamp: uint64(time.Now().UnixMilli()), - } - } - buf := make([]byte, 0, 1<<10) - for _, obj := range objects { - buf = append(buf, obj.meta.Id...) - } - seed := xxhash.Sum64(buf) - // Reference block. - // We're using its timestamp in all the generated ULIDs. - // Assuming that the first object is the oldest one. - r := objects[0] - return &ULIDGenerator{ - timestamp: ulid.MustParse(r.meta.Id).Time(), - entropy: rand.New(rand.NewSource(int64(seed))), - } -} - -func (g *ULIDGenerator) ULID() ulid.ULID { - return ulid.MustNew(g.timestamp, g.entropy) -} diff --git a/pkg/experiment/block/metadata/metadata_labels.go b/pkg/experiment/block/metadata/metadata_labels.go new file mode 100644 index 0000000000..df57b50245 --- /dev/null +++ b/pkg/experiment/block/metadata/metadata_labels.go @@ -0,0 +1,278 @@ +package metadata + +import ( + "slices" + + "github.com/prometheus/prometheus/model/labels" + "golang.org/x/exp/maps" + + typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" + "github.com/grafana/pyroscope/pkg/iter" + "github.com/grafana/pyroscope/pkg/model" +) + +// TODO(kolesnikovae): LabelBuilder pool. + +type LabelBuilder struct { + strings *StringTable + labels []int32 + constant []int32 + keys []int32 + seen map[string]struct{} +} + +func NewLabelBuilder(strings *StringTable) *LabelBuilder { + return &LabelBuilder{strings: strings} +} + +func (lb *LabelBuilder) WithConstantPairs(pairs ...string) *LabelBuilder { + if len(pairs)%2 == 1 { + return lb + } + lb.constant = slices.Grow(lb.constant[:0], len(pairs))[:len(pairs)] + for i := 0; i < len(pairs); i++ { + lb.constant[i] = lb.strings.Put(pairs[i]) + } + return lb +} + +func (lb *LabelBuilder) WithLabelNames(names ...string) *LabelBuilder { + lb.keys = slices.Grow(lb.keys[:0], len(names))[:len(names)] + for i, n := range names { + lb.keys[i] = lb.strings.Put(n) + } + return lb +} + +func (lb *LabelBuilder) CreateLabels(values ...string) bool { + if len(values) != len(lb.keys) { + return false + } + // We're going to add the length of pairs, the constant pairs, + // and then the variadic key-value pairs: p pairs total. + p := len(lb.constant)/2 + len(lb.keys) + n := 1 + p*2 // n elems total. + lb.labels = slices.Grow(lb.labels, n) + lb.labels = append(lb.labels, int32(p)) + lb.labels = append(lb.labels, lb.constant...) + for i := 0; i < len(values); i++ { + lb.labels = append(lb.labels, lb.keys[i], lb.strings.Put(values[i])) + } + return true +} + +func (lb *LabelBuilder) Put(x []int32, strings []string) { + if len(x) == 0 { + return + } + if lb.seen == nil { + lb.seen = make(map[string]struct{}) + } + var skip int + for i, v := range x { + if i == skip { + skip += int(v)*2 + 1 + continue + } + x[i] = lb.strings.Put(strings[v]) + } + lb.labels = slices.Grow(lb.labels, len(x)) + pairs := LabelPairs(x) + for pairs.Next() { + lb.putPairs(pairs.At()) + } +} + +func (lb *LabelBuilder) putPairs(p []int32) { + if len(p) == 0 { + return + } + // We only copy the labels if this is the first time we see it. + // The fact that we assume that the order of labels is the same + // across all datasets is a precondition, therefore, we can + // use pairs as a key. + k := string(p) + if _, ok := lb.seen[k]; ok { + return + } + lb.labels = append(lb.labels, int32(len(p)/2)) + lb.labels = append(lb.labels, p...) + lb.seen[k] = struct{}{} +} + +func (lb *LabelBuilder) Build() []int32 { + c := make([]int32, len(lb.labels)) + copy(c, lb.labels) + lb.labels = lb.labels[:0] + return c +} + +func LabelPairs(ls []int32) iter.Iterator[[]int32] { return &labelPairs{labels: ls} } + +type labelPairs struct { + labels []int32 + off int + len int +} + +func (p *labelPairs) Err() error { return nil } +func (p *labelPairs) Close() error { return nil } + +func (p *labelPairs) At() []int32 { return p.labels[p.off : p.off+p.len] } + +func (p *labelPairs) Next() bool { + if p.len > 0 { + p.off += p.len + } + if p.off >= len(p.labels) { + return false + } + p.len = int(p.labels[p.off]) * 2 + p.off++ + return p.off+p.len <= len(p.labels) +} + +type LabelMatcher struct { + eq []matcher + neq []matcher + keep []int32 + keepStr []string + + strings []string + checked map[string]bool + matched int32 + nomatch bool +} + +type matcher struct { + *labels.Matcher + name int32 +} + +func NewLabelMatcher(strings *StringTable, matchers []*labels.Matcher, keep ...string) *LabelMatcher { + lm := &LabelMatcher{ + eq: make([]matcher, 0, len(matchers)), + neq: make([]matcher, 0, len(matchers)), + keep: make([]int32, len(keep)), + keepStr: keep, + checked: make(map[string]bool), + strings: strings.Strings, + } + for _, m := range matchers { + n := strings.LookupString(m.Name) + if m.Type == labels.MatchEqual || m.Type == labels.MatchRegexp { + if n < 1 { + // No matches are possible if a label is not found + // in the string table or is an empty string (0). + lm.nomatch = true + return lm + } + lm.eq = append(lm.eq, matcher{Matcher: m, name: n}) + } else { + lm.neq = append(lm.neq, matcher{Matcher: m, name: n}) + } + } + // Find the indices of the labels to keep. + // If the label is not found or is an empty string, + // it will always be an empty string at the output. + for i, k := range keep { + lm.keep[i] = strings.LookupString(k) + } + return lm +} + +func (lm *LabelMatcher) IsValid() bool { return !lm.nomatch } + +func (lm *LabelMatcher) Matches(pairs []int32) bool { + k := string(pairs) + m, found := lm.checked[k] + if !found { + m = lm.checkMatches(pairs) + // Copy the key. + lm.checked[k] = m + if m { + lm.matched++ + } + } + return m +} + +func (lm *LabelMatcher) checkMatches(pairs []int32) bool { + if len(pairs)%2 == 1 { + // Invalid pairs. + return false + } + for _, m := range lm.eq { + var matches bool + for k := 0; k < len(pairs); k += 2 { + if pairs[k] != m.name { + continue + } + v := lm.strings[pairs[k+1]] + matches = m.Matches(v) + break + } + if !matches { + return false + } + } + // At this point, we know that all eq matchers have matched. + for _, m := range lm.neq { + for k := 0; k < len(pairs); k += 2 { + if pairs[k] != m.name { + continue + } + v := lm.strings[pairs[k+1]] + if !m.Matches(v) { + return false + } + break + } + } + return true +} + +func (lm *LabelMatcher) Matched() []model.Labels { + if len(lm.keep) == 0 || lm.nomatch || len(lm.checked) == 0 { + return nil + } + matched := make(map[string]model.Labels, lm.matched) + for k, match := range lm.checked { + if match { + values := lm.values(k) + if _, found := matched[values]; !found { + matched[values] = lm.labels([]int32(values)) + } + } + } + return maps.Values(matched) +} + +func (lm *LabelMatcher) values(pairs string) string { + p := []int32(pairs) + values := make([]int32, len(lm.keep)) + for i, n := range lm.keep { + if n < 1 { + // Skip invalid keep labels. + continue + } + for k := 0; k < len(pairs); k += 2 { + if p[k] == n { + values[i] = p[k+1] + break + } + } + } + return string(values) +} + +func (lm *LabelMatcher) labels(values []int32) model.Labels { + ls := make(model.Labels, len(values)) + for i, v := range values { + ls[i] = &typesv1.LabelPair{ + Name: lm.keepStr[i], + Value: lm.strings[v], + } + } + return ls +} diff --git a/pkg/experiment/block/metadata/metadata_labels_test.go b/pkg/experiment/block/metadata/metadata_labels_test.go new file mode 100644 index 0000000000..f2209424ca --- /dev/null +++ b/pkg/experiment/block/metadata/metadata_labels_test.go @@ -0,0 +1,172 @@ +package metadata + +import ( + "testing" + + "github.com/prometheus/prometheus/model/labels" + "github.com/stretchr/testify/assert" + + typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" + "github.com/grafana/pyroscope/pkg/model" +) + +func TestLabelBuilder_CreateLabels(t *testing.T) { + strings := NewStringTable() + b := NewLabelBuilder(strings). + WithConstantPairs("foo", "0"). + WithLabelNames("bar", "baz") + + b.CreateLabels("1", "2") + b.CreateLabels("1", "2") + b.CreateLabels("3", "4") + + assert.Equal(t, []int32{ + 3, 1, 2, 3, 5, 4, 6, // foo=0, bar=1, baz=2 + 3, 1, 2, 3, 5, 4, 6, // foo=0, bar=1, baz=2 + 3, 1, 2, 3, 7, 4, 8, // foo=0, bar=3, baz=4 + }, b.Build()) + + assert.EqualValues(t, 5, strings.LookupString("1")) + assert.EqualValues(t, 6, strings.LookupString("2")) + assert.EqualValues(t, 7, strings.LookupString("3")) + assert.EqualValues(t, 8, strings.LookupString("4")) +} + +func TestLabelBuilder_Reuse(t *testing.T) { + strings := NewStringTable() + b := NewLabelBuilder(strings). + WithConstantPairs("service_name", "service_a"). + WithLabelNames("__profile_type__") + + b.CreateLabels("cpu:a") + b.CreateLabels("cpu:b") + b.CreateLabels("memory") + assert.Equal(t, []string{ + "service_name=service_a;__profile_type__=cpu:a;", + "service_name=service_a;__profile_type__=cpu:b;", + "service_name=service_a;__profile_type__=memory;", + }, labelStrings(b.Build(), strings)) + + b.WithConstantPairs("service_name", "service_b") + assert.True(t, b.CreateLabels("cpu:a")) + assert.Equal(t, []string{ + "service_name=service_b;__profile_type__=cpu:a;", + }, labelStrings(b.Build(), strings)) + + b = b.WithLabelNames("another_label") + b.CreateLabels("another_value") + assert.Equal(t, []string{ + "service_name=service_b;another_label=another_value;", + }, labelStrings(b.Build(), strings)) +} + +func TestLabelBuilder_Put(t *testing.T) { + strings := NewStringTable() + b := NewLabelBuilder(strings) + + // a=b, a=b; a=b, a=b; + b.Put([]int32{2, 1, 2, 1, 2, 2, 1, 2, 1, 2}, []string{"", "a", "b"}) + b.Put([]int32{2, 1, 2, 1, 2, 2, 1, 2, 1, 2}, []string{"", "a", "b"}) + + // c=d, c=d; c=d, c=d; + b.Put([]int32{2, 1, 2, 1, 2, 2, 1, 2, 1, 2}, []string{"", "c", "d"}) + b.Put([]int32{2, 1, 2, 1, 2}, []string{"", "c", "d"}) + + assert.Equal(t, []int32{ + 2, 1, 2, 1, 2, + 2, 3, 4, 3, 4, + }, b.Build()) +} + +func labelStrings(v []int32, s *StringTable) []string { + var ls []string + pairs := LabelPairs(v) + for pairs.Next() { + p := pairs.At() + var l string + for len(p) > 0 { + l += s.Lookup(p[0]) + "=" + s.Lookup(p[1]) + ";" + p = p[2:] + } + ls = append(ls, l) + } + return ls +} + +func TestLabelMatcher_Matches(t *testing.T) { + strings := NewStringTable() + b := NewLabelBuilder(strings) + + b.WithConstantPairs("service_name", "service_a") + b.WithLabelNames("__profile_type__") + b.CreateLabels("cpu:a") + b.CreateLabels("cpu:b") + b.CreateLabels("memory") + setA := b.Build() + assert.Equal(t, []string{ + "service_name=service_a;__profile_type__=cpu:a;", + "service_name=service_a;__profile_type__=cpu:b;", + "service_name=service_a;__profile_type__=memory;", + }, labelStrings(setA, strings)) + + b.WithConstantPairs("service_name", "service_b") + b.CreateLabels("cpu:a") + b.CreateLabels("cpu:b") + setB := b.Build() + assert.Equal(t, []string{ + "service_name=service_b;__profile_type__=cpu:a;", + "service_name=service_b;__profile_type__=cpu:b;", + }, labelStrings(setB, strings)) + + m := NewLabelMatcher(strings, []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchEqual, "service_name", "service_a"), + labels.MustNewMatcher(labels.MatchEqual, "__profile_type__", "cpu:a")}, + "service_name", + "__profile_type__", + "none") + assert.True(t, m.IsValid()) + + expected := []bool{true, false, false, false, false} + matches := make([]bool, 0, len(expected)) + + pairs := LabelPairs(setA) + for pairs.Next() { + matches = append(matches, m.Matches(pairs.At())) + } + + pairs = LabelPairs(setB) + for pairs.Next() { + matches = append(matches, m.Matches(pairs.At())) + } + + assert.Equal(t, expected, matches) + assert.Equal(t, []model.Labels{{ + &typesv1.LabelPair{Name: "service_name", Value: "service_a"}, + &typesv1.LabelPair{Name: "__profile_type__", Value: "cpu:a"}, + &typesv1.LabelPair{Name: "none", Value: ""}, + }}, m.Matched()) +} + +func Benchmark_LabelMatcher_Matches(b *testing.B) { + strings := NewStringTable() + + lb := NewLabelBuilder(strings). + WithConstantPairs("service_name", "service_a"). + WithLabelNames("__profile_type__") + lb.CreateLabels("cpu") + ls := lb.Build() + + m := NewLabelMatcher(strings, + []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "service_name", "service_a")}, + "service_name", "__profile_type__") + + assert.True(b, m.IsValid()) + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + pairs := LabelPairs(ls) + for pairs.Next() { + m.Matches(pairs.At()) + } + } +} diff --git a/pkg/experiment/block/metadata/metadata_test.go b/pkg/experiment/block/metadata/metadata_test.go new file mode 100644 index 0000000000..e6b7727a74 --- /dev/null +++ b/pkg/experiment/block/metadata/metadata_test.go @@ -0,0 +1,218 @@ +package metadata + +import ( + "bytes" + "testing" + + "github.com/oklog/ulid" + "github.com/stretchr/testify/assert" + + metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" +) + +func TestMetadata_New(t *testing.T) { + blockID := ulid.MustNew(123, bytes.NewReader([]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})).String() + strings := NewStringTable() + md := &metastorev1.BlockMeta{ + FormatVersion: 0, + Id: blockID, + Tenant: 0, + Shard: 1, + CompactionLevel: 0, + MinTime: 123, + MaxTime: 456, + CreatedBy: strings.Put("ingester-a"), + Size: 567, + Datasets: nil, + StringTable: nil, + } + + b := NewLabelBuilder(strings) + for _, tenant := range []string{"tenant-a", "tenant-b"} { + for _, dataset := range []string{"service_a", "service_b"} { + ds := &metastorev1.Dataset{ + Tenant: strings.Put(tenant), + Name: strings.Put(dataset), + MinTime: 123, + MaxTime: 456, + TableOfContents: []uint64{1, 2, 3}, + Size: 567, + Labels: nil, + } + + b.WithConstantPairs("service_name", dataset).WithLabelNames("__profile_type__") + for _, n := range []string{"cpu", "memory"} { + b.CreateLabels(n) + } + ds.Labels = b.Build() + md.Datasets = append(md.Datasets, ds) + } + } + md.StringTable = strings.Strings + + expected := &metastorev1.BlockMeta{ + FormatVersion: 0, + Id: "000000003V000G40R40M30E209", + Tenant: 0, + Shard: 1, + CompactionLevel: 0, + MinTime: 123, + MaxTime: 456, + CreatedBy: 1, + Size: 567, + Datasets: []*metastorev1.Dataset{ + { + Tenant: 2, + Name: 3, + MinTime: 123, + MaxTime: 456, + TableOfContents: []uint64{1, 2, 3}, + Size: 567, + Labels: []int32{2, 4, 3, 5, 6, 2, 4, 3, 5, 7}, + }, + { + Tenant: 2, + Name: 8, + MinTime: 123, + MaxTime: 456, + TableOfContents: []uint64{1, 2, 3}, + Size: 567, + Labels: []int32{2, 4, 8, 5, 6, 2, 4, 8, 5, 7}, + }, + { + Tenant: 9, + Name: 3, + MinTime: 123, + MaxTime: 456, + TableOfContents: []uint64{1, 2, 3}, + Size: 567, + Labels: []int32{2, 4, 3, 5, 6, 2, 4, 3, 5, 7}, + }, + { + Tenant: 9, + Name: 8, + MinTime: 123, + MaxTime: 456, + TableOfContents: []uint64{1, 2, 3}, + Size: 567, + Labels: []int32{2, 4, 8, 5, 6, 2, 4, 8, 5, 7}, + }, + }, + StringTable: []string{ + "", "ingester-a", + "tenant-a", "service_a", "service_name", "__profile_type__", "cpu", "memory", + "service_b", "tenant-b", + }, + } + + assert.Equal(t, expected, md) +} + +func TestMetadataStrings_Import(t *testing.T) { + md1 := &metastorev1.BlockMeta{ + Id: "block_id", + Tenant: 0, + CreatedBy: 1, + Datasets: []*metastorev1.Dataset{ + {Tenant: 2, Name: 3, Labels: []int32{2, 10, 3, 11, 4, 2, 10, 3, 11, 5, 2, 10, 3, 11, 6}}, + {Tenant: 7, Name: 8, Labels: []int32{2, 10, 8, 11, 5, 2, 10, 8, 11, 6, 2, 10, 8, 11, 9}}, + }, + StringTable: []string{ + "", "ingester", + "tenant-a", "dataset-a", "1", "2", "3", + "tenant-b", "dataset-b", "4", + "service_name", "__profile_type__", + }, + } + + table := NewStringTable() + md1c := md1.CloneVT() + table.Import(md1c) + assert.Equal(t, md1, md1c) + assert.Equal(t, table.Strings, md1.StringTable) + + // Exactly the same metadata. + md2 := md1.CloneVT() + table.Import(md2) + assert.Len(t, md2.StringTable, 12) + assert.Len(t, table.Strings, 12) + assert.Equal(t, table.Strings, md2.StringTable) + + md3 := &metastorev1.BlockMeta{ + Id: "block_id_3", + Tenant: 0, + CreatedBy: 1, + Datasets: []*metastorev1.Dataset{ + {Tenant: 2, Name: 3, Labels: []int32{2, 10, 3, 11, 4, 2, 10, 3, 11, 5, 2, 10, 3, 11, 6}}, + {Tenant: 7, Name: 8, Labels: []int32{2, 10, 8, 11, 4, 2, 10, 8, 11, 9}}, + }, + StringTable: []string{ + "", "ingester", + "tenant-a", "dataset-a", "1", "2", "3", + "tenant-c", "dataset-c", "5", + "service_name", "__profile_type__", + }, + } + + table.Import(md3) + expected := &metastorev1.BlockMeta{ + Id: "block_id_3", + Tenant: 0, + CreatedBy: 1, + Datasets: []*metastorev1.Dataset{ + {Tenant: 2, Name: 3, Labels: []int32{2, 10, 3, 11, 4, 2, 10, 3, 11, 5, 2, 10, 3, 11, 6}}, + {Tenant: 12, Name: 13, Labels: []int32{2, 10, 13, 11, 4, 2, 10, 13, 11, 14}}, + }, + StringTable: []string{ + "", "ingester", + "tenant-a", "dataset-a", "1", "2", "3", + "tenant-c", "dataset-c", "5", + "service_name", "__profile_type__", + }, + } + + assert.Equal(t, expected, md3) + assert.Len(t, table.Strings, 15) +} + +func TestMetadataStrings_Export(t *testing.T) { + table := NewStringTable() + for _, s := range []string{ + "", "x1", "x2", "x3", "x4", "x5", + "ingester", + "tenant-a", "dataset-a", "1", "2", "3", + "tenant-b", "dataset-b", "4", + "service_name", "__profile_type__", + } { + table.Put(s) + } + + md := &metastorev1.BlockMeta{ + Id: "1", + Tenant: 0, + CreatedBy: 6, + Datasets: []*metastorev1.Dataset{ + {Tenant: 7, Name: 8, Labels: []int32{2, 15, 8, 16, 9, 2, 15, 8, 16, 10, 2, 15, 8, 16, 11}}, + {Tenant: 12, Name: 13, Labels: []int32{2, 15, 13, 16, 10, 2, 15, 13, 16, 11, 2, 15, 13, 16, 14}}, + }, + } + + table.Export(md) + + expected := &metastorev1.BlockMeta{ + Id: "1", + Tenant: 0, + CreatedBy: 1, + Datasets: []*metastorev1.Dataset{ + {Tenant: 2, Name: 3, Labels: []int32{2, 4, 3, 5, 6, 2, 4, 3, 5, 7, 2, 4, 3, 5, 8}}, + {Tenant: 9, Name: 10, Labels: []int32{2, 4, 10, 5, 7, 2, 4, 10, 5, 8, 2, 4, 10, 5, 11}}, + }, + StringTable: []string{ + "", "ingester", + "tenant-a", "dataset-a", "service_name", "__profile_type__", "1", "2", "3", + "tenant-b", "dataset-b", "4", + }, + } + + assert.Equal(t, expected, md) +} diff --git a/pkg/experiment/block/metadata_test.go b/pkg/experiment/block/metadata_test.go deleted file mode 100644 index 0575cd80a2..0000000000 --- a/pkg/experiment/block/metadata_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package block - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" -) - -func TestMetadataStrings_Import(t *testing.T) { - md1 := &metastorev1.BlockMeta{ - Id: "block_id", - Tenant: 0, - CreatedBy: 1, - Datasets: []*metastorev1.Dataset{ - {Tenant: 2, Name: 3, ProfileTypes: []int32{4, 5, 6}}, - {Tenant: 7, Name: 8, ProfileTypes: []int32{5, 6, 9}}, - }, - StringTable: []string{ - "", "ingester", - "tenant-a", "dataset-a", "1", "2", "3", - "tenant-b", "dataset-b", "4", - }, - } - - table := NewMetadataStringTable() - md1c := md1.CloneVT() - table.Import(md1c) - assert.Equal(t, md1, md1c) - assert.Equal(t, table.Strings, md1.StringTable) - - // Exactly the same metadata. - md2 := md1.CloneVT() - table.Import(md2) - assert.Len(t, md2.StringTable, 10) - assert.Len(t, table.Strings, 10) - assert.Equal(t, table.Strings, md2.StringTable) - - md3 := &metastorev1.BlockMeta{ - Id: "block_id_3", - Tenant: 0, - CreatedBy: 1, - Datasets: []*metastorev1.Dataset{ - {Tenant: 2, Name: 3, ProfileTypes: []int32{4, 5, 6}}, - {Tenant: 7, Name: 8, ProfileTypes: []int32{4, 9}}, - }, - StringTable: []string{ - "", "ingester", - "tenant-a", "dataset-a", "1", "2", "3", - "tenant-c", "dataset-c", "5", - }, - } - - table.Import(md3) - expected := &metastorev1.BlockMeta{ - Id: "block_id_3", - Tenant: 0, - CreatedBy: 1, - Datasets: []*metastorev1.Dataset{ - {Tenant: 2, Name: 3, ProfileTypes: []int32{4, 5, 6}}, - {Tenant: 10, Name: 11, ProfileTypes: []int32{4, 12}}, - }, - StringTable: []string{ - "", "ingester", - "tenant-a", "dataset-a", "1", "2", "3", - "tenant-c", "dataset-c", "5", - }, - } - - assert.Equal(t, expected, md3) - assert.Len(t, table.Strings, 13) -} - -func TestMetadataStrings_Export(t *testing.T) { - table := NewMetadataStringTable() - for _, s := range []string{ - "", "x1", "x2", "x3", "x4", "x5", - "ingester", - "tenant-a", "dataset-a", "1", "2", "3", - "tenant-b", "dataset-b", "4", - } { - table.Put(s) - } - - md := &metastorev1.BlockMeta{ - Id: "1", - Tenant: 0, - CreatedBy: 6, - Datasets: []*metastorev1.Dataset{ - {Tenant: 7, Name: 8, ProfileTypes: []int32{9, 10, 11}}, - {Tenant: 12, Name: 13, ProfileTypes: []int32{10, 11, 14}}, - }, - } - - table.Export(md) - - expected := &metastorev1.BlockMeta{ - Id: "1", - Tenant: 0, - CreatedBy: 1, - Datasets: []*metastorev1.Dataset{ - {Tenant: 2, Name: 3, ProfileTypes: []int32{4, 5, 6}}, - {Tenant: 7, Name: 8, ProfileTypes: []int32{5, 6, 9}}, - }, - StringTable: []string{ - "", "ingester", - "tenant-a", "dataset-a", "1", "2", "3", - "tenant-b", "dataset-b", "4", - }, - } - - assert.Equal(t, expected, md) -} diff --git a/pkg/experiment/block/object.go b/pkg/experiment/block/object.go index b900f667b3..e4c1eca270 100644 --- a/pkg/experiment/block/object.go +++ b/pkg/experiment/block/object.go @@ -11,6 +11,7 @@ import ( "golang.org/x/sync/errgroup" metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" + "github.com/grafana/pyroscope/pkg/experiment/block/metadata" "github.com/grafana/pyroscope/pkg/objstore" "github.com/grafana/pyroscope/pkg/util" "github.com/grafana/pyroscope/pkg/util/bufferpool" @@ -109,7 +110,7 @@ func NewObject(storage objstore.Bucket, md *metastorev1.BlockMeta, opts ...Objec } func ObjectPath(md *metastorev1.BlockMeta) string { - return BuildObjectPath(Tenant(md), md.Shard, md.CompactionLevel, md.Id) + return BuildObjectPath(metadata.Tenant(md), md.Shard, md.CompactionLevel, md.Id) } func BuildObjectPath(tenant string, shard uint32, level uint32, block string) string { @@ -136,7 +137,7 @@ func MetadataDLQObjectPath(md *metastorev1.BlockMeta) string { var b strings.Builder tenantDirName := DirNameAnonTenant if md.CompactionLevel > 0 { - tenantDirName = Tenant(md) + tenantDirName = metadata.Tenant(md) } b.WriteString(DirNameDLQ) b.WriteByte('/') diff --git a/pkg/experiment/block/ulid_generator.go b/pkg/experiment/block/ulid_generator.go new file mode 100644 index 0000000000..b985df2a2e --- /dev/null +++ b/pkg/experiment/block/ulid_generator.go @@ -0,0 +1,46 @@ +package block + +import ( + "io" + "math/rand" + "time" + + "github.com/cespare/xxhash/v2" + "github.com/oklog/ulid" +) + +// ULIDGenerator generates deterministic ULIDs for blocks in an +// idempotent way: for the same set of objects, the generator +// will always produce the same set of ULIDs. +// +// We require block identifiers to be deterministic to ensure +// deduplication of the blocks. +type ULIDGenerator struct { + timestamp uint64 // Unix millis. + entropy io.Reader +} + +func NewULIDGenerator(objects Objects) *ULIDGenerator { + if len(objects) == 0 { + return &ULIDGenerator{ + timestamp: uint64(time.Now().UnixMilli()), + } + } + buf := make([]byte, 0, 1<<10) + for _, obj := range objects { + buf = append(buf, obj.meta.Id...) + } + seed := xxhash.Sum64(buf) + // Reference block. + // We're using its timestamp in all the generated ULIDs. + // Assuming that the first object is the oldest one. + r := objects[0] + return &ULIDGenerator{ + timestamp: ulid.MustParse(r.Meta().Id).Time(), + entropy: rand.New(rand.NewSource(int64(seed))), + } +} + +func (g *ULIDGenerator) ULID() ulid.ULID { + return ulid.MustNew(g.timestamp, g.entropy) +} diff --git a/pkg/experiment/compactor/compaction_worker.go b/pkg/experiment/compactor/compaction_worker.go index c03bd5bcb9..7579246791 100644 --- a/pkg/experiment/compactor/compaction_worker.go +++ b/pkg/experiment/compactor/compaction_worker.go @@ -24,6 +24,7 @@ import ( metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" "github.com/grafana/pyroscope/pkg/experiment/block" + "github.com/grafana/pyroscope/pkg/experiment/block/metadata" "github.com/grafana/pyroscope/pkg/objstore" "github.com/grafana/pyroscope/pkg/util" ) @@ -380,7 +381,7 @@ func (w *Worker) runCompaction(job *compactionJob) { level.Info(logger).Log( "msg", "new compacted block", "block_id", c.Id, - "block_tenant", block.Tenant(c), + "block_tenant", metadata.Tenant(c), "block_shard", c.Shard, "block_compaction_level", c.CompactionLevel, "block_min_time", c.MinTime, @@ -400,7 +401,7 @@ func (w *Worker) runCompaction(job *compactionJob) { }, } - firstBlock := block.Timestamp(job.blocks[0]) + firstBlock := metadata.Timestamp(job.blocks[0]) w.metrics.timeToCompaction.WithLabelValues(labels...).Observe(time.Since(firstBlock).Seconds()) case errors.Is(err, context.Canceled): diff --git a/pkg/experiment/ingester/segment.go b/pkg/experiment/ingester/segment.go index 414983c29e..70e8dc27a6 100644 --- a/pkg/experiment/ingester/segment.go +++ b/pkg/experiment/ingester/segment.go @@ -22,6 +22,7 @@ import ( metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" "github.com/grafana/pyroscope/pkg/experiment/block" + "github.com/grafana/pyroscope/pkg/experiment/block/metadata" "github.com/grafana/pyroscope/pkg/experiment/ingester/memdb" "github.com/grafana/pyroscope/pkg/model" pprofsplit "github.com/grafana/pyroscope/pkg/model/pprof_split" @@ -239,7 +240,7 @@ func (s *segment) flushBlock(heads []flushedServiceHead) ([]byte, *metastorev1.B t1 := time.Now() hostname, _ := os.Hostname() - stringTable := block.NewMetadataStringTable() + stringTable := metadata.NewStringTable() meta := &metastorev1.BlockMeta{ FormatVersion: 1, Id: s.ulid.String(), @@ -280,7 +281,7 @@ func (s *segment) flushBlock(heads []flushedServiceHead) ([]byte, *metastorev1.B return blockFile.Bytes(), meta, nil } -func concatSegmentHead(e flushedServiceHead, w *writerOffset, s *block.MetadataStrings) (*metastorev1.Dataset, error) { +func concatSegmentHead(e flushedServiceHead, w *writerOffset, s *metadata.StringTable) (*metastorev1.Dataset, error) { tenantServiceOffset := w.offset ptypes := e.head.Meta.ProfileTypeNames @@ -298,7 +299,7 @@ func concatSegmentHead(e flushedServiceHead, w *writerOffset, s *block.MetadataS tenantServiceSize := w.offset - tenantServiceOffset - svc := &metastorev1.Dataset{ + ds := &metastorev1.Dataset{ Tenant: s.Put(e.key.tenant), Name: s.Put(e.key.service), MinTime: e.head.Meta.MinTimeNanos / 1e6, @@ -308,13 +309,20 @@ func concatSegmentHead(e flushedServiceHead, w *writerOffset, s *block.MetadataS // - 1: index.tsdb // - 2: symbols.symdb TableOfContents: offsets, - ProfileTypes: make([]int32, len(ptypes)), + Labels: nil, } - for i, p := range ptypes { - svc.ProfileTypes[i] = s.Put(p) + + lb := metadata.NewLabelBuilder(s). + WithConstantPairs(model.LabelNameServiceName, e.key.service). + WithLabelNames(model.LabelNameProfileType) + + for _, profileType := range ptypes { + lb.CreateLabels(profileType) } - return svc, nil + ds.Labels = lb.Build() + + return ds, nil } func (s *segment) flushHeads(ctx context.Context) (moved []flushedServiceHead) { diff --git a/pkg/experiment/metastore/client/methods.go b/pkg/experiment/metastore/client/methods.go index e6d7bbf027..bfda970906 100644 --- a/pkg/experiment/metastore/client/methods.go +++ b/pkg/experiment/metastore/client/methods.go @@ -102,6 +102,12 @@ func (c *Client) QueryMetadata(ctx context.Context, in *metastorev1.QueryMetadat }) } +func (c *Client) QueryMetadataLabels(ctx context.Context, in *metastorev1.QueryMetadataLabelsRequest, opts ...grpc.CallOption) (*metastorev1.QueryMetadataLabelsResponse, error) { + return invoke(ctx, c, func(ctx context.Context, instance instance) (*metastorev1.QueryMetadataLabelsResponse, error) { + return instance.QueryMetadataLabels(ctx, in, opts...) + }) +} + func (c *Client) PollCompactionJobs(ctx context.Context, in *metastorev1.PollCompactionJobsRequest, opts ...grpc.CallOption) (*metastorev1.PollCompactionJobsResponse, error) { return invoke(ctx, c, func(ctx context.Context, instance instance) (*metastorev1.PollCompactionJobsResponse, error) { return instance.PollCompactionJobs(ctx, in, opts...) diff --git a/pkg/experiment/metastore/client/server_mock_test.go b/pkg/experiment/metastore/client/server_mock_test.go index 4f39e71f3c..bc6b6484f3 100644 --- a/pkg/experiment/metastore/client/server_mock_test.go +++ b/pkg/experiment/metastore/client/server_mock_test.go @@ -65,6 +65,10 @@ func (m *mockServer) QueryMetadata(ctx context.Context, request *metastorev1.Que return m.metadata.QueryMetadata(ctx, request) } +func (m *mockServer) QueryMetadataLabels(ctx context.Context, request *metastorev1.QueryMetadataLabelsRequest) (*metastorev1.QueryMetadataLabelsResponse, error) { + return m.metadata.QueryMetadataLabels(ctx, request) +} + func (m *mockServer) ReadIndex(ctx context.Context, request *raftnodepb.ReadIndexRequest) (*raftnodepb.ReadIndexResponse, error) { return m.raftNode.ReadIndex(ctx, request) } diff --git a/pkg/experiment/metastore/compaction/compaction.go b/pkg/experiment/metastore/compaction/compaction.go index 71def608d9..e79eb93118 100644 --- a/pkg/experiment/metastore/compaction/compaction.go +++ b/pkg/experiment/metastore/compaction/compaction.go @@ -6,7 +6,7 @@ import ( metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1/raft_log" - "github.com/grafana/pyroscope/pkg/experiment/block" + "github.com/grafana/pyroscope/pkg/experiment/block/metadata" ) type Compactor interface { @@ -75,7 +75,7 @@ func NewBlockEntry(cmd *raft.Log, md *metastorev1.BlockMeta) BlockEntry { Index: cmd.Index, AppendedAt: cmd.AppendedAt.UnixNano(), ID: md.Id, - Tenant: block.Tenant(md), + Tenant: metadata.Tenant(md), Shard: md.Shard, Level: md.CompactionLevel, } diff --git a/pkg/experiment/metastore/index/index.go b/pkg/experiment/metastore/index/index.go index c2a9dc2c12..5b0f4d4183 100644 --- a/pkg/experiment/metastore/index/index.go +++ b/pkg/experiment/metastore/index/index.go @@ -6,19 +6,21 @@ import ( "fmt" "math" "slices" - "strings" "sync" "time" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/oklog/ulid" + "github.com/prometheus/prometheus/model/labels" "go.etcd.io/bbolt" metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" - "github.com/grafana/pyroscope/pkg/experiment/block" + typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" + "github.com/grafana/pyroscope/pkg/experiment/block/metadata" "github.com/grafana/pyroscope/pkg/experiment/metastore/index/store" "github.com/grafana/pyroscope/pkg/iter" + phlaremodel "github.com/grafana/pyroscope/pkg/model" ) const ( @@ -198,7 +200,7 @@ func (i *Index) InsertBlock(tx *bbolt.Tx, b *metastorev1.BlockMeta) error { // concurrent access. The method will create a new partition if needed. func (i *Index) insertBlock(tx *bbolt.Tx, b *metastorev1.BlockMeta) error { p := i.getOrCreatePartition(b) - shard, err := i.getOrCreateTenantShard(tx, p, block.Tenant(b), b.Shard) + shard, err := i.getOrCreateTenantShard(tx, p, metadata.Tenant(b), b.Shard) if err != nil { return err } @@ -244,7 +246,7 @@ func (i *Index) getOrCreateTenantShard(tx *bbolt.Tx, p *store.Partition, tenant Partition: p.Key, Tenant: tenant, Shard: shard, - StringTable: block.NewMetadataStringTable(), + StringTable: metadata.NewStringTable(), }) partition.shards[shard] = s // This is the only way we "remember" the tenant shard. @@ -424,48 +426,48 @@ func (i *Index) GetTenantStats(tenant string) *metastorev1.TenantStats { stats.NewestProfileTime = newest } } + if !stats.DataIngested { + return new(metastorev1.TenantStats) + } return stats } +// TODO(kolesnikovae): We query meta with the mutex held, which +// will cause contention and latency issues. Fix it once we make +// locks more granular (partition-tenant-shard level). + func (i *Index) QueryMetadata(tx *bbolt.Tx, query MetadataQuery) iter.Iterator[*metastorev1.BlockMeta] { q, err := newMetadataQuery(i, query) if err != nil { return iter.NewErrIterator[*metastorev1.BlockMeta](err) } + // Currently, we only inspect the service name label. + // We could extend this to match any labels of a dataset. + q.matchers = slices.DeleteFunc(q.matchers, func(m *labels.Matcher) bool { + return m.Name != phlaremodel.LabelNameServiceName + }) i.mu.Lock() - defer i.mu.Unlock() - // TODO(kolesnikovae): We collect blocks with the mutex held, which - // will cause contention and latency issues. Fix it once we make - // locks more granular (partition-tenant-shard level). - metas, err := iter.Slice[*metastorev1.BlockMeta](q.iterator(tx)) + metas, err := iter.Slice[*metastorev1.BlockMeta](newBlockMetadataIterator(tx, q)) + i.mu.Unlock() if err != nil { return iter.NewErrIterator[*metastorev1.BlockMeta](err) } return iter.NewSliceIterator(metas) } -func (i *Index) shardIterator(tx *bbolt.Tx, startTime, endTime time.Time, tenants ...string) iter.Iterator[*indexShard] { - startTime = startTime.Add(-i.config.QueryLookaroundPeriod) - endTime = endTime.Add(i.config.QueryLookaroundPeriod) - si := shardIterator{ - tx: tx, - partitions: make([]*store.Partition, 0, len(i.partitions)), - tenants: tenants, - index: i, +func (i *Index) QueryMetadataLabels(tx *bbolt.Tx, query MetadataLabelQuery) ([]*typesv1.Labels, error) { + q, err := newMetadataQuery(i, query.MetadataQuery, query.Labels...) + if err != nil { + return nil, err } - for _, p := range i.partitions { - if !p.Overlaps(startTime, endTime) { - continue - } - for _, t := range si.tenants { - if p.HasTenant(t) { - si.partitions = append(si.partitions, p) - break - } - } + i.mu.Lock() + r, err := newMetadataLabelQuerier(tx, q).queryLabels() + i.mu.Unlock() + if err != nil { + return nil, err } - return &si + return r.Labels(), nil } func newIndexPartition(p *store.Partition, tenant string) *indexPartition { @@ -518,62 +520,3 @@ func (s *indexShard) getBlock(blockID string) *metastorev1.BlockMeta { s.TenantShard.StringTable.Export(mdCopy) return mdCopy } - -type shardIterator struct { - tx *bbolt.Tx - index *Index - tenants []string - partitions []*store.Partition - shards []*indexShard - cur int - err error -} - -func (si *shardIterator) Close() error { return nil } - -func (si *shardIterator) Err() error { return si.err } - -func (si *shardIterator) At() *indexShard { return si.shards[si.cur] } - -func (si *shardIterator) Next() bool { - if n := si.cur + 1; n < len(si.shards) { - si.cur = n - return true - } - si.cur = 0 - si.shards = si.shards[:0] - for len(si.shards) == 0 && len(si.partitions) > 0 { - si.loadShards(si.partitions[0]) - si.partitions = si.partitions[1:] - } - return si.cur < len(si.shards) -} - -func (si *shardIterator) loadShards(p *store.Partition) { - for _, t := range si.tenants { - shards := p.TenantShards[t] - if shards == nil { - continue - } - for s := range shards { - shard, err := si.index.getOrLoadTenantShard(si.tx, p, t, s) - if err != nil { - si.err = err - return - } - if shard != nil { - si.shards = append(si.shards, shard) - } - } - } - slices.SortFunc(si.shards, compareShards) - si.shards = slices.Compact(si.shards) -} - -func compareShards(a, b *indexShard) int { - cmp := strings.Compare(a.Tenant, b.Tenant) - if cmp == 0 { - return int(a.Shard) - int(b.Shard) - } - return cmp -} diff --git a/pkg/experiment/metastore/index/query.go b/pkg/experiment/metastore/index/query.go index eaa6be40e3..1d553eb23c 100644 --- a/pkg/experiment/metastore/index/query.go +++ b/pkg/experiment/metastore/index/query.go @@ -12,7 +12,8 @@ import ( "go.etcd.io/bbolt" metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" - "github.com/grafana/pyroscope/pkg/experiment/block" + "github.com/grafana/pyroscope/pkg/experiment/block/metadata" + "github.com/grafana/pyroscope/pkg/experiment/metastore/index/store" "github.com/grafana/pyroscope/pkg/iter" "github.com/grafana/pyroscope/pkg/model" ) @@ -33,6 +34,11 @@ type MetadataQuery struct { Tenant []string } +type MetadataLabelQuery struct { + MetadataQuery + Labels []string +} + func (q *MetadataQuery) String() string { return fmt.Sprintf("start: %v, end: %v, tenants: %v, expr: %v", q.StartTime, @@ -45,79 +51,69 @@ type metadataQuery struct { startTime time.Time endTime time.Time tenants []string + tenantMap map[string]struct{} + matchers []*labels.Matcher + labels []string index *Index - // Currently, we only check if the dataset name matches - // the service name label. We could extend this to match - // any labels of a dataset. - matcher *labels.Matcher } -func newMetadataQuery(index *Index, query MetadataQuery) (*metadataQuery, error) { +func newMetadataQuery(index *Index, query MetadataQuery, labels ...string) (*metadataQuery, error) { if len(query.Tenant) == 0 { return nil, &InvalidQueryError{Query: query, Err: fmt.Errorf("tenant_id is required")} } - selectors, err := parser.ParseMetricSelector(query.Expr) + matchers, err := parser.ParseMetricSelector(query.Expr) if err != nil { - return nil, &InvalidQueryError{Query: query, Err: fmt.Errorf("failed to parse label selectors: %w", err)} + return nil, &InvalidQueryError{Query: query, Err: fmt.Errorf("failed to parse label matcher: %w", err)} } - // TODO: Validate the time range. q := &metadataQuery{ startTime: query.StartTime, endTime: query.EndTime, index: index, + matchers: matchers, + labels: labels, } - q.buildTenantList(query.Tenant) - for _, m := range selectors { - if m.Name == model.LabelNameServiceName && q.matcher == nil { - q.matcher = m - break - } - } - // We could also validate that the service has the profile type - // queried, but that's not really necessary: querying an irrelevant - // profile type is rather a rare/invalid case. + q.buildTenantMap(query.Tenant) return q, nil } -func (q *metadataQuery) buildTenantList(tenants []string) { - m := make(map[string]struct{}, len(tenants)+1) +func (q *metadataQuery) buildTenantMap(tenants []string) { + q.tenantMap = make(map[string]struct{}, len(tenants)+1) for _, t := range tenants { - m[t] = struct{}{} + q.tenantMap[t] = struct{}{} } - m[""] = struct{}{} - q.tenants = make([]string, 0, len(m)) - for t := range m { + // Always query the anonymous blocks: tenant datasets will be filtered out later. + q.tenantMap[""] = struct{}{} + q.tenants = make([]string, 0, len(q.tenantMap)) + for t := range q.tenantMap { q.tenants = append(q.tenants, t) } sort.Strings(q.tenants) } -func (q *metadataQuery) iterator(tx *bbolt.Tx) *iterator { - shards := q.index.shardIterator(tx, q.startTime, q.endTime, q.tenants...) - si := iterator{ - query: q, - tenants: make(map[string]struct{}, len(q.tenants)), - shards: shards, - } - for _, t := range q.tenants { - si.tenants[t] = struct{}{} +func (q *metadataQuery) overlaps(start, end time.Time) bool { + return start.Before(q.endTime) && !end.Before(q.startTime) +} + +func newBlockMetadataIterator(tx *bbolt.Tx, q *metadataQuery) *blockMetadataIterator { + return &blockMetadataIterator{ + query: q, + shards: newShardIterator(tx, q.index, q.startTime, q.endTime, q.tenants...), + metas: make([]*metastorev1.BlockMeta, 0, 256), } - return &si } -type iterator struct { - query *metadataQuery - tenants map[string]struct{} - shards iter.Iterator[*indexShard] - metas []*metastorev1.BlockMeta - cur int +type blockMetadataIterator struct { + query *metadataQuery + shards iter.Iterator[*indexShard] + metas []*metastorev1.BlockMeta + cur int } -func (mi *iterator) Close() error { return mi.shards.Close() } -func (mi *iterator) Err() error { return mi.shards.Err() } -func (mi *iterator) At() *metastorev1.BlockMeta { return mi.metas[mi.cur] } +func (mi *blockMetadataIterator) Close() error { return mi.shards.Close() } +func (mi *blockMetadataIterator) Err() error { return mi.shards.Err() } +func (mi *blockMetadataIterator) At() *metastorev1.BlockMeta { return mi.metas[mi.cur] } -func (mi *iterator) Next() bool { +func (mi *blockMetadataIterator) Next() bool { if n := mi.cur + 1; n < len(mi.metas) { mi.cur = n return true @@ -125,17 +121,21 @@ func (mi *iterator) Next() bool { mi.cur = 0 mi.metas = mi.metas[:0] for mi.shards.Next() { - if mi.copyMatched(mi.shards.At()) { + if mi.collectBlockMetadata(mi.shards.At()) { break } } return len(mi.metas) > 0 } -func (mi *iterator) copyMatched(shard *indexShard) bool { +func (mi *blockMetadataIterator) collectBlockMetadata(shard *indexShard) bool { + matcher := metadata.NewLabelMatcher(shard.StringTable, mi.query.matchers) + if !matcher.IsValid() { + return false + } for _, md := range shard.blocks { - if match := mi.metadataMatch(shard.StringTable, md); match != nil { - mi.metas = append(mi.metas, match) + if m := blockMetadataMatches(mi.query, shard.StringTable, matcher, md); m != nil { + mi.metas = append(mi.metas, m) } } slices.SortFunc(mi.metas, func(a, b *metastorev1.BlockMeta) int { @@ -144,18 +144,24 @@ func (mi *iterator) copyMatched(shard *indexShard) bool { return len(mi.metas) > 0 } -func (mi *iterator) metadataMatch(s *block.MetadataStrings, md *metastorev1.BlockMeta) *metastorev1.BlockMeta { - if !mi.query.overlaps(time.UnixMilli(md.MinTime), time.UnixMilli(md.MaxTime)) { +func blockMetadataMatches( + q *metadataQuery, + s *metadata.StringTable, + m *metadata.LabelMatcher, + md *metastorev1.BlockMeta, +) *metastorev1.BlockMeta { + if !q.overlaps(time.UnixMilli(md.MinTime), time.UnixMilli(md.MaxTime)) { return nil } var mdCopy *metastorev1.BlockMeta datasets := md.Datasets for _, ds := range datasets { - if mi.datasetMatches(s, ds) { + if datasetMatches(q, s, m, ds) { if mdCopy == nil { - mdCopy = cloneMetadataForQuery(md) + mdCopy = cloneBlockMetadataForQuery(md) } - mdCopy.Datasets = append(mdCopy.Datasets, ds.CloneVT()) + dsCopy := cloneDatasetMetadataForQuery(ds) + mdCopy.Datasets = append(mdCopy.Datasets, dsCopy) } } if mdCopy != nil { @@ -165,29 +171,176 @@ func (mi *iterator) metadataMatch(s *block.MetadataStrings, md *metastorev1.Bloc return mdCopy } -func (mi *iterator) datasetMatches(s *block.MetadataStrings, ds *metastorev1.Dataset) bool { - if _, ok := mi.tenants[s.Lookup(ds.Tenant)]; !ok { +func cloneBlockMetadataForQuery(b *metastorev1.BlockMeta) *metastorev1.BlockMeta { + datasets := b.Datasets + b.Datasets = nil + c := b.CloneVT() + b.Datasets = datasets + c.Datasets = make([]*metastorev1.Dataset, 0, len(b.Datasets)) + return c +} + +func cloneDatasetMetadataForQuery(ds *metastorev1.Dataset) *metastorev1.Dataset { + ls := ds.Labels + ds.Labels = nil + c := ds.CloneVT() + ds.Labels = ls + return c +} + +func datasetMatches( + q *metadataQuery, + s *metadata.StringTable, + m *metadata.LabelMatcher, + ds *metastorev1.Dataset, +) bool { + if _, ok := q.tenantMap[s.Lookup(ds.Tenant)]; !ok { return false } - if !mi.query.overlaps(time.UnixMilli(ds.MinTime), time.UnixMilli(ds.MaxTime)) { + if !q.overlaps(time.UnixMilli(ds.MinTime), time.UnixMilli(ds.MaxTime)) { return false } - // TODO: Cache; we shouldn't check the same name multiple times. - if mi.query.matcher != nil { - return mi.query.matcher.Matches(s.Lookup(ds.Name)) + pairs := metadata.LabelPairs(ds.Labels) + var matches bool + for pairs.Next() { + if m.Matches(pairs.At()) { + matches = true + } + // If no labels are specified, we can return early. + // Otherwise, we need to scan all the label sets to + // collect matching ones. + if matches && len(q.labels) == 0 { + return true + } } - return true + return matches } -func (q *metadataQuery) overlaps(start, end time.Time) bool { - return start.Before(q.endTime) && !end.Before(q.startTime) +func newMetadataLabelQuerier(tx *bbolt.Tx, q *metadataQuery) *metadataLabelQuerier { + return &metadataLabelQuerier{ + query: q, + shards: newShardIterator(tx, q.index, q.startTime, q.endTime, q.tenants...), + labels: model.NewLabelMerger(), + } } -func cloneMetadataForQuery(b *metastorev1.BlockMeta) *metastorev1.BlockMeta { - datasets := b.Datasets - b.Datasets = nil - c := b.CloneVT() - b.Datasets = datasets - c.Datasets = make([]*metastorev1.Dataset, 0, len(b.Datasets)) - return c +type metadataLabelQuerier struct { + query *metadataQuery + shards iter.Iterator[*indexShard] + labels *model.LabelMerger +} + +func (mi *metadataLabelQuerier) queryLabels() (*model.LabelMerger, error) { + if len(mi.query.labels) == 0 { + return mi.labels, nil + } + for mi.shards.Next() { + mi.collectLabels(mi.shards.At()) + } + if err := mi.shards.Err(); err != nil { + return nil, err + } + return mi.labels, nil +} + +func (mi *metadataLabelQuerier) collectLabels(shard *indexShard) { + m := metadata.NewLabelMatcher( + shard.StringTable, + mi.query.matchers, + mi.query.labels..., + ) + if !m.IsValid() { + return + } + for _, md := range shard.blocks { + if !mi.query.overlaps(time.UnixMilli(md.MinTime), time.UnixMilli(md.MaxTime)) { + continue + } + for _, ds := range md.Datasets { + datasetMatches(mi.query, shard.StringTable, m, ds) + } + } + mi.labels.MergeLabels(m.Matched()) +} + +type shardIterator struct { + tx *bbolt.Tx + index *Index + tenants []string + partitions []*store.Partition + shards []*indexShard + cur int + err error +} + +func newShardIterator(tx *bbolt.Tx, index *Index, startTime, endTime time.Time, tenants ...string) iter.Iterator[*indexShard] { + startTime = startTime.Add(-index.config.QueryLookaroundPeriod) + endTime = endTime.Add(index.config.QueryLookaroundPeriod) + si := shardIterator{ + tx: tx, + partitions: make([]*store.Partition, 0, len(index.partitions)), + tenants: tenants, + index: index, + } + for _, p := range index.partitions { + if !p.Overlaps(startTime, endTime) { + continue + } + for _, t := range si.tenants { + if p.HasTenant(t) { + si.partitions = append(si.partitions, p) + break + } + } + } + return &si +} + +func (si *shardIterator) Close() error { return nil } + +func (si *shardIterator) Err() error { return si.err } + +func (si *shardIterator) At() *indexShard { return si.shards[si.cur] } + +func (si *shardIterator) Next() bool { + if n := si.cur + 1; n < len(si.shards) { + si.cur = n + return true + } + si.cur = 0 + si.shards = si.shards[:0] + for len(si.shards) == 0 && len(si.partitions) > 0 { + si.loadShards(si.partitions[0]) + si.partitions = si.partitions[1:] + } + return si.cur < len(si.shards) +} + +func (si *shardIterator) loadShards(p *store.Partition) { + for _, t := range si.tenants { + shards := p.TenantShards[t] + if shards == nil { + continue + } + for s := range shards { + shard, err := si.index.getOrLoadTenantShard(si.tx, p, t, s) + if err != nil { + si.err = err + return + } + if shard != nil { + si.shards = append(si.shards, shard) + } + } + } + slices.SortFunc(si.shards, compareShards) + si.shards = slices.Compact(si.shards) +} + +func compareShards(a, b *indexShard) int { + cmp := strings.Compare(a.Tenant, b.Tenant) + if cmp == 0 { + return int(a.Shard) - int(b.Shard) + } + return cmp } diff --git a/pkg/experiment/metastore/index/query_test.go b/pkg/experiment/metastore/index/query_test.go index f389aef319..0f41a7ef2e 100644 --- a/pkg/experiment/metastore/index/query_test.go +++ b/pkg/experiment/metastore/index/query_test.go @@ -4,11 +4,14 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.etcd.io/bbolt" metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" + typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" "github.com/grafana/pyroscope/pkg/iter" + "github.com/grafana/pyroscope/pkg/model" "github.com/grafana/pyroscope/pkg/test" "github.com/grafana/pyroscope/pkg/util" ) @@ -26,12 +29,12 @@ func TestIndex_Query(t *testing.T) { MaxTime: maxT, CreatedBy: 1, Datasets: []*metastorev1.Dataset{ - {Tenant: 2, Name: 3, ProfileTypes: []int32{4, 5, 6}, MinTime: minT, MaxTime: minT}, - {Tenant: 7, Name: 8, ProfileTypes: []int32{5, 6, 9}, MinTime: maxT, MaxTime: maxT}, + {Tenant: 2, Name: 3, MinTime: minT, MaxTime: minT, Labels: []int32{2, 4, 3, 5, 6}}, + {Tenant: 7, Name: 8, MinTime: maxT, MaxTime: maxT, Labels: []int32{2, 4, 8, 5, 9}}, }, StringTable: []string{ "", "ingester", - "tenant-a", "dataset-a", "1", "2", "3", + "tenant-a", "dataset-a", "service_name", "__profile_type__", "1", "tenant-b", "dataset-b", "4", }, } @@ -44,11 +47,10 @@ func TestIndex_Query(t *testing.T) { MaxTime: maxT, CreatedBy: 2, Datasets: []*metastorev1.Dataset{ - {Tenant: 1, Name: 3, ProfileTypes: []int32{4, 5, 6}, MinTime: minT, MaxTime: minT}, + {Tenant: 1, Name: 3, MinTime: minT, MaxTime: minT, Labels: []int32{2, 4, 3, 5, 6}}, }, StringTable: []string{ - "", "tenant-a", "ingester", - "dataset-a", "1", "2", "3", + "", "tenant-a", "ingester", "dataset-a", "service_name", "__profile_type__", "1", }, } @@ -60,11 +62,10 @@ func TestIndex_Query(t *testing.T) { MaxTime: maxT, CreatedBy: 2, Datasets: []*metastorev1.Dataset{ - {Tenant: 1, Name: 3, ProfileTypes: []int32{4, 5, 6}, MinTime: minT, MaxTime: minT}, + {Tenant: 1, Name: 3, MinTime: minT, MaxTime: minT, Labels: []int32{2, 4, 3, 5, 6}}, }, StringTable: []string{ - "", "tenant-a", "ingester", - "dataset-a", "1", "2", "3", + "", "tenant-a", "ingester", "dataset-a", "service_name", "__profile_type__", "1", }, } @@ -104,39 +105,33 @@ func TestIndex_Query(t *testing.T) { t.Run("DatasetFilter", func(t *testing.T) { expected := []*metastorev1.BlockMeta{ { - Id: md.Id, - Tenant: 0, - MinTime: minT, - MaxTime: maxT, - CreatedBy: 1, - Datasets: []*metastorev1.Dataset{ - {Tenant: 2, Name: 3, ProfileTypes: []int32{4, 5, 6}, MinTime: minT, MaxTime: minT}, - }, - StringTable: []string{"", "ingester", "tenant-a", "dataset-a", "1", "2", "3"}, + Id: md.Id, + Tenant: 0, + MinTime: minT, + MaxTime: maxT, + CreatedBy: 1, + Datasets: []*metastorev1.Dataset{{Tenant: 2, Name: 3, MinTime: minT, MaxTime: minT}}, + StringTable: []string{"", "ingester", "tenant-a", "dataset-a"}, }, { - Id: md2.Id, - Tenant: 1, - Shard: 1, - MinTime: minT, - MaxTime: maxT, - CreatedBy: 2, - Datasets: []*metastorev1.Dataset{ - {Tenant: 1, Name: 3, ProfileTypes: []int32{4, 5, 6}, MinTime: minT, MaxTime: minT}, - }, - StringTable: []string{"", "tenant-a", "ingester", "dataset-a", "1", "2", "3"}, + Id: md2.Id, + Tenant: 1, + Shard: 1, + MinTime: minT, + MaxTime: maxT, + CreatedBy: 2, + Datasets: []*metastorev1.Dataset{{Tenant: 1, Name: 3, MinTime: minT, MaxTime: minT}}, + StringTable: []string{"", "tenant-a", "ingester", "dataset-a"}, }, { - Id: md3.Id, - Tenant: 1, - Shard: 1, - MinTime: minT, - MaxTime: maxT, - CreatedBy: 2, - Datasets: []*metastorev1.Dataset{ - {Tenant: 1, Name: 3, ProfileTypes: []int32{4, 5, 6}, MinTime: minT, MaxTime: minT}, - }, - StringTable: []string{"", "tenant-a", "ingester", "dataset-a", "1", "2", "3"}, + Id: md3.Id, + Tenant: 1, + Shard: 1, + MinTime: minT, + MaxTime: maxT, + CreatedBy: 2, + Datasets: []*metastorev1.Dataset{{Tenant: 1, Name: 3, MinTime: minT, MaxTime: minT}}, + StringTable: []string{"", "tenant-a", "ingester", "dataset-a"}, }, } @@ -160,6 +155,27 @@ func TestIndex_Query(t *testing.T) { require.NoError(t, err) require.Empty(t, found) }) + + t.Run("Labels", func(t *testing.T) { + labels, err := index.QueryMetadataLabels(tx, MetadataLabelQuery{ + Labels: []string{ + model.LabelNameProfileType, + model.LabelNameServiceName, + }, + MetadataQuery: MetadataQuery{ + Expr: `{service_name=~"dataset.*"}`, + StartTime: time.UnixMilli(minT), + EndTime: time.UnixMilli(maxT), + Tenant: []string{"tenant-a"}, + }, + }) + require.NoError(t, err) + require.NotEmpty(t, labels) + assert.Equal(t, []*typesv1.Labels{{Labels: []*typesv1.LabelPair{ + {Name: model.LabelNameProfileType, Value: "1"}, + {Name: model.LabelNameServiceName, Value: "dataset-a"}, + }}}, labels) + }) } idx := NewIndex(util.Logger, NewStore(), &DefaultConfig) diff --git a/pkg/experiment/metastore/index/store/index_store.go b/pkg/experiment/metastore/index/store/index_store.go index a192a89539..4814423c07 100644 --- a/pkg/experiment/metastore/index/store/index_store.go +++ b/pkg/experiment/metastore/index/store/index_store.go @@ -9,7 +9,7 @@ import ( "go.etcd.io/bbolt" metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" - "github.com/grafana/pyroscope/pkg/experiment/block" + "github.com/grafana/pyroscope/pkg/experiment/block/metadata" "github.com/grafana/pyroscope/pkg/experiment/metastore/store" "github.com/grafana/pyroscope/pkg/iter" ) @@ -36,7 +36,7 @@ type Entry struct { Tenant string BlockID string BlockMeta *metastorev1.BlockMeta - StringTable *block.MetadataStrings + StringTable *metadata.StringTable } type TenantShard struct { @@ -44,7 +44,7 @@ type TenantShard struct { Tenant string Shard uint32 Blocks []*metastorev1.BlockMeta - StringTable *block.MetadataStrings + StringTable *metadata.StringTable } type IndexStore struct{} @@ -179,7 +179,7 @@ func (m *IndexStore) loadTenantShard(tx *bbolt.Tx, p PartitionKey, tenant string Partition: p, Tenant: tenant, Shard: shard, - StringTable: block.NewMetadataStringTable(), + StringTable: metadata.NewStringTable(), } strings := tenantShard.Bucket(tenantShardStringsBucketNameBytes) diff --git a/pkg/experiment/metastore/index_service.go b/pkg/experiment/metastore/index_service.go index e97a64a943..10953dbfe2 100644 --- a/pkg/experiment/metastore/index_service.go +++ b/pkg/experiment/metastore/index_service.go @@ -11,7 +11,7 @@ import ( metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1/raft_log" - "github.com/grafana/pyroscope/pkg/experiment/block" + "github.com/grafana/pyroscope/pkg/experiment/block/metadata" placement "github.com/grafana/pyroscope/pkg/experiment/distributor/placement/adaptive_placement" "github.com/grafana/pyroscope/pkg/experiment/metastore/fsm" "github.com/grafana/pyroscope/pkg/experiment/metastore/raftnode" @@ -75,7 +75,7 @@ func (svc *IndexService) addBlockMetadata( _ context.Context, req *metastorev1.AddBlockRequest, ) (*metastorev1.AddBlockResponse, error) { - if err := block.SanitizeMetadata(req.Block); err != nil { + if err := metadata.Sanitize(req.Block); err != nil { level.Warn(svc.logger).Log("invalid metadata", "block", req.Block.Id, "err", err) return nil, err } diff --git a/pkg/experiment/metastore/query_service.go b/pkg/experiment/metastore/query_service.go index cc9acdf240..a2c4ae5039 100644 --- a/pkg/experiment/metastore/query_service.go +++ b/pkg/experiment/metastore/query_service.go @@ -12,6 +12,7 @@ import ( "google.golang.org/grpc/status" metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" + typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" "github.com/grafana/pyroscope/pkg/experiment/metastore/index" "github.com/grafana/pyroscope/pkg/experiment/metastore/raftnode" "github.com/grafana/pyroscope/pkg/iter" @@ -19,6 +20,7 @@ import ( type IndexQuerier interface { QueryMetadata(*bbolt.Tx, index.MetadataQuery) iter.Iterator[*metastorev1.BlockMeta] + QueryMetadataLabels(*bbolt.Tx, index.MetadataLabelQuery) ([]*typesv1.Labels, error) } type MetadataQueryService struct { @@ -75,3 +77,41 @@ func (svc *MetadataQueryService) queryMetadata( level.Error(svc.logger).Log("msg", "failed to query metadata", "err", err) return nil, status.Error(codes.Internal, err.Error()) } + +func (svc *MetadataQueryService) QueryMetadataLabels( + ctx context.Context, + req *metastorev1.QueryMetadataLabelsRequest, +) (resp *metastorev1.QueryMetadataLabelsResponse, err error) { + read := func(tx *bbolt.Tx, _ raftnode.ReadIndex) { + resp, err = svc.queryMetadataLabels(ctx, tx, req) + } + if readErr := svc.state.ConsistentRead(ctx, read); readErr != nil { + return nil, status.Error(codes.Unavailable, readErr.Error()) + } + return resp, err +} + +func (svc *MetadataQueryService) queryMetadataLabels( + _ context.Context, + tx *bbolt.Tx, + req *metastorev1.QueryMetadataLabelsRequest, +) (*metastorev1.QueryMetadataLabelsResponse, error) { + labels, err := svc.index.QueryMetadataLabels(tx, index.MetadataLabelQuery{ + Labels: req.Labels, + MetadataQuery: index.MetadataQuery{ + Expr: req.Query, + StartTime: time.UnixMilli(req.StartTime), + EndTime: time.UnixMilli(req.EndTime), + Tenant: req.TenantId, + }, + }) + if err == nil { + return &metastorev1.QueryMetadataLabelsResponse{Labels: labels}, nil + } + var invalid *index.InvalidQueryError + if errors.As(err, &invalid) { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + level.Error(svc.logger).Log("msg", "failed to query metadata labels", "err", err) + return nil, status.Error(codes.Internal, err.Error()) +} diff --git a/pkg/experiment/query_backend/query_series_labels.go b/pkg/experiment/query_backend/query_series_labels.go index c3f1735279..436687e30c 100644 --- a/pkg/experiment/query_backend/query_series_labels.go +++ b/pkg/experiment/query_backend/query_series_labels.go @@ -88,7 +88,7 @@ func (a *seriesLabelsAggregator) build() *queryv1.Report { return &queryv1.Report{ SeriesLabels: &queryv1.SeriesLabelsReport{ Query: a.query, - SeriesLabels: a.series.SeriesLabels(), + SeriesLabels: a.series.Labels(), }, } } diff --git a/pkg/frontend/read_path/query_frontend/compat.go b/pkg/frontend/read_path/query_frontend/compat.go index 7f152326e3..654a21620b 100644 --- a/pkg/frontend/read_path/query_frontend/compat.go +++ b/pkg/frontend/read_path/query_frontend/compat.go @@ -1,107 +1,16 @@ package query_frontend import ( - "context" "fmt" - "slices" - "sort" "strings" - "connectrpc.com/connect" - "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/promql/parser" - metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" - querierv1 "github.com/grafana/pyroscope/api/gen/proto/go/querier/v1" queryv1 "github.com/grafana/pyroscope/api/gen/proto/go/query/v1" - typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" phlaremodel "github.com/grafana/pyroscope/pkg/model" ) -// TODO(kolesnikovae): Extend the metastore API to query arbitrary dataset labels. - -var profileTypeLabels2 = []string{ - "__profile_type__", - "service_name", -} - -var profileTypeLabels5 = []string{ - "__name__", - "__profile_type__", - "__type__", - "pyroscope_app", - "service_name", -} - -func isProfileTypeQuery(labels, matchers []string) bool { - if len(matchers) > 0 { - return false - } - var s []string - switch len(labels) { - case 2: - s = profileTypeLabels2 - case 5: - s = profileTypeLabels5 - default: - return false - } - sort.Strings(labels) - return slices.Compare(s, labels) == 0 -} - -func listProfileTypesFromMetadataAsSeriesLabels( - ctx context.Context, - client metastorev1.MetadataQueryServiceClient, - tenants []string, - startTime int64, - endTime int64, - labels []string, -) (*connect.Response[querierv1.SeriesResponse], error) { - resp, err := listProfileTypesFromMetadata(ctx, client, tenants, startTime, endTime) - if err != nil { - return nil, err - } - return connect.NewResponse(&querierv1.SeriesResponse{ - LabelsSet: resp.buildSeriesLabels(labels), - }), nil -} - -func listProfileTypesFromMetadata( - ctx context.Context, - client metastorev1.MetadataQueryServiceClient, - tenants []string, - startTime int64, - endTime int64, -) (*ptypes, error) { - resp, err := client.QueryMetadata(ctx, &metastorev1.QueryMetadataRequest{ - TenantId: tenants, - StartTime: startTime, - EndTime: endTime, - Query: "{}", - }) - if err != nil { - return nil, err - } - p := newProfileTypesResponseBuilder(len(resp.Blocks) * 8) - for _, md := range resp.Blocks { - for _, ds := range md.Datasets { - s := md.StringTable[ds.Name] - sp, ok := p.services[s] - if !ok { - sp = make(map[string]struct{}, len(ds.ProfileTypes)) - p.services[s] = sp - } - for _, t := range ds.ProfileTypes { - sp[md.StringTable[t]] = struct{}{} - } - } - } - return p, nil -} - func buildLabelSelectorFromMatchers(matchers []string) (string, error) { parsed, err := parseMatchers(matchers) if err != nil { @@ -160,101 +69,3 @@ func findReport(r queryv1.ReportType, reports []*queryv1.Report) *queryv1.Report } return nil } - -type ptypes struct { - services map[string]map[string]struct{} -} - -func newProfileTypesResponseBuilder(size int) *ptypes { - return &ptypes{services: make(map[string]map[string]struct{}, size)} -} - -func (p *ptypes) buildSeriesLabels(names []string) (labels []*typesv1.Labels) { - switch len(names) { - case 2: - labels = p.buildSeriesLabels2() - case 5: - labels = p.buildSeriesLabels5() - default: - panic("bug: invalid request: expected 2 or 5 label names") - } - slices.SortFunc(labels, func(a, b *typesv1.Labels) int { - return phlaremodel.CompareLabelPairs(a.Labels, b.Labels) - }) - return labels -} - -func (p *ptypes) buildSeriesLabels2() []*typesv1.Labels { - labels := make([]*typesv1.Labels, 0, len(p.services)*4) - for n, types := range p.services { - for t := range types { - labels = append(labels, &typesv1.Labels{ - Labels: []*typesv1.LabelPair{ - {Name: "__profile_type__", Value: t}, - {Name: "service_name", Value: n}, - }, - }) - } - } - return labels -} - -func (p *ptypes) buildSeriesLabels5() []*typesv1.Labels { - labels := make([]*typesv1.Labels, 0, len(p.services)*4) - for n, types := range p.services { - for t := range types { - pt, err := phlaremodel.ParseProfileTypeSelector(t) - if err != nil { - panic("bug: invalid profile type: " + err.Error()) - } - labels = append(labels, &typesv1.Labels{ - Labels: []*typesv1.LabelPair{ - {Name: "__profile_type__", Value: t}, - {Name: "service_name", Value: n}, - {Name: "__name__", Value: pt.Name}, - {Name: "__type__", Value: pt.SampleType}, - }, - }) - } - } - return labels -} - -//nolint:unused -func printStats(logger log.Logger, blocks []*metastorev1.BlockMeta) { - type blockMetaStats struct { - level uint32 - minTime int64 - maxTime int64 - size uint64 - count int - } - m := make(map[uint32]*blockMetaStats) - for _, b := range blocks { - s, ok := m[b.CompactionLevel] - if !ok { - s = &blockMetaStats{level: b.CompactionLevel} - m[b.CompactionLevel] = s - } - for _, x := range b.Datasets { - s.size += x.Size - } - s.count++ - } - sorted := make([]*blockMetaStats, 0, len(m)) - for _, s := range m { - sorted = append(sorted, s) - } - slices.SortFunc(sorted, func(a, b *blockMetaStats) int { - return int(a.level - b.level) - }) - fields := make([]interface{}, 0, 4+len(sorted)*2) - fields = append(fields, "msg", "block metadata list", "blocks_total", fmt.Sprint(len(blocks))) - for _, s := range sorted { - fields = append(fields, - fmt.Sprintf("l%d_blocks", s.level), fmt.Sprint(s.count), - fmt.Sprintf("l%d_size", s.level), fmt.Sprint(s.size), - ) - } - _ = level.Info(logger).Log(fields...) -} diff --git a/pkg/frontend/read_path/query_frontend/query_profile_types.go b/pkg/frontend/read_path/query_frontend/query_profile_types.go index 381c0b7a1c..a56ef67331 100644 --- a/pkg/frontend/read_path/query_frontend/query_profile_types.go +++ b/pkg/frontend/read_path/query_frontend/query_profile_types.go @@ -2,9 +2,11 @@ package query_frontend import ( "context" - "sort" + "slices" + "strings" "connectrpc.com/connect" + "github.com/go-kit/log/level" "github.com/grafana/dskit/tenant" metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" @@ -18,7 +20,6 @@ func (q *QueryFrontend) ProfileTypes( ctx context.Context, req *connect.Request[querierv1.ProfileTypesRequest], ) (*connect.Response[querierv1.ProfileTypesResponse], error) { - tenants, err := tenant.TenantIDs(ctx) if err != nil { return nil, connect.NewError(connect.CodeInvalidArgument, err) @@ -31,43 +32,34 @@ func (q *QueryFrontend) ProfileTypes( return connect.NewResponse(&querierv1.ProfileTypesResponse{}), nil } - resp, err := q.metadataQueryClient.QueryMetadata(ctx, &metastorev1.QueryMetadataRequest{ + resp, err := q.metadataQueryClient.QueryMetadataLabels(ctx, &metastorev1.QueryMetadataLabelsRequest{ TenantId: tenants, StartTime: req.Msg.Start, EndTime: req.Msg.End, Query: "{}", + Labels: []string{phlaremodel.LabelNameProfileType}, }) if err != nil { return nil, err } - pTypesFromMetadata := make(map[string]*typesv1.ProfileType) - for _, md := range resp.Blocks { - for _, ds := range md.Datasets { - for _, p := range ds.ProfileTypes { - t := md.StringTable[p] - if _, ok := pTypesFromMetadata[t]; !ok { - profileType, err := phlaremodel.ParseProfileTypeSelector(t) - if err != nil { - return nil, err - } - pTypesFromMetadata[t] = profileType - } - } + types := make([]*typesv1.ProfileType, 0, len(resp.Labels)) + for _, ls := range resp.Labels { + var typ *typesv1.ProfileType + if len(ls.Labels) == 1 && ls.Labels[0].Name == phlaremodel.LabelNameProfileType { + typ, err = phlaremodel.ParseProfileTypeSelector(ls.Labels[0].Value) } + if err != nil || typ == nil { + level.Warn(q.logger).Log("msg", "malformed label set", "labels", phlaremodel.LabelPairsString(ls.Labels)) + continue + } + types = append(types, typ) } - var profileTypes []*typesv1.ProfileType - for _, pType := range pTypesFromMetadata { - profileTypes = append(profileTypes, pType) - } - - sort.Slice(profileTypes, func(i, j int) bool { - return profileTypes[i].ID < profileTypes[j].ID + slices.SortFunc(types, func(a, b *typesv1.ProfileType) int { + return strings.Compare(a.ID, b.ID) }) - return connect.NewResponse(&querierv1.ProfileTypesResponse{ - ProfileTypes: profileTypes, - }), nil + return connect.NewResponse(&querierv1.ProfileTypesResponse{ProfileTypes: types}), nil } diff --git a/pkg/frontend/read_path/query_frontend/query_profile_types_test.go b/pkg/frontend/read_path/query_frontend/query_profile_types_test.go deleted file mode 100644 index cc27e7d7a7..0000000000 --- a/pkg/frontend/read_path/query_frontend/query_profile_types_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package query_frontend - -import ( - "context" - "testing" - "time" - - "connectrpc.com/connect" - "github.com/go-kit/log" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" - querierv1 "github.com/grafana/pyroscope/api/gen/proto/go/querier/v1" - "github.com/grafana/pyroscope/pkg/tenant" - "github.com/grafana/pyroscope/pkg/test/mocks/mockfrontend" - "github.com/grafana/pyroscope/pkg/test/mocks/mockmetastorev1" -) - -func TestQueryFrontend_ProfileTypes(t *testing.T) { - metaClient := mockmetastorev1.NewMockMetadataQueryServiceClient(t) - limits := mockfrontend.NewMockLimits(t) - f := NewQueryFrontend(log.NewNopLogger(), limits, metaClient, nil, nil) - require.NotNil(t, f) - - limits.On("MaxQueryLookback", mock.Anything).Return(24 * time.Hour) - limits.On("MaxQueryLength", mock.Anything).Return(2 * time.Hour) - metaClient.On("QueryMetadata", mock.Anything, mock.Anything).Maybe().Return(&metastorev1.QueryMetadataResponse{ - Blocks: []*metastorev1.BlockMeta{ - { - Datasets: []*metastorev1.Dataset{ - {ProfileTypes: []int32{1, 2, 3}}, - {ProfileTypes: []int32{4, 2}}, - }, - StringTable: []string{ - "", - "memory:inuse_space:bytes:space:byte", - "process_cpu:cpu:nanoseconds:cpu:nanoseconds", - "mutex:delay:nanoseconds:mutex:count", - "memory:alloc_in_new_tlab_objects:count:space:bytes", - }, - }, - { - Datasets: []*metastorev1.Dataset{ - {ProfileTypes: []int32{1, 2}}, - }, - StringTable: []string{ - "", - "mutex:contentions:count:mutex:count", - "mutex:delay:nanoseconds:mutex:count", - }, - }, - }, - }, nil) - - ctx := tenant.InjectTenantID(context.Background(), "tenant") - types, err := f.ProfileTypes(ctx, connect.NewRequest(&querierv1.ProfileTypesRequest{ - Start: time.Now().Add(-time.Hour).UnixMilli(), - End: time.Now().UnixMilli(), - })) - require.NoError(t, err) - require.Equal(t, 5, len(types.Msg.ProfileTypes)) - require.Equal(t, "memory:alloc_in_new_tlab_objects:count:space:bytes", types.Msg.ProfileTypes[0].ID) - require.Equal(t, "memory:inuse_space:bytes:space:byte", types.Msg.ProfileTypes[1].ID) - require.Equal(t, "mutex:contentions:count:mutex:count", types.Msg.ProfileTypes[2].ID) - require.Equal(t, "mutex:delay:nanoseconds:mutex:count", types.Msg.ProfileTypes[3].ID) - require.Equal(t, "process_cpu:cpu:nanoseconds:cpu:nanoseconds", types.Msg.ProfileTypes[4].ID) -} diff --git a/pkg/frontend/read_path/query_frontend/query_series_labels.go b/pkg/frontend/read_path/query_frontend/query_series_labels.go index 7ce318cb76..59ac1fd4df 100644 --- a/pkg/frontend/read_path/query_frontend/query_series_labels.go +++ b/pkg/frontend/read_path/query_frontend/query_series_labels.go @@ -36,9 +36,9 @@ func (q *QueryFrontend) Series( return connect.NewResponse(&querierv1.SeriesResponse{}), nil } - if isProfileTypeQuery(c.Msg.LabelNames, c.Msg.Matchers) { - _ = level.Debug(q.logger).Log("msg", "listing profile types from metadata as series labels") - return listProfileTypesFromMetadataAsSeriesLabels(ctx, q.metadataQueryClient, tenantIDs, c.Msg.Start, c.Msg.End, c.Msg.LabelNames) + if q.isProfileTypeQuery(c.Msg.LabelNames, c.Msg.Matchers) { + level.Debug(q.logger).Log("msg", "listing profile types from metadata as series labels") + return q.queryProfileTypeMetadataLabels(ctx, q.metadataQueryClient, tenantIDs, c.Msg.Start, c.Msg.End, c.Msg.LabelNames) } labelSelector, err := buildLabelSelectorFromMatchers(c.Msg.Matchers) diff --git a/pkg/frontend/read_path/query_frontend/query_series_labels_compat.go b/pkg/frontend/read_path/query_frontend/query_series_labels_compat.go new file mode 100644 index 0000000000..daec77ab93 --- /dev/null +++ b/pkg/frontend/read_path/query_frontend/query_series_labels_compat.go @@ -0,0 +1,127 @@ +package query_frontend + +import ( + "context" + "fmt" + "slices" + "sort" + + "connectrpc.com/connect" + "github.com/go-kit/log/level" + "github.com/pkg/errors" + + metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1" + querierv1 "github.com/grafana/pyroscope/api/gen/proto/go/querier/v1" + typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" + phlaremodel "github.com/grafana/pyroscope/pkg/model" +) + +var ( + errMissingServiceName = errors.New("service name is missing") + errMissingProfileType = errors.New("profile type is missing") + errInvalidProfileType = errors.New("invalid profile type") +) + +var profileTypeLabels2 = []string{ + phlaremodel.LabelNameProfileType, + phlaremodel.LabelNameServiceName, +} + +var profileTypeLabels5 = []string{ + phlaremodel.LabelNameProfileName, + phlaremodel.LabelNameProfileType, + phlaremodel.LabelNameType, + "pyroscope_app", + phlaremodel.LabelNameServiceName, +} + +func (q *QueryFrontend) isProfileTypeQuery(labels, matchers []string) bool { + if len(matchers) > 0 { + return false + } + var s []string + switch len(labels) { + case 2: + s = profileTypeLabels2 + case 5: + s = profileTypeLabels5 + default: + return false + } + sort.Strings(labels) + return slices.Compare(s, labels) == 0 +} + +func (q *QueryFrontend) queryProfileTypeMetadataLabels( + ctx context.Context, + client metastorev1.MetadataQueryServiceClient, + tenants []string, + startTime int64, + endTime int64, + labels []string, +) (*connect.Response[querierv1.SeriesResponse], error) { + meta, err := client.QueryMetadataLabels(ctx, &metastorev1.QueryMetadataLabelsRequest{ + TenantId: tenants, + StartTime: startTime, + EndTime: endTime, + Query: "{}", + Labels: []string{ + phlaremodel.LabelNameServiceName, + phlaremodel.LabelNameProfileType, + }, + }) + if err != nil { + return nil, err + } + meta.Labels = q.buildProfileTypeMetadataLabels(meta.Labels, labels) + return connect.NewResponse(&querierv1.SeriesResponse{LabelsSet: meta.Labels}), nil +} + +func (q *QueryFrontend) buildProfileTypeMetadataLabels(labels []*typesv1.Labels, names []string) []*typesv1.Labels { + for _, ls := range labels { + if err := sanitizeProfileTypeMetadataLabels(ls, names); err != nil { + level.Warn(q.logger).Log("msg", "malformed label set", "labels", phlaremodel.LabelPairsString(ls.Labels), "err", err) + ls.Labels = nil + } + } + labels = slices.DeleteFunc(labels, func(ls *typesv1.Labels) bool { + return len(ls.Labels) == 0 + }) + slices.SortFunc(labels, func(a, b *typesv1.Labels) int { + return phlaremodel.CompareLabelPairs(a.Labels, b.Labels) + }) + return labels +} + +func sanitizeProfileTypeMetadataLabels(ls *typesv1.Labels, names []string) error { + var serviceName, profileType string + for _, l := range ls.Labels { + switch l.Name { + case phlaremodel.LabelNameServiceName: + serviceName = l.Value + case phlaremodel.LabelNameProfileType: + profileType = l.Value + } + } + if serviceName == "" { + return errMissingServiceName + } + if profileType == "" { + return errMissingProfileType + } + pt, err := phlaremodel.ParseProfileTypeSelector(profileType) + if err != nil { + return fmt.Errorf("%w: %w", errInvalidProfileType, err) + } + if len(names) == 5 { + // Replace the labels with the expected ones. + ls.Labels = append(ls.Labels[:0], []*typesv1.LabelPair{ + {Name: phlaremodel.LabelNameProfileType, Value: profileType}, + {Name: phlaremodel.LabelNameServiceName, Value: serviceName}, + {Name: phlaremodel.LabelNameProfileName, Value: pt.Name}, + {Name: phlaremodel.LabelNameType, Value: pt.SampleType}, + }...) + } + sort.Sort(phlaremodel.Labels(ls.Labels)) + return nil +} diff --git a/pkg/frontend/read_path/query_service_handler.go b/pkg/frontend/read_path/query_service_handler.go index 223de07277..ff7fbcd8aa 100644 --- a/pkg/frontend/read_path/query_service_handler.go +++ b/pkg/frontend/read_path/query_service_handler.go @@ -52,7 +52,7 @@ func (r *Router) Series( m := phlaremodel.NewLabelMerger() m.MergeSeries(a.LabelsSet) m.MergeSeries(b.LabelsSet) - return &querierv1.SeriesResponse{LabelsSet: m.SeriesLabels()}, nil + return &querierv1.SeriesResponse{LabelsSet: m.Labels()}, nil }) } diff --git a/pkg/model/labels.go b/pkg/model/labels.go index 666074d128..f1240e0b2d 100644 --- a/pkg/model/labels.go +++ b/pkg/model/labels.go @@ -313,7 +313,7 @@ func LabelsFromStrings(ss ...string) Labels { // CompareLabelPairs compares the two label sets. // The result will be 0 if a==b, <0 if a < b, and >0 if a > b. -func CompareLabelPairs(a []*typesv1.LabelPair, b []*typesv1.LabelPair) int { +func CompareLabelPairs(a, b []*typesv1.LabelPair) int { l := len(a) if len(b) < l { l = len(b) @@ -337,6 +337,10 @@ func CompareLabelPairs(a []*typesv1.LabelPair, b []*typesv1.LabelPair) int { return len(a) - len(b) } +func CompareLabels(a, b Labels) int { + return CompareLabelPairs(a, b) +} + // LabelsBuilder allows modifying Labels. type LabelsBuilder struct { base Labels diff --git a/pkg/model/labels_merger.go b/pkg/model/labels_merger.go index ebfb66cfa6..b1d7e8760e 100644 --- a/pkg/model/labels_merger.go +++ b/pkg/model/labels_merger.go @@ -81,7 +81,15 @@ func (m *LabelMerger) MergeSeries(series []*typesv1.Labels) { } } -func (m *LabelMerger) SeriesLabels() []*typesv1.Labels { +func (m *LabelMerger) MergeLabels(ls []Labels) { + m.mu.Lock() + defer m.mu.Unlock() + for _, l := range ls { + m.series[l.Hash()] = &typesv1.Labels{Labels: l} + } +} + +func (m *LabelMerger) Labels() []*typesv1.Labels { m.mu.Lock() defer m.mu.Unlock() s := make([]*typesv1.Labels, len(m.series)) diff --git a/pkg/test/mocks/mockindex/mock_store.go b/pkg/test/mocks/mockindex/mock_store.go index c6fdf16abf..41ebbd527b 100644 --- a/pkg/test/mocks/mockindex/mock_store.go +++ b/pkg/test/mocks/mockindex/mock_store.go @@ -119,75 +119,34 @@ func (_c *MockStore_DeleteBlockList_Call) RunAndReturn(run func(*bbolt.Tx, store return _c } -// ListBlocks provides a mock function with given fields: tx, p, shard, tenant -func (_m *MockStore) ListBlocks(tx *bbolt.Tx, p store.PartitionKey, shard uint32, tenant string) []*metastorev1.BlockMeta { - ret := _m.Called(tx, p, shard, tenant) - - if len(ret) == 0 { - panic("no return value specified for ListBlocks") - } - - var r0 []*metastorev1.BlockMeta - if rf, ok := ret.Get(0).(func(*bbolt.Tx, store.PartitionKey, uint32, string) []*metastorev1.BlockMeta); ok { - r0 = rf(tx, p, shard, tenant) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*metastorev1.BlockMeta) - } - } - - return r0 -} - -// MockStore_ListBlocks_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListBlocks' -type MockStore_ListBlocks_Call struct { - *mock.Call -} - -// ListBlocks is a helper method to define mock.On call -// - tx *bbolt.Tx -// - p store.PartitionKey -// - shard uint32 -// - tenant string -func (_e *MockStore_Expecter) ListBlocks(tx interface{}, p interface{}, shard interface{}, tenant interface{}) *MockStore_ListBlocks_Call { - return &MockStore_ListBlocks_Call{Call: _e.mock.On("ListBlocks", tx, p, shard, tenant)} -} - -func (_c *MockStore_ListBlocks_Call) Run(run func(tx *bbolt.Tx, p store.PartitionKey, shard uint32, tenant string)) *MockStore_ListBlocks_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(*bbolt.Tx), args[1].(store.PartitionKey), args[2].(uint32), args[3].(string)) - }) - return _c -} - -func (_c *MockStore_ListBlocks_Call) Return(_a0 []*metastorev1.BlockMeta) *MockStore_ListBlocks_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *MockStore_ListBlocks_Call) RunAndReturn(run func(*bbolt.Tx, store.PartitionKey, uint32, string) []*metastorev1.BlockMeta) *MockStore_ListBlocks_Call { - _c.Call.Return(run) - return _c -} - // ListPartitions provides a mock function with given fields: _a0 -func (_m *MockStore) ListPartitions(_a0 *bbolt.Tx) []store.PartitionKey { +func (_m *MockStore) ListPartitions(_a0 *bbolt.Tx) ([]*store.Partition, error) { ret := _m.Called(_a0) if len(ret) == 0 { panic("no return value specified for ListPartitions") } - var r0 []store.PartitionKey - if rf, ok := ret.Get(0).(func(*bbolt.Tx) []store.PartitionKey); ok { + var r0 []*store.Partition + var r1 error + if rf, ok := ret.Get(0).(func(*bbolt.Tx) ([]*store.Partition, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(*bbolt.Tx) []*store.Partition); ok { r0 = rf(_a0) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]store.PartitionKey) + r0 = ret.Get(0).([]*store.Partition) } } - return r0 + if rf, ok := ret.Get(1).(func(*bbolt.Tx) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } // MockStore_ListPartitions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListPartitions' @@ -208,126 +167,88 @@ func (_c *MockStore_ListPartitions_Call) Run(run func(_a0 *bbolt.Tx)) *MockStore return _c } -func (_c *MockStore_ListPartitions_Call) Return(_a0 []store.PartitionKey) *MockStore_ListPartitions_Call { - _c.Call.Return(_a0) +func (_c *MockStore_ListPartitions_Call) Return(_a0 []*store.Partition, _a1 error) *MockStore_ListPartitions_Call { + _c.Call.Return(_a0, _a1) return _c } -func (_c *MockStore_ListPartitions_Call) RunAndReturn(run func(*bbolt.Tx) []store.PartitionKey) *MockStore_ListPartitions_Call { +func (_c *MockStore_ListPartitions_Call) RunAndReturn(run func(*bbolt.Tx) ([]*store.Partition, error)) *MockStore_ListPartitions_Call { _c.Call.Return(run) return _c } -// ListShards provides a mock function with given fields: _a0, _a1 -func (_m *MockStore) ListShards(_a0 *bbolt.Tx, _a1 store.PartitionKey) []uint32 { - ret := _m.Called(_a0, _a1) +// LoadTenantShard provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *MockStore) LoadTenantShard(_a0 *bbolt.Tx, _a1 store.PartitionKey, _a2 string, _a3 uint32) (*store.TenantShard, error) { + ret := _m.Called(_a0, _a1, _a2, _a3) if len(ret) == 0 { - panic("no return value specified for ListShards") + panic("no return value specified for LoadTenantShard") } - var r0 []uint32 - if rf, ok := ret.Get(0).(func(*bbolt.Tx, store.PartitionKey) []uint32); ok { - r0 = rf(_a0, _a1) + var r0 *store.TenantShard + var r1 error + if rf, ok := ret.Get(0).(func(*bbolt.Tx, store.PartitionKey, string, uint32) (*store.TenantShard, error)); ok { + return rf(_a0, _a1, _a2, _a3) + } + if rf, ok := ret.Get(0).(func(*bbolt.Tx, store.PartitionKey, string, uint32) *store.TenantShard); ok { + r0 = rf(_a0, _a1, _a2, _a3) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]uint32) + r0 = ret.Get(0).(*store.TenantShard) } } - return r0 -} - -// MockStore_ListShards_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListShards' -type MockStore_ListShards_Call struct { - *mock.Call -} - -// ListShards is a helper method to define mock.On call -// - _a0 *bbolt.Tx -// - _a1 store.PartitionKey -func (_e *MockStore_Expecter) ListShards(_a0 interface{}, _a1 interface{}) *MockStore_ListShards_Call { - return &MockStore_ListShards_Call{Call: _e.mock.On("ListShards", _a0, _a1)} -} - -func (_c *MockStore_ListShards_Call) Run(run func(_a0 *bbolt.Tx, _a1 store.PartitionKey)) *MockStore_ListShards_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(*bbolt.Tx), args[1].(store.PartitionKey)) - }) - return _c -} - -func (_c *MockStore_ListShards_Call) Return(_a0 []uint32) *MockStore_ListShards_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *MockStore_ListShards_Call) RunAndReturn(run func(*bbolt.Tx, store.PartitionKey) []uint32) *MockStore_ListShards_Call { - _c.Call.Return(run) - return _c -} - -// ListTenants provides a mock function with given fields: tx, p, shard -func (_m *MockStore) ListTenants(tx *bbolt.Tx, p store.PartitionKey, shard uint32) []string { - ret := _m.Called(tx, p, shard) - - if len(ret) == 0 { - panic("no return value specified for ListTenants") - } - - var r0 []string - if rf, ok := ret.Get(0).(func(*bbolt.Tx, store.PartitionKey, uint32) []string); ok { - r0 = rf(tx, p, shard) + if rf, ok := ret.Get(1).(func(*bbolt.Tx, store.PartitionKey, string, uint32) error); ok { + r1 = rf(_a0, _a1, _a2, _a3) } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]string) - } + r1 = ret.Error(1) } - return r0 + return r0, r1 } -// MockStore_ListTenants_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListTenants' -type MockStore_ListTenants_Call struct { +// MockStore_LoadTenantShard_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'LoadTenantShard' +type MockStore_LoadTenantShard_Call struct { *mock.Call } -// ListTenants is a helper method to define mock.On call -// - tx *bbolt.Tx -// - p store.PartitionKey -// - shard uint32 -func (_e *MockStore_Expecter) ListTenants(tx interface{}, p interface{}, shard interface{}) *MockStore_ListTenants_Call { - return &MockStore_ListTenants_Call{Call: _e.mock.On("ListTenants", tx, p, shard)} +// LoadTenantShard is a helper method to define mock.On call +// - _a0 *bbolt.Tx +// - _a1 store.PartitionKey +// - _a2 string +// - _a3 uint32 +func (_e *MockStore_Expecter) LoadTenantShard(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}) *MockStore_LoadTenantShard_Call { + return &MockStore_LoadTenantShard_Call{Call: _e.mock.On("LoadTenantShard", _a0, _a1, _a2, _a3)} } -func (_c *MockStore_ListTenants_Call) Run(run func(tx *bbolt.Tx, p store.PartitionKey, shard uint32)) *MockStore_ListTenants_Call { +func (_c *MockStore_LoadTenantShard_Call) Run(run func(_a0 *bbolt.Tx, _a1 store.PartitionKey, _a2 string, _a3 uint32)) *MockStore_LoadTenantShard_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(*bbolt.Tx), args[1].(store.PartitionKey), args[2].(uint32)) + run(args[0].(*bbolt.Tx), args[1].(store.PartitionKey), args[2].(string), args[3].(uint32)) }) return _c } -func (_c *MockStore_ListTenants_Call) Return(_a0 []string) *MockStore_ListTenants_Call { - _c.Call.Return(_a0) +func (_c *MockStore_LoadTenantShard_Call) Return(_a0 *store.TenantShard, _a1 error) *MockStore_LoadTenantShard_Call { + _c.Call.Return(_a0, _a1) return _c } -func (_c *MockStore_ListTenants_Call) RunAndReturn(run func(*bbolt.Tx, store.PartitionKey, uint32) []string) *MockStore_ListTenants_Call { +func (_c *MockStore_LoadTenantShard_Call) RunAndReturn(run func(*bbolt.Tx, store.PartitionKey, string, uint32) (*store.TenantShard, error)) *MockStore_LoadTenantShard_Call { _c.Call.Return(run) return _c } -// StoreBlock provides a mock function with given fields: _a0, _a1, _a2 -func (_m *MockStore) StoreBlock(_a0 *bbolt.Tx, _a1 store.PartitionKey, _a2 *metastorev1.BlockMeta) error { - ret := _m.Called(_a0, _a1, _a2) +// StoreBlock provides a mock function with given fields: tx, shard, md +func (_m *MockStore) StoreBlock(tx *bbolt.Tx, shard *store.TenantShard, md *metastorev1.BlockMeta) error { + ret := _m.Called(tx, shard, md) if len(ret) == 0 { panic("no return value specified for StoreBlock") } var r0 error - if rf, ok := ret.Get(0).(func(*bbolt.Tx, store.PartitionKey, *metastorev1.BlockMeta) error); ok { - r0 = rf(_a0, _a1, _a2) + if rf, ok := ret.Get(0).(func(*bbolt.Tx, *store.TenantShard, *metastorev1.BlockMeta) error); ok { + r0 = rf(tx, shard, md) } else { r0 = ret.Error(0) } @@ -341,16 +262,16 @@ type MockStore_StoreBlock_Call struct { } // StoreBlock is a helper method to define mock.On call -// - _a0 *bbolt.Tx -// - _a1 store.PartitionKey -// - _a2 *metastorev1.BlockMeta -func (_e *MockStore_Expecter) StoreBlock(_a0 interface{}, _a1 interface{}, _a2 interface{}) *MockStore_StoreBlock_Call { - return &MockStore_StoreBlock_Call{Call: _e.mock.On("StoreBlock", _a0, _a1, _a2)} +// - tx *bbolt.Tx +// - shard *store.TenantShard +// - md *metastorev1.BlockMeta +func (_e *MockStore_Expecter) StoreBlock(tx interface{}, shard interface{}, md interface{}) *MockStore_StoreBlock_Call { + return &MockStore_StoreBlock_Call{Call: _e.mock.On("StoreBlock", tx, shard, md)} } -func (_c *MockStore_StoreBlock_Call) Run(run func(_a0 *bbolt.Tx, _a1 store.PartitionKey, _a2 *metastorev1.BlockMeta)) *MockStore_StoreBlock_Call { +func (_c *MockStore_StoreBlock_Call) Run(run func(tx *bbolt.Tx, shard *store.TenantShard, md *metastorev1.BlockMeta)) *MockStore_StoreBlock_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(*bbolt.Tx), args[1].(store.PartitionKey), args[2].(*metastorev1.BlockMeta)) + run(args[0].(*bbolt.Tx), args[1].(*store.TenantShard), args[2].(*metastorev1.BlockMeta)) }) return _c } @@ -360,7 +281,7 @@ func (_c *MockStore_StoreBlock_Call) Return(_a0 error) *MockStore_StoreBlock_Cal return _c } -func (_c *MockStore_StoreBlock_Call) RunAndReturn(run func(*bbolt.Tx, store.PartitionKey, *metastorev1.BlockMeta) error) *MockStore_StoreBlock_Call { +func (_c *MockStore_StoreBlock_Call) RunAndReturn(run func(*bbolt.Tx, *store.TenantShard, *metastorev1.BlockMeta) error) *MockStore_StoreBlock_Call { _c.Call.Return(run) return _c } diff --git a/pkg/test/mocks/mockmetastorev1/mock_metadata_query_service_client.go b/pkg/test/mocks/mockmetastorev1/mock_metadata_query_service_client.go index 66cb43f8fc..5b5d2b18c7 100644 --- a/pkg/test/mocks/mockmetastorev1/mock_metadata_query_service_client.go +++ b/pkg/test/mocks/mockmetastorev1/mock_metadata_query_service_client.go @@ -99,6 +99,80 @@ func (_c *MockMetadataQueryServiceClient_QueryMetadata_Call) RunAndReturn(run fu return _c } +// QueryMetadataLabels provides a mock function with given fields: ctx, in, opts +func (_m *MockMetadataQueryServiceClient) QueryMetadataLabels(ctx context.Context, in *metastorev1.QueryMetadataLabelsRequest, opts ...grpc.CallOption) (*metastorev1.QueryMetadataLabelsResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for QueryMetadataLabels") + } + + var r0 *metastorev1.QueryMetadataLabelsResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *metastorev1.QueryMetadataLabelsRequest, ...grpc.CallOption) (*metastorev1.QueryMetadataLabelsResponse, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *metastorev1.QueryMetadataLabelsRequest, ...grpc.CallOption) *metastorev1.QueryMetadataLabelsResponse); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*metastorev1.QueryMetadataLabelsResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *metastorev1.QueryMetadataLabelsRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockMetadataQueryServiceClient_QueryMetadataLabels_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'QueryMetadataLabels' +type MockMetadataQueryServiceClient_QueryMetadataLabels_Call struct { + *mock.Call +} + +// QueryMetadataLabels is a helper method to define mock.On call +// - ctx context.Context +// - in *metastorev1.QueryMetadataLabelsRequest +// - opts ...grpc.CallOption +func (_e *MockMetadataQueryServiceClient_Expecter) QueryMetadataLabels(ctx interface{}, in interface{}, opts ...interface{}) *MockMetadataQueryServiceClient_QueryMetadataLabels_Call { + return &MockMetadataQueryServiceClient_QueryMetadataLabels_Call{Call: _e.mock.On("QueryMetadataLabels", + append([]interface{}{ctx, in}, opts...)...)} +} + +func (_c *MockMetadataQueryServiceClient_QueryMetadataLabels_Call) Run(run func(ctx context.Context, in *metastorev1.QueryMetadataLabelsRequest, opts ...grpc.CallOption)) *MockMetadataQueryServiceClient_QueryMetadataLabels_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]grpc.CallOption, len(args)-2) + for i, a := range args[2:] { + if a != nil { + variadicArgs[i] = a.(grpc.CallOption) + } + } + run(args[0].(context.Context), args[1].(*metastorev1.QueryMetadataLabelsRequest), variadicArgs...) + }) + return _c +} + +func (_c *MockMetadataQueryServiceClient_QueryMetadataLabels_Call) Return(_a0 *metastorev1.QueryMetadataLabelsResponse, _a1 error) *MockMetadataQueryServiceClient_QueryMetadataLabels_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockMetadataQueryServiceClient_QueryMetadataLabels_Call) RunAndReturn(run func(context.Context, *metastorev1.QueryMetadataLabelsRequest, ...grpc.CallOption) (*metastorev1.QueryMetadataLabelsResponse, error)) *MockMetadataQueryServiceClient_QueryMetadataLabels_Call { + _c.Call.Return(run) + return _c +} + // NewMockMetadataQueryServiceClient creates a new instance of MockMetadataQueryServiceClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewMockMetadataQueryServiceClient(t interface { diff --git a/pkg/test/mocks/mockmetastorev1/mock_metadata_query_service_server.go b/pkg/test/mocks/mockmetastorev1/mock_metadata_query_service_server.go index 5745ce7273..706face915 100644 --- a/pkg/test/mocks/mockmetastorev1/mock_metadata_query_service_server.go +++ b/pkg/test/mocks/mockmetastorev1/mock_metadata_query_service_server.go @@ -81,6 +81,65 @@ func (_c *MockMetadataQueryServiceServer_QueryMetadata_Call) RunAndReturn(run fu return _c } +// QueryMetadataLabels provides a mock function with given fields: _a0, _a1 +func (_m *MockMetadataQueryServiceServer) QueryMetadataLabels(_a0 context.Context, _a1 *metastorev1.QueryMetadataLabelsRequest) (*metastorev1.QueryMetadataLabelsResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for QueryMetadataLabels") + } + + var r0 *metastorev1.QueryMetadataLabelsResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *metastorev1.QueryMetadataLabelsRequest) (*metastorev1.QueryMetadataLabelsResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *metastorev1.QueryMetadataLabelsRequest) *metastorev1.QueryMetadataLabelsResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*metastorev1.QueryMetadataLabelsResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *metastorev1.QueryMetadataLabelsRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockMetadataQueryServiceServer_QueryMetadataLabels_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'QueryMetadataLabels' +type MockMetadataQueryServiceServer_QueryMetadataLabels_Call struct { + *mock.Call +} + +// QueryMetadataLabels is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *metastorev1.QueryMetadataLabelsRequest +func (_e *MockMetadataQueryServiceServer_Expecter) QueryMetadataLabels(_a0 interface{}, _a1 interface{}) *MockMetadataQueryServiceServer_QueryMetadataLabels_Call { + return &MockMetadataQueryServiceServer_QueryMetadataLabels_Call{Call: _e.mock.On("QueryMetadataLabels", _a0, _a1)} +} + +func (_c *MockMetadataQueryServiceServer_QueryMetadataLabels_Call) Run(run func(_a0 context.Context, _a1 *metastorev1.QueryMetadataLabelsRequest)) *MockMetadataQueryServiceServer_QueryMetadataLabels_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*metastorev1.QueryMetadataLabelsRequest)) + }) + return _c +} + +func (_c *MockMetadataQueryServiceServer_QueryMetadataLabels_Call) Return(_a0 *metastorev1.QueryMetadataLabelsResponse, _a1 error) *MockMetadataQueryServiceServer_QueryMetadataLabels_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockMetadataQueryServiceServer_QueryMetadataLabels_Call) RunAndReturn(run func(context.Context, *metastorev1.QueryMetadataLabelsRequest) (*metastorev1.QueryMetadataLabelsResponse, error)) *MockMetadataQueryServiceServer_QueryMetadataLabels_Call { + _c.Call.Return(run) + return _c +} + // mustEmbedUnimplementedMetadataQueryServiceServer provides a mock function with given fields: func (_m *MockMetadataQueryServiceServer) mustEmbedUnimplementedMetadataQueryServiceServer() { _m.Called()