diff --git a/docs/api/get-v1-importfindings.md b/docs/api/get-v1-importfindings.md new file mode 100644 index 00000000000..b925e79ed1f --- /dev/null +++ b/docs/api/get-v1-importfindings.md @@ -0,0 +1,59 @@ +--- +layout: page +title: GET /v1experimental/importfindings +permalink: /get-v1-importfindings/ +parent: API +nav_order: 5 +--- +# GET /v1experimental/importfindings/{source} +Experimental +{: .label } + +Given a specific OSV.dev source, report any records that are failing import-time quality checks. + +{: .no_toc } + +
+ + Table of contents + + {: .text-delta } +- TOC +{:toc} +
+ +## Experimental endpoint + +This API endpoint is still considered experimental. It is targeted to operators +of home databases that OSV.dev imports from. We would value any and all +feedback. If you give this a try, please consider [opening an +issue](https://github.com/google/osv.dev/issues/new) and letting us know about +any pain points or highlights. + +## Purpose + +The purpose of this endpoint is give OSV record providers (home database +operators) a machine-readable way to reason about records they have published that +do not meet [OSV.dev's quality bar](data_quality.html) (and therefore have not been imported). + +## Parameters + +The only parameter you need for this API call is the source, in order to construct the URL. + +`https://api.osv.dev/v1/importfindings/{source}` + +The `source` value is the same as the `name` value in [`source.yaml`](https://github.com/google/osv.dev/blob/master/source.yaml) + +Case Sensitivity: API requests are case-sensitive. Please ensure that you use the correct case for parameter names and values. For example, use 'ghsa' instead of 'GHSA'. + +## Request sample + +```bash +curl "https://api.osv.dev/v1experimental/importfindings/example" +``` + +## Example 200 response + +``` +{"invalid_records":[{"bug_id":"EX-1234","source":"example","findings":["IMPORT_FINDING_TYPE_INVALID_JSON"],"first_seen":"2024-12-19T15:18:00.945105Z","last_attempt":"2024-12-19T15:18:00.945105Z"}]} +``` diff --git a/docs/api/index.md b/docs/api/index.md index e19f2f52407..8b5970f1553 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -8,21 +8,25 @@ nav_order: 2 # API (1.0) ## Download the OpenAPI specification + [Download Here](https://osv.dev/docs/osv_service_v1.swagger.json){: .btn .btn-purple} ## OSV API ### Want a quick example? -Please see the [quickstart](api-quickstart.md). + +Please see the [quickstart](api-quickstart.md). ### How does the API work? -There are four different types of requests that can be made of the API. +There are five different types of requests that can be made of the API. 1. Query vulnerabilities for a particular project at a given [commit hash or version](post-v1-query.md). 2. [Batched query vulnerabilities](post-v1-querybatch.md) for given package versions and commit hashes. 3. Return a `Vulnerability` object for a given [OSV ID](get-v1-vulns.md). 4. Return a list of [probable versions](post-v1-determineversion.md) of a specified C/C++ project. (**Experimental**) +5. Retrieve [records failing import-time quality checks](get-v1-importfindings.md), by record source (**Experimental**) ### Is the API rate limited? -Currently there are no limits on the API. \ No newline at end of file + +Currently there are no limits on the API. diff --git a/gcp/api/osv_service_v1_pb2.py b/gcp/api/osv_service_v1_pb2.py index b8d6466bc11..1c6806737b6 100644 --- a/gcp/api/osv_service_v1_pb2.py +++ b/gcp/api/osv_service_v1_pb2.py @@ -13,10 +13,11 @@ from osv import vulnerability_pb2 as osv_dot_vulnerability__pb2 +from osv import importfinding_pb2 as osv_dot_importfinding__pb2 from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x14osv_service_v1.proto\x12\x06osv.v1\x1a\x17osv/vulnerability.proto\x1a\x1cgoogle/api/annotations.proto\"O\n\x11VulnerabilityList\x12!\n\x05vulns\x18\x01 \x03(\x0b\x32\x12.osv.Vulnerability\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"D\n\x16\x42\x61tchVulnerabilityList\x12*\n\x07results\x18\x01 \x03(\x0b\x32\x19.osv.v1.VulnerabilityList\"h\n\x05Query\x12\x10\n\x06\x63ommit\x18\x01 \x01(\tH\x00\x12\x11\n\x07version\x18\x02 \x01(\tH\x00\x12\x1d\n\x07package\x18\x04 \x01(\x0b\x32\x0c.osv.Package\x12\x12\n\npage_token\x18\x05 \x01(\tB\x07\n\x05param\",\n\nBatchQuery\x12\x1e\n\x07queries\x18\x01 \x03(\x0b\x32\r.osv.v1.Query\"#\n\x15GetVulnByIdParameters\x12\n\n\x02id\x18\x01 \x01(\t\"7\n\x17QueryAffectedParameters\x12\x1c\n\x05query\x18\x01 \x01(\x0b\x32\r.osv.v1.Query\"A\n\x1cQueryAffectedBatchParameters\x12!\n\x05query\x18\x01 \x01(\x0b\x32\x12.osv.v1.BatchQuery\"A\n\x1a\x44\x65termineVersionParameters\x12#\n\x05query\x18\x01 \x01(\x0b\x32\x14.osv.v1.VersionQuery\"C\n\x0cVersionQuery\x12\x0c\n\x04name\x18\x01 \x01(\t\x12%\n\x0b\x66ile_hashes\x18\x02 \x03(\x0b\x32\x10.osv.v1.FileHash\"n\n\x08\x46ileHash\x12\x11\n\tfile_path\x18\x01 \x01(\t\x12,\n\thash_type\x18\x02 \x01(\x0e\x32\x19.osv.v1.FileHash.HashType\x12\x0c\n\x04hash\x18\x03 \x01(\x0c\"\x13\n\x08HashType\x12\x07\n\x03MD5\x10\x00\"9\n\x10VersionMatchList\x12%\n\x07matches\x18\x01 \x03(\x0b\x32\x14.osv.v1.VersionMatch\"\xc7\x01\n\x0cVersionMatch\x12\r\n\x05score\x18\x01 \x01(\x01\x12\x37\n\trepo_info\x18\x02 \x01(\x0b\x32$.osv.v1.VersionRepositoryInformation\x12$\n\x0eosv_identifier\x18\x03 \x01(\x0b\x32\x0c.osv.Package\x12\r\n\x05\x63pe23\x18\x05 \x01(\t\x12\x1c\n\x14minimum_file_matches\x18\x06 \x01(\x03\x12\x1c\n\x14\x65stimated_diff_files\x18\x07 \x01(\x03\"\xc0\x01\n\x1cVersionRepositoryInformation\x12;\n\x04type\x18\x01 \x01(\x0e\x32-.osv.v1.VersionRepositoryInformation.RepoType\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\t\x12\x0b\n\x03tag\x18\x04 \x01(\t\x12\x0f\n\x07version\x18\x05 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x06 \x01(\t\"$\n\x08RepoType\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x07\n\x03GIT\x10\x01\x32\xc5\x03\n\x03OSV\x12X\n\x0bGetVulnById\x12\x1d.osv.v1.GetVulnByIdParameters\x1a\x12.osv.Vulnerability\"\x16\x82\xd3\xe4\x93\x02\x10\x12\x0e/v1/vulns/{id}\x12\x65\n\rQueryAffected\x12\x1f.osv.v1.QueryAffectedParameters\x1a\x19.osv.v1.VulnerabilityList\"\x18\x82\xd3\xe4\x93\x02\x12\"\t/v1/query:\x05query\x12y\n\x12QueryAffectedBatch\x12$.osv.v1.QueryAffectedBatchParameters\x1a\x1e.osv.v1.BatchVulnerabilityList\"\x1d\x82\xd3\xe4\x93\x02\x17\"\x0e/v1/querybatch:\x05query\x12\x81\x01\n\x10\x44\x65termineVersion\x12\".osv.v1.DetermineVersionParameters\x1a\x18.osv.v1.VersionMatchList\"/\x82\xd3\xe4\x93\x02)\" /v1experimental/determineversion:\x05queryb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x14osv_service_v1.proto\x12\x06osv.v1\x1a\x17osv/vulnerability.proto\x1a\x17osv/importfinding.proto\x1a\x1cgoogle/api/annotations.proto\"O\n\x11VulnerabilityList\x12!\n\x05vulns\x18\x01 \x03(\x0b\x32\x12.osv.Vulnerability\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"D\n\x16\x42\x61tchVulnerabilityList\x12*\n\x07results\x18\x01 \x03(\x0b\x32\x19.osv.v1.VulnerabilityList\"h\n\x05Query\x12\x10\n\x06\x63ommit\x18\x01 \x01(\tH\x00\x12\x11\n\x07version\x18\x02 \x01(\tH\x00\x12\x1d\n\x07package\x18\x04 \x01(\x0b\x32\x0c.osv.Package\x12\x12\n\npage_token\x18\x05 \x01(\tB\x07\n\x05param\",\n\nBatchQuery\x12\x1e\n\x07queries\x18\x01 \x03(\x0b\x32\r.osv.v1.Query\"#\n\x15GetVulnByIdParameters\x12\n\n\x02id\x18\x01 \x01(\t\"7\n\x17QueryAffectedParameters\x12\x1c\n\x05query\x18\x01 \x01(\x0b\x32\r.osv.v1.Query\"A\n\x1cQueryAffectedBatchParameters\x12!\n\x05query\x18\x01 \x01(\x0b\x32\x12.osv.v1.BatchQuery\"A\n\x1a\x44\x65termineVersionParameters\x12#\n\x05query\x18\x01 \x01(\x0b\x32\x14.osv.v1.VersionQuery\"*\n\x18ImportFindingsParameters\x12\x0e\n\x06source\x18\x01 \x01(\t\"C\n\x0cVersionQuery\x12\x0c\n\x04name\x18\x01 \x01(\t\x12%\n\x0b\x66ile_hashes\x18\x02 \x03(\x0b\x32\x10.osv.v1.FileHash\"n\n\x08\x46ileHash\x12\x11\n\tfile_path\x18\x01 \x01(\t\x12,\n\thash_type\x18\x02 \x01(\x0e\x32\x19.osv.v1.FileHash.HashType\x12\x0c\n\x04hash\x18\x03 \x01(\x0c\"\x13\n\x08HashType\x12\x07\n\x03MD5\x10\x00\"9\n\x10VersionMatchList\x12%\n\x07matches\x18\x01 \x03(\x0b\x32\x14.osv.v1.VersionMatch\"@\n\x11ImportFindingList\x12+\n\x0finvalid_records\x18\x01 \x03(\x0b\x32\x12.osv.ImportFinding\"\xc7\x01\n\x0cVersionMatch\x12\r\n\x05score\x18\x01 \x01(\x01\x12\x37\n\trepo_info\x18\x02 \x01(\x0b\x32$.osv.v1.VersionRepositoryInformation\x12$\n\x0eosv_identifier\x18\x03 \x01(\x0b\x32\x0c.osv.Package\x12\r\n\x05\x63pe23\x18\x05 \x01(\t\x12\x1c\n\x14minimum_file_matches\x18\x06 \x01(\x03\x12\x1c\n\x14\x65stimated_diff_files\x18\x07 \x01(\x03\"\xc0\x01\n\x1cVersionRepositoryInformation\x12;\n\x04type\x18\x01 \x01(\x0e\x32-.osv.v1.VersionRepositoryInformation.RepoType\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\t\x12\x0b\n\x03tag\x18\x04 \x01(\t\x12\x0f\n\x07version\x18\x05 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x06 \x01(\t\"$\n\x08RepoType\x12\x0f\n\x0bUNSPECIFIED\x10\x00\x12\x07\n\x03GIT\x10\x01\x32\xc5\x04\n\x03OSV\x12X\n\x0bGetVulnById\x12\x1d.osv.v1.GetVulnByIdParameters\x1a\x12.osv.Vulnerability\"\x16\x82\xd3\xe4\x93\x02\x10\x12\x0e/v1/vulns/{id}\x12\x65\n\rQueryAffected\x12\x1f.osv.v1.QueryAffectedParameters\x1a\x19.osv.v1.VulnerabilityList\"\x18\x82\xd3\xe4\x93\x02\x12\"\t/v1/query:\x05query\x12y\n\x12QueryAffectedBatch\x12$.osv.v1.QueryAffectedBatchParameters\x1a\x1e.osv.v1.BatchVulnerabilityList\"\x1d\x82\xd3\xe4\x93\x02\x17\"\x0e/v1/querybatch:\x05query\x12\x81\x01\n\x10\x44\x65termineVersion\x12\".osv.v1.DetermineVersionParameters\x1a\x18.osv.v1.VersionMatchList\"/\x82\xd3\xe4\x93\x02)\" /v1experimental/determineversion:\x05query\x12~\n\x0eImportFindings\x12 .osv.v1.ImportFindingsParameters\x1a\x19.osv.v1.ImportFindingList\"/\x82\xd3\xe4\x93\x02)\x12\'/v1experimental/importfindings/{source}b\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -31,36 +32,42 @@ _globals['_OSV'].methods_by_name['QueryAffectedBatch']._serialized_options = b'\202\323\344\223\002\027\"\016/v1/querybatch:\005query' _globals['_OSV'].methods_by_name['DetermineVersion']._loaded_options = None _globals['_OSV'].methods_by_name['DetermineVersion']._serialized_options = b'\202\323\344\223\002)\" /v1experimental/determineversion:\005query' - _globals['_VULNERABILITYLIST']._serialized_start=87 - _globals['_VULNERABILITYLIST']._serialized_end=166 - _globals['_BATCHVULNERABILITYLIST']._serialized_start=168 - _globals['_BATCHVULNERABILITYLIST']._serialized_end=236 - _globals['_QUERY']._serialized_start=238 - _globals['_QUERY']._serialized_end=342 - _globals['_BATCHQUERY']._serialized_start=344 - _globals['_BATCHQUERY']._serialized_end=388 - _globals['_GETVULNBYIDPARAMETERS']._serialized_start=390 - _globals['_GETVULNBYIDPARAMETERS']._serialized_end=425 - _globals['_QUERYAFFECTEDPARAMETERS']._serialized_start=427 - _globals['_QUERYAFFECTEDPARAMETERS']._serialized_end=482 - _globals['_QUERYAFFECTEDBATCHPARAMETERS']._serialized_start=484 - _globals['_QUERYAFFECTEDBATCHPARAMETERS']._serialized_end=549 - _globals['_DETERMINEVERSIONPARAMETERS']._serialized_start=551 - _globals['_DETERMINEVERSIONPARAMETERS']._serialized_end=616 - _globals['_VERSIONQUERY']._serialized_start=618 - _globals['_VERSIONQUERY']._serialized_end=685 - _globals['_FILEHASH']._serialized_start=687 - _globals['_FILEHASH']._serialized_end=797 - _globals['_FILEHASH_HASHTYPE']._serialized_start=778 - _globals['_FILEHASH_HASHTYPE']._serialized_end=797 - _globals['_VERSIONMATCHLIST']._serialized_start=799 - _globals['_VERSIONMATCHLIST']._serialized_end=856 - _globals['_VERSIONMATCH']._serialized_start=859 - _globals['_VERSIONMATCH']._serialized_end=1058 - _globals['_VERSIONREPOSITORYINFORMATION']._serialized_start=1061 - _globals['_VERSIONREPOSITORYINFORMATION']._serialized_end=1253 - _globals['_VERSIONREPOSITORYINFORMATION_REPOTYPE']._serialized_start=1217 - _globals['_VERSIONREPOSITORYINFORMATION_REPOTYPE']._serialized_end=1253 - _globals['_OSV']._serialized_start=1256 - _globals['_OSV']._serialized_end=1709 + _globals['_OSV'].methods_by_name['ImportFindings']._loaded_options = None + _globals['_OSV'].methods_by_name['ImportFindings']._serialized_options = b'\202\323\344\223\002)\022\'/v1experimental/importfindings/{source}' + _globals['_VULNERABILITYLIST']._serialized_start=112 + _globals['_VULNERABILITYLIST']._serialized_end=191 + _globals['_BATCHVULNERABILITYLIST']._serialized_start=193 + _globals['_BATCHVULNERABILITYLIST']._serialized_end=261 + _globals['_QUERY']._serialized_start=263 + _globals['_QUERY']._serialized_end=367 + _globals['_BATCHQUERY']._serialized_start=369 + _globals['_BATCHQUERY']._serialized_end=413 + _globals['_GETVULNBYIDPARAMETERS']._serialized_start=415 + _globals['_GETVULNBYIDPARAMETERS']._serialized_end=450 + _globals['_QUERYAFFECTEDPARAMETERS']._serialized_start=452 + _globals['_QUERYAFFECTEDPARAMETERS']._serialized_end=507 + _globals['_QUERYAFFECTEDBATCHPARAMETERS']._serialized_start=509 + _globals['_QUERYAFFECTEDBATCHPARAMETERS']._serialized_end=574 + _globals['_DETERMINEVERSIONPARAMETERS']._serialized_start=576 + _globals['_DETERMINEVERSIONPARAMETERS']._serialized_end=641 + _globals['_IMPORTFINDINGSPARAMETERS']._serialized_start=643 + _globals['_IMPORTFINDINGSPARAMETERS']._serialized_end=685 + _globals['_VERSIONQUERY']._serialized_start=687 + _globals['_VERSIONQUERY']._serialized_end=754 + _globals['_FILEHASH']._serialized_start=756 + _globals['_FILEHASH']._serialized_end=866 + _globals['_FILEHASH_HASHTYPE']._serialized_start=847 + _globals['_FILEHASH_HASHTYPE']._serialized_end=866 + _globals['_VERSIONMATCHLIST']._serialized_start=868 + _globals['_VERSIONMATCHLIST']._serialized_end=925 + _globals['_IMPORTFINDINGLIST']._serialized_start=927 + _globals['_IMPORTFINDINGLIST']._serialized_end=991 + _globals['_VERSIONMATCH']._serialized_start=994 + _globals['_VERSIONMATCH']._serialized_end=1193 + _globals['_VERSIONREPOSITORYINFORMATION']._serialized_start=1196 + _globals['_VERSIONREPOSITORYINFORMATION']._serialized_end=1388 + _globals['_VERSIONREPOSITORYINFORMATION_REPOTYPE']._serialized_start=1352 + _globals['_VERSIONREPOSITORYINFORMATION_REPOTYPE']._serialized_end=1388 + _globals['_OSV']._serialized_start=1391 + _globals['_OSV']._serialized_end=1972 # @@protoc_insertion_point(module_scope) diff --git a/gcp/api/osv_service_v1_pb2.pyi b/gcp/api/osv_service_v1_pb2.pyi index fcaa5899dc5..a8e58762f05 100644 --- a/gcp/api/osv_service_v1_pb2.pyi +++ b/gcp/api/osv_service_v1_pb2.pyi @@ -22,6 +22,7 @@ import google.protobuf.descriptor import google.protobuf.internal.containers import google.protobuf.internal.enum_type_wrapper import google.protobuf.message +import osv.importfinding_pb2 import osv.vulnerability_pb2 import sys import typing @@ -203,6 +204,23 @@ class DetermineVersionParameters(google.protobuf.message.Message): global___DetermineVersionParameters = DetermineVersionParameters +@typing.final +class ImportFindingsParameters(google.protobuf.message.Message): + """Parameters for ImportFindings.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SOURCE_FIELD_NUMBER: builtins.int + source: builtins.str + def __init__( + self, + *, + source: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["source", b"source"]) -> None: ... + +global___ImportFindingsParameters = ImportFindingsParameters + @typing.final class VersionQuery(google.protobuf.message.Message): """The version query.""" @@ -280,6 +298,24 @@ class VersionMatchList(google.protobuf.message.Message): global___VersionMatchList = VersionMatchList +@typing.final +class ImportFindingList(google.protobuf.message.Message): + """Result of ImportFindings.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + INVALID_RECORDS_FIELD_NUMBER: builtins.int + @property + def invalid_records(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[osv.importfinding_pb2.ImportFinding]: ... + def __init__( + self, + *, + invalid_records: collections.abc.Iterable[osv.importfinding_pb2.ImportFinding] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing.Literal["invalid_records", b"invalid_records"]) -> None: ... + +global___ImportFindingList = ImportFindingList + @typing.final class VersionMatch(google.protobuf.message.Message): """Match information for the provided VersionQuery.""" diff --git a/gcp/api/osv_service_v1_pb2_grpc.py b/gcp/api/osv_service_v1_pb2_grpc.py index 6df3c17b669..f4823f14ca4 100644 --- a/gcp/api/osv_service_v1_pb2_grpc.py +++ b/gcp/api/osv_service_v1_pb2_grpc.py @@ -61,6 +61,11 @@ def __init__(self, channel): request_serializer=osv__service__v1__pb2.DetermineVersionParameters.SerializeToString, response_deserializer=osv__service__v1__pb2.VersionMatchList.FromString, _registered_method=True) + self.ImportFindings = channel.unary_unary( + '/osv.v1.OSV/ImportFindings', + request_serializer=osv__service__v1__pb2.ImportFindingsParameters.SerializeToString, + response_deserializer=osv__service__v1__pb2.ImportFindingList.FromString, + _registered_method=True) class OSVServicer(object): @@ -98,6 +103,13 @@ def DetermineVersion(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def ImportFindings(self, request, context): + """Get import findings per source. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def add_OSVServicer_to_server(servicer, server): rpc_method_handlers = { @@ -121,6 +133,11 @@ def add_OSVServicer_to_server(servicer, server): request_deserializer=osv__service__v1__pb2.DetermineVersionParameters.FromString, response_serializer=osv__service__v1__pb2.VersionMatchList.SerializeToString, ), + 'ImportFindings': grpc.unary_unary_rpc_method_handler( + servicer.ImportFindings, + request_deserializer=osv__service__v1__pb2.ImportFindingsParameters.FromString, + response_serializer=osv__service__v1__pb2.ImportFindingList.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'osv.v1.OSV', rpc_method_handlers) @@ -240,3 +257,30 @@ def DetermineVersion(request, timeout, metadata, _registered_method=True) + + @staticmethod + def ImportFindings(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/osv.v1.OSV/ImportFindings', + osv__service__v1__pb2.ImportFindingsParameters.SerializeToString, + osv__service__v1__pb2.ImportFindingList.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/gcp/api/server.py b/gcp/api/server.py index 6ab56533127..13e0cfb1e5f 100644 --- a/gcp/api/server.py +++ b/gcp/api/server.py @@ -345,6 +345,27 @@ def DetermineVersion(self, request, context: grpc.ServicerContext): res = determine_version(request.query, context).result() return res + @ndb_context + @trace_filter.log_trace + def ImportFindings(self, request, context: grpc.ServicerContext): + """Return a list of `ImportFinding` for a given source.""" + source = request.source + # TODO(gongh@): add source check, + # check if the source name exists in the source repository. + if not source: + context.abort(grpc.StatusCode.INVALID_ARGUMENT, + 'Missing Source: Please specify the source') + if get_gcp_project() == _TEST_INSTANCE: + logging.info('Checking import finding for %s\n', source) + + query = osv.ImportFinding.query(osv.ImportFinding.source == source) + import_findings: list[osv.ImportFinding] = query.fetch() + invalid_records = [] + for finding in import_findings: + invalid_records.append(finding.to_proto()) + + return osv_service_v1_pb2.ImportFindingList(invalid_records=invalid_records) + @ndb_context def Check(self, request, context: grpc.ServicerContext): """Health check per the gRPC health check protocol.""" diff --git a/gcp/api/v1/api_descriptor.pb b/gcp/api/v1/api_descriptor.pb index 8a0c1e373a8..993a5c17e94 100644 Binary files a/gcp/api/v1/api_descriptor.pb and b/gcp/api/v1/api_descriptor.pb differ diff --git a/gcp/api/v1/osv_service_v1.proto b/gcp/api/v1/osv_service_v1.proto index c4b306ab6ca..5da0e1afa41 100644 --- a/gcp/api/v1/osv_service_v1.proto +++ b/gcp/api/v1/osv_service_v1.proto @@ -17,6 +17,7 @@ syntax = "proto3"; package osv.v1; import "osv/vulnerability.proto"; +import "osv/importfinding.proto"; import "google/api/annotations.proto"; // A list of Vulnerability entries. @@ -72,6 +73,11 @@ message DetermineVersionParameters { VersionQuery query = 1; } +// Parameters for ImportFindings. +message ImportFindingsParameters { + string source = 1; +} + // The version query. message VersionQuery { // The name of the dependency. Can be empty. @@ -97,6 +103,11 @@ message VersionMatchList { repeated VersionMatch matches = 1; } +// Result of ImportFindings. +message ImportFindingList { + repeated osv.ImportFinding invalid_records = 1; +} + // Match information for the provided VersionQuery. message VersionMatch { // Score in the interval (0.0, 1.0] with 1.0 being a perfect match. @@ -166,4 +177,11 @@ service OSV { body: "query" }; } + + // Get import findings per source. + rpc ImportFindings(ImportFindingsParameters) returns (ImportFindingList) { + option (google.api.http) = { + get: "/v1experimental/importfindings/{source}" + }; + } } diff --git a/osv/build_protos.sh b/osv/build_protos.sh index 8343d03b07c..2da41e3a213 100755 --- a/osv/build_protos.sh +++ b/osv/build_protos.sh @@ -15,4 +15,4 @@ cd ../ -python -m grpc_tools.protoc --python_out=. --mypy_out=. --proto_path=. osv/vulnerability.proto +python -m grpc_tools.protoc --python_out=. --mypy_out=. --proto_path=. osv/*.proto diff --git a/osv/importfinding.proto b/osv/importfinding.proto new file mode 100644 index 00000000000..2af070149a6 --- /dev/null +++ b/osv/importfinding.proto @@ -0,0 +1,42 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package osv; + +import "google/protobuf/timestamp.proto"; + +enum ImportFindingType { + IMPORT_FINDING_TYPE_NONE = 0; + IMPORT_FINDING_TYPE_DELETED = 1; + IMPORT_FINDING_TYPE_INVALID_JSON = 2; + IMPORT_FINDING_TYPE_INVALID_PACKAGE = 3; + IMPORT_FINDING_TYPE_INVALID_PURL = 4; + IMPORT_FINDING_TYPE_INVALID_VERSION = 5; + IMPORT_FINDING_TYPE_INVALID_COMMIT = 6; + IMPORT_FINDING_TYPE_INVALID_RANGE = 7; + IMPORT_FINDING_TYPE_BAD_ALIASED_CVE = 8; +} + +// An importfinding entry. +// The protobuf representation is *NOT* stable and only used for implementing +// the JSON based API. +message ImportFinding { + string bug_id = 1; + string source = 2; + repeated ImportFindingType findings = 3; + google.protobuf.Timestamp first_seen = 4; + google.protobuf.Timestamp last_attempt = 5; +} \ No newline at end of file diff --git a/osv/importfinding_pb2.py b/osv/importfinding_pb2.py new file mode 100644 index 00000000000..4930c7a6d95 --- /dev/null +++ b/osv/importfinding_pb2.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: osv/importfinding.proto +# Protobuf Python Version: 5.26.1 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17osv/importfinding.proto\x12\x03osv\x1a\x1fgoogle/protobuf/timestamp.proto\"\xbb\x01\n\rImportFinding\x12\x0e\n\x06\x62ug_id\x18\x01 \x01(\t\x12\x0e\n\x06source\x18\x02 \x01(\t\x12(\n\x08\x66indings\x18\x03 \x03(\x0e\x32\x16.osv.ImportFindingType\x12.\n\nfirst_seen\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x30\n\x0clast_attempt\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp*\xe8\x02\n\x11ImportFindingType\x12\x1c\n\x18IMPORT_FINDING_TYPE_NONE\x10\x00\x12\x1f\n\x1bIMPORT_FINDING_TYPE_DELETED\x10\x01\x12$\n IMPORT_FINDING_TYPE_INVALID_JSON\x10\x02\x12\'\n#IMPORT_FINDING_TYPE_INVALID_PACKAGE\x10\x03\x12$\n IMPORT_FINDING_TYPE_INVALID_PURL\x10\x04\x12\'\n#IMPORT_FINDING_TYPE_INVALID_VERSION\x10\x05\x12&\n\"IMPORT_FINDING_TYPE_INVALID_COMMIT\x10\x06\x12%\n!IMPORT_FINDING_TYPE_INVALID_RANGE\x10\x07\x12\'\n#IMPORT_FINDING_TYPE_BAD_ALIASED_CVE\x10\x08\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'osv.importfinding_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_IMPORTFINDINGTYPE']._serialized_start=256 + _globals['_IMPORTFINDINGTYPE']._serialized_end=616 + _globals['_IMPORTFINDING']._serialized_start=66 + _globals['_IMPORTFINDING']._serialized_end=253 +# @@protoc_insertion_point(module_scope) diff --git a/osv/importfinding_pb2.pyi b/osv/importfinding_pb2.pyi new file mode 100644 index 00000000000..cfbb3df93ae --- /dev/null +++ b/osv/importfinding_pb2.pyi @@ -0,0 +1,99 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import builtins +import collections.abc +import google.protobuf.descriptor +import google.protobuf.internal.containers +import google.protobuf.internal.enum_type_wrapper +import google.protobuf.message +import google.protobuf.timestamp_pb2 +import sys +import typing + +if sys.version_info >= (3, 10): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class _ImportFindingType: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _ImportFindingTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_ImportFindingType.ValueType], builtins.type): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + IMPORT_FINDING_TYPE_NONE: _ImportFindingType.ValueType # 0 + IMPORT_FINDING_TYPE_DELETED: _ImportFindingType.ValueType # 1 + IMPORT_FINDING_TYPE_INVALID_JSON: _ImportFindingType.ValueType # 2 + IMPORT_FINDING_TYPE_INVALID_PACKAGE: _ImportFindingType.ValueType # 3 + IMPORT_FINDING_TYPE_INVALID_PURL: _ImportFindingType.ValueType # 4 + IMPORT_FINDING_TYPE_INVALID_VERSION: _ImportFindingType.ValueType # 5 + IMPORT_FINDING_TYPE_INVALID_COMMIT: _ImportFindingType.ValueType # 6 + IMPORT_FINDING_TYPE_INVALID_RANGE: _ImportFindingType.ValueType # 7 + IMPORT_FINDING_TYPE_BAD_ALIASED_CVE: _ImportFindingType.ValueType # 8 + +class ImportFindingType(_ImportFindingType, metaclass=_ImportFindingTypeEnumTypeWrapper): ... + +IMPORT_FINDING_TYPE_NONE: ImportFindingType.ValueType # 0 +IMPORT_FINDING_TYPE_DELETED: ImportFindingType.ValueType # 1 +IMPORT_FINDING_TYPE_INVALID_JSON: ImportFindingType.ValueType # 2 +IMPORT_FINDING_TYPE_INVALID_PACKAGE: ImportFindingType.ValueType # 3 +IMPORT_FINDING_TYPE_INVALID_PURL: ImportFindingType.ValueType # 4 +IMPORT_FINDING_TYPE_INVALID_VERSION: ImportFindingType.ValueType # 5 +IMPORT_FINDING_TYPE_INVALID_COMMIT: ImportFindingType.ValueType # 6 +IMPORT_FINDING_TYPE_INVALID_RANGE: ImportFindingType.ValueType # 7 +IMPORT_FINDING_TYPE_BAD_ALIASED_CVE: ImportFindingType.ValueType # 8 +global___ImportFindingType = ImportFindingType + +@typing.final +class ImportFinding(google.protobuf.message.Message): + """An importfinding entry. + The protobuf representation is *NOT* stable and only used for implementing + the JSON based API. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + BUG_ID_FIELD_NUMBER: builtins.int + SOURCE_FIELD_NUMBER: builtins.int + FINDINGS_FIELD_NUMBER: builtins.int + FIRST_SEEN_FIELD_NUMBER: builtins.int + LAST_ATTEMPT_FIELD_NUMBER: builtins.int + bug_id: builtins.str + source: builtins.str + @property + def findings(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[global___ImportFindingType.ValueType]: ... + @property + def first_seen(self) -> google.protobuf.timestamp_pb2.Timestamp: ... + @property + def last_attempt(self) -> google.protobuf.timestamp_pb2.Timestamp: ... + def __init__( + self, + *, + bug_id: builtins.str = ..., + source: builtins.str = ..., + findings: collections.abc.Iterable[global___ImportFindingType.ValueType] | None = ..., + first_seen: google.protobuf.timestamp_pb2.Timestamp | None = ..., + last_attempt: google.protobuf.timestamp_pb2.Timestamp | None = ..., + ) -> None: ... + def HasField(self, field_name: typing.Literal["first_seen", b"first_seen", "last_attempt", b"last_attempt"]) -> builtins.bool: ... + def ClearField(self, field_name: typing.Literal["bug_id", b"bug_id", "findings", b"findings", "first_seen", b"first_seen", "last_attempt", b"last_attempt", "source", b"source"]) -> None: ... + +global___ImportFinding = ImportFinding diff --git a/osv/models.py b/osv/models.py index 8dd7eabe81a..631b5dd64bb 100644 --- a/osv/models.py +++ b/osv/models.py @@ -25,6 +25,7 @@ from google.cloud import ndb from google.protobuf import json_format from google.protobuf import timestamp_pb2 +from osv import importfinding_pb2 # pylint: disable=relative-beyond-top-level from . import bug @@ -930,6 +931,16 @@ def _pre_put_hook(self): # pylint: disable=arguments-differ if not self.key: # pylint: disable=access-member-before-definition self.key = ndb.Key(ImportFinding, self.bug_id) + def to_proto(self): + """Converts to ImportFinding proto.""" + return importfinding_pb2.ImportFinding( + bug_id=self.bug_id, + source=self.source, + findings=self.findings, # type: ignore + first_seen=self.first_seen.timestamp_pb(), #type: ignore + last_attempt=self.last_attempt.timestamp_pb(), #type: ignore + ) + def get_source_repository(source_name: str) -> SourceRepository: """Get source repository.""" diff --git a/osv/vulnerability_pb2.py b/osv/vulnerability_pb2.py index 9916ad35fe8..62ecf4bfa07 100644 --- a/osv/vulnerability_pb2.py +++ b/osv/vulnerability_pb2.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: osv/vulnerability.proto -# Protobuf Python Version: 4.25.1 +# Protobuf Python Version: 5.26.1 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -21,8 +21,8 @@ _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'osv.vulnerability_pb2', _globals) -if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None _globals['_COMMIT']._serialized_start=95 _globals['_COMMIT']._serialized_end=216 _globals['_COMMIT_REPOTYPE']._serialized_start=180