Merge branch 'google:master' into dependabot

.gitignore

@@ -10,3 +10,4 @@ build/lib
 .history/*
 **/.history/*
 */datastore-helper/
+hurl-scripts/

deployment/clouddeploy/gke-workers/environments/oss-vdb-test/staging-api-test.yaml

@@ -3,7 +3,8 @@ kind: CronJob
 metadata:
   name: staging-api-test
 spec:
-  schedule: "50 9 * * *"
+  timeZone: Australia/Sydney
+  schedule: "0 9 * * *"
   concurrencyPolicy: Forbid
   jobTemplate:
     spec:
@@ -19,8 +20,8 @@ spec:
             resources:
               requests:
                 cpu: 1.5
-                memory: "4G"
+                memory: "10G"
               limits:
                 cpu: 2
-                memory: "10G"
+                memory: "30G"
           restartPolicy: Never

deployment/clouddeploy/osv-api/run.yaml

@@ -12,7 +12,8 @@ spec:
       - image: osv-server
         resources:
           limits:
-            memory: 4Gi
+            cpu: 2
+            memory: 8Gi
         startupProbe:
           grpc:
             service: osv.v1.OSV

deployment/clouddeploy/osv-website/run-prod.yaml

@@ -22,3 +22,7 @@ spec:
         livenessProbe:
           httpGet:
             path: /_ah/warmup
+        resources:
+          limits:
+            cpu: 1
+            memory: 4Gi

deployment/clouddeploy/osv-website/run-staging.yaml

@@ -22,3 +22,7 @@ spec:
         livenessProbe:
           httpGet:
             path: /_ah/warmup
+        resources:
+          limits:
+            cpu: 1
+            memory: 4Gi

docker/deployment/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04@sha256:58b87898e82351c6cf9cf5b9f3c20257bb9e2dcf33af051e12ce532d7f94e3fe
+FROM ubuntu:22.04@sha256:0e5e4a57c2499249aafc3b40fcd541e9a456aab7296681a3994d631587203f97
 
 RUN apt-get update && \
     apt-get install -y curl jq

docker/exporter/exporter.py

@@ -47,9 +47,13 @@ def run(self):
     """Run exporter."""
     if self._ecosystem == "list":
       query = osv.Bug.query(projection=[osv.Bug.ecosystem], distinct=True)
-      #TODO(gongh@): remove all ecosystem releases from ecosystem.txt
-      # after notifying users.
-      ecosystems = [bug.ecosystem[0] for bug in query if bug.ecosystem]
+      # Filter out ecosystems that contain a colon,
+      # as these represent Linux distro releases.
+      ecosystems = [
+          bug.ecosystem[0]
+          for bug in query
+          if bug.ecosystem and ':' not in bug.ecosystem[0]
+      ]
       self._export_ecosystem_list_to_bucket(ecosystems, self._work_dir)
     else:
       self._export_ecosystem_to_bucket(self._ecosystem, self._work_dir)

docker/indexer/Dockerfile

@@ -24,7 +24,7 @@ RUN go mod download
 COPY ./ /build
 RUN CGO_ENABLED=0 ./build.sh
 
-FROM gcr.io/distroless/base-debian12@sha256:6ae5fe659f28c6afe9cc2903aebc78a5c6ad3aaa3d9d0369760ac6aaea2529c8
+FROM gcr.io/distroless/base-debian12@sha256:8fe31fb9d159141d9c3ff99f1fd287239d89d97ea95fea1f08f82ea5f2b544da
 COPY --from=GO_BUILD build/indexer /indexer
 ENTRYPOINT ["/indexer"]
 CMD ["--help"]

docker/indexer/go.mod

@@ -3,21 +3,21 @@ module github.com/google/osv.dev/docker/indexer
 go 1.23.1
 
 require (
-	cloud.google.com/go/datastore v1.19.0
-	cloud.google.com/go/pubsub v1.44.0
-	cloud.google.com/go/storage v1.44.0
+	cloud.google.com/go/datastore v1.20.0
+	cloud.google.com/go/pubsub v1.45.1
+	cloud.google.com/go/storage v1.45.0
 	github.com/go-git/go-git/v5 v5.12.0
 	github.com/golang/glog v1.2.2
 	github.com/google/go-cmp v0.6.0
 	golang.org/x/sync v0.8.0
-	google.golang.org/api v0.201.0
+	google.golang.org/api v0.203.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	cel.dev/expr v0.16.1 // indirect
 	cloud.google.com/go v0.116.0 // indirect
-	cloud.google.com/go/auth v0.9.8 // indirect
+	cloud.google.com/go/auth v0.9.9 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
 	cloud.google.com/go/compute/metadata v0.5.2 // indirect
 	cloud.google.com/go/iam v1.2.1 // indirect
@@ -70,9 +70,9 @@ require (
 	golang.org/x/text v0.19.0 // indirect
 	golang.org/x/time v0.7.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
-	google.golang.org/genproto v0.0.0-20241007155032-5fefd90f89a9 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 // indirect
+	google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect
 	google.golang.org/grpc v1.67.1 // indirect
 	google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a // indirect
 	google.golang.org/protobuf v1.35.1 // indirect

docker/indexer/go.sum

@@ -3,14 +3,14 @@ cel.dev/expr v0.16.1/go.mod h1:AsGA5zb3WruAEQeQng1RZdGEXmBj0jvMWh6l5SnNuC8=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE=
 cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U=
-cloud.google.com/go/auth v0.9.8 h1:+CSJ0Gw9iVeSENVCKJoLHhdUykDgXSc4Qn+gu2BRtR8=
-cloud.google.com/go/auth v0.9.8/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI=
+cloud.google.com/go/auth v0.9.9 h1:BmtbpNQozo8ZwW2t7QJjnrQtdganSdmqeIBxHxNkEZQ=
+cloud.google.com/go/auth v0.9.9/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI=
 cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY=
 cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc=
 cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo=
 cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k=
-cloud.google.com/go/datastore v1.19.0 h1:p5H3bUQltOa26GcMRAxPoNwoqGkq5v8ftx9/ZBB35MI=
-cloud.google.com/go/datastore v1.19.0/go.mod h1:KGzkszuj87VT8tJe67GuB+qLolfsOt6bZq/KFuWaahc=
+cloud.google.com/go/datastore v1.20.0 h1:NNpXoyEqIJmZFc0ACcwBEaXnmscUpcG4NkKnbCePmiM=
+cloud.google.com/go/datastore v1.20.0/go.mod h1:uFo3e+aEpRfHgtp5pp0+6M0o147KoPaYNaPAKpfh8Ew=
 cloud.google.com/go/iam v1.2.1 h1:QFct02HRb7H12J/3utj0qf5tobFh9V4vR6h9eX5EBRU=
 cloud.google.com/go/iam v1.2.1/go.mod h1:3VUIJDPpwT6p/amXRC5GY8fCCh70lxPygguVtI0Z4/g=
 cloud.google.com/go/kms v1.20.0 h1:uKUvjGqbBlI96xGE669hcVnEMw1Px/Mvfa62dhM5UrY=
@@ -21,10 +21,10 @@ cloud.google.com/go/longrunning v0.6.1 h1:lOLTFxYpr8hcRtcwWir5ITh1PAKUD/sG2lKrTS
 cloud.google.com/go/longrunning v0.6.1/go.mod h1:nHISoOZpBcmlwbJmiVk5oDRz0qG/ZxPynEGs1iZ79s0=
 cloud.google.com/go/monitoring v1.21.1 h1:zWtbIoBMnU5LP9A/fz8LmWMGHpk4skdfeiaa66QdFGc=
 cloud.google.com/go/monitoring v1.21.1/go.mod h1:Rj++LKrlht9uBi8+Eb530dIrzG/cU/lB8mt+lbeFK1c=
-cloud.google.com/go/pubsub v1.44.0 h1:pLaMJVDTlnUDIKT5L0k53YyLszfBbGoUBo/IqDK/fEI=
-cloud.google.com/go/pubsub v1.44.0/go.mod h1:BD4a/kmE8OePyHoa1qAHEw1rMzXX+Pc8Se54T/8mc3I=
-cloud.google.com/go/storage v1.44.0 h1:abBzXf4UJKMmQ04xxJf9dYM/fNl24KHoTuBjyJDX2AI=
-cloud.google.com/go/storage v1.44.0/go.mod h1:wpPblkIuMP5jCB/E48Pz9zIo2S/zD8g+ITmxKkPCITE=
+cloud.google.com/go/pubsub v1.45.1 h1:ZC/UzYcrmK12THWn1P72z+Pnp2vu/zCZRXyhAfP1hJY=
+cloud.google.com/go/pubsub v1.45.1/go.mod h1:3bn7fTmzZFwaUjllitv1WlsNMkqBgGUb3UdMhI54eCc=
+cloud.google.com/go/storage v1.45.0 h1:5av0QcIVj77t+44mV4gffFC/LscFRUhto6UBMB5SimM=
+cloud.google.com/go/storage v1.45.0/go.mod h1:wpPblkIuMP5jCB/E48Pz9zIo2S/zD8g+ITmxKkPCITE=
 cloud.google.com/go/trace v1.11.1 h1:UNqdP+HYYtnm6lb91aNA5JQ0X14GnxkABGlfz2PzPew=
 cloud.google.com/go/trace v1.11.1/go.mod h1:IQKNQuBzH72EGaXEodKlNJrWykGZxet2zgjtS60OtjA=
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
@@ -282,19 +282,19 @@ golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJ
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.201.0 h1:+7AD9JNM3tREtawRMu8sOjSbb8VYcYXJG/2eEOmfDu0=
-google.golang.org/api v0.201.0/go.mod h1:HVY0FCHVs89xIW9fzf/pBvOEm+OolHa86G/txFezyq4=
+google.golang.org/api v0.203.0 h1:SrEeuwU3S11Wlscsn+LA1kb/Y5xT8uggJSkIhD08NAU=
+google.golang.org/api v0.203.0/go.mod h1:BuOVyCSYEPwJb3npWvDnNmFI92f3GeRnHNkETneT3SI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20241007155032-5fefd90f89a9 h1:nFS3IivktIU5Mk6KQa+v6RKkHUpdQpphqGNLxqNnbEk=
-google.golang.org/genproto v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:tEzYTYZxbmVNOu0OAFH9HzdJtLn6h4Aj89zzlBCdHms=
-google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f h1:jTm13A2itBi3La6yTGqn8bVSrc3ZZ1r8ENHlIXBfnRA=
-google.golang.org/genproto/googleapis/api v0.0.0-20240930140551-af27646dc61f/go.mod h1:CLGoBuH1VHxAUXVPP8FfPwPEVJB6lz3URE5mY2SuayE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 h1:QCqS/PdaHTSWGvupk2F/ehwHtGc0/GYkT+3GAcR1CCc=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
+google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 h1:Df6WuGvthPzc+JiQ/G+m+sNX24kc0aTBqoDN/0yyykE=
+google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53/go.mod h1:fheguH3Am2dGp1LfXkrvwqC/KlFq8F0nLq3LryOMrrE=
+google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 h1:T6rh4haD3GVYsgEfWExoCZA2o2FmbNyKpTuAxbEFPTg=
+google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:wp2WsuBYj6j8wUdo3ToZsdxxixbvQNAHqVJrTgi5E5M=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=

docker/staging_api_test/Dockerfile

@@ -16,11 +16,11 @@ FROM gcr.io/oss-vdb/worker
 
 WORKDIR /staging_api_test
 
-COPY retrieve_bugs_from_db.py perform_api_calls.py run.sh ./
+COPY retrieve_bugs_from_bucket.py perform_api_calls.py run.sh ./
 
 # Add aiohttp lib
 RUN cd /env/docker/worker && POETRY_VIRTUALENVS_CREATE=false poetry add aiohttp
 
-RUN chmod 755 retrieve_bugs_from_db.py perform_api_calls.py run.sh
+RUN chmod 755 retrieve_bugs_from_bucket.py perform_api_calls.py run.sh
 
 ENTRYPOINT ["./run.sh"]

docker/staging_api_test/perform_api_calls.py

@@ -21,7 +21,6 @@
 import asyncio
 import os
 import random
-import sys
 import time
 import json
 
@@ -36,7 +35,7 @@
 BUG_DIR = './all_bugs'
 
 # Total run time in seconds
-TOTAL_RUNTIME = 3600
+TOTAL_RUNTIME = 3600 * 5  # 5 hours
 # Execute all pending batch size requests within the specified time interval.
 FREQUENCY_IN_SECONDS = 1
 
@@ -59,21 +58,12 @@ class SimpleBug:
   def __init__(self, bug_dict: dict):
     self.db_id = bug_dict['db_id']
     # If the package/ecosystem/version value is None, then add a fake value in.
-    if not bug_dict['project']:
-      self.packages = 'foo'
-    else:
-      self.packages = list(bug_dict['project'])
-    self.purl = bug_dict['purl']
-    if not bug_dict['ecosystem']:
-      self.ecosystems = 'foo'
-    else:
-      self.ecosystems = list(bug_dict['ecosystem'])
+    self.package = bug_dict.get('project', 'foo')
+    self.ecosystem = bug_dict.get('ecosystem', 'foo')
 
     # Use the `affected fuzzy` value as the query version.
     # If no 'affected fuzzy' is present, assign a default value.
-    self.affected_fuzzy = bug_dict['affected_fuzzy']
-    if not self.affected_fuzzy:
-      self.affected_fuzzy = '1.0.0'
+    self.affected_fuzzy = bug_dict.get('affected_fuzzy', '1.0.0')
 
 
 def read_from_json(filename: str, ecosystem_map: defaultdict, bug_map: dict,
@@ -102,10 +92,8 @@ def read_from_json(filename: str, ecosystem_map: defaultdict, bug_map: dict,
     json_file = json.load(f)
     for bug_data in json_file:
       bug = SimpleBug(bug_data)
-      for ecosystem in bug.ecosystems:
-        ecosystem_map[ecosystem].add(bug.db_id)
-      for package in bug.packages:
-        package_map[package].add(bug.db_id)
+      ecosystem_map[bug.ecosystem].add(bug.db_id)
+      package_map[bug.package].add(bug.db_id)
       bug_map[bug.db_id] = bug
 
 
@@ -240,9 +228,12 @@ def build_package_payload(request_id: str, bug_map: dict) -> dict[str, any]:
     '"package": {"name": "mruby","ecosystem": "OSS-Fuzz"}}'
   """
 
-  package = random.choice(bug_map[request_id].packages)
-  ecosystem = random.choice(bug_map[request_id].ecosystems)
-  return {"package": {"name": package, "ecosystem": ecosystem}}
+  return {
+      "package": {
+          "name": bug_map[request_id].package,
+          "ecosystem": bug_map[request_id].ecosystem
+      }
+  }
 
 
 def build_version_payload(request_id: str, bug_map: dict) -> dict:
@@ -259,13 +250,12 @@ def build_version_payload(request_id: str, bug_map: dict) -> dict:
     '{"package": {
       "name": "mruby","ecosystem": "OSS-Fuzz"}, "version": "2.1.2rc"}'
   """
-  package = random.choice(bug_map[request_id].packages)
-  ecosystem = random.choice(bug_map[request_id].ecosystems)
+
   return {
       "version": bug_map[request_id].affected_fuzzy,
       "package": {
-          "name": package,
-          "ecosystem": ecosystem
+          "name": bug_map[request_id].package,
+          "ecosystem": bug_map[request_id].ecosystem
       }
   }
 
@@ -427,10 +417,12 @@ async def send_batch_requests(request_ids: list, bug_map: dict,
 
 async def main() -> None:
   osv.logs.setup_gcp_logging('staging-test')
-  seed = random.randrange(sys.maxsize)
-  # The seed value can be replaced for debugging
-  random.seed(seed)
+  seed = random.randrange(1000)
   logging.info('Random seed %d', seed)
+  # Log the seed value. This allows us to use the same seed later
+  # and reproduce this random result for debugging purposes.
+  random.seed(seed)
+
   # The `ecosystem_map` can be used to filter our queries for a
   # specific ecosystem.
   ecosystem_map, bug_map, package_map = load_all_bugs()