From adff65e7992f84bf94d57a9f321cda9cbbb474c6 Mon Sep 17 00:00:00 2001
From: Dongge Liu <donggeliu@google.com>
Date: Fri, 16 Aug 2024 18:46:28 +1000
Subject: [PATCH] Disable incompatible benchmarks for cloud experiments (#2030)

Temporarily disable benchmark `stb_stbi_read_fuzzer` and
`openh264_decoder_fuzzer`from cloud experiments, becaue they are
[proven](https://github.com/google/fuzzbench/pull/2023#issuecomment-2284069911)
to be incompatible in cloud build/run environment.

@addisoncrump kindly confirmed that they [work in local
experiments](https://github.com/google/fuzzbench/pull/2023#issuecomment-2286187880).
---
 experiment/run_experiment.py   | 14 +++++++++++++-
 service/experiment-config.yaml |  6 +++---
 service/gcbrun_experiment.py   |  1 +
 3 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/experiment/run_experiment.py b/experiment/run_experiment.py
index e7fa3eaba..6475eeb34 100644
--- a/experiment/run_experiment.py
+++ b/experiment/run_experiment.py
@@ -298,6 +298,18 @@ def get_git_hash(allow_uncommitted_changes):
         return ''
 
 
+def _filter_incompatible_benchmarks(config: dict,
+                                    benchmarks: List[str]) -> List[str]:
+    """Removes benchmarks that are incompatible with build/run environment."""
+    if config['local_experiment']:
+        return benchmarks
+    if 'openh264_decoder_fuzzer' in benchmarks:
+        benchmarks.remove('openh264_decoder_fuzzer')
+    if 'stb_stbi_read_fuzzer' in benchmarks:
+        benchmarks.remove('stb_stbi_read_fuzzer')
+    return benchmarks
+
+
 def start_experiment(  # pylint: disable=too-many-arguments
         experiment_name: str,
         config_filename: str,
@@ -322,7 +334,7 @@ def start_experiment(  # pylint: disable=too-many-arguments
 
     config = read_and_validate_experiment_config(config_filename)
     config['fuzzers'] = fuzzers
-    config['benchmarks'] = benchmarks
+    config['benchmarks'] = _filter_incompatible_benchmarks(config, benchmarks)
     config['experiment'] = experiment_name
     config['git_hash'] = get_git_hash(allow_uncommitted_changes)
     config['no_seeds'] = no_seeds
diff --git a/service/experiment-config.yaml b/service/experiment-config.yaml
index b9acb09f8..aadf084a6 100644
--- a/service/experiment-config.yaml
+++ b/service/experiment-config.yaml
@@ -2,8 +2,8 @@
 # Unless you are a fuzzbench maintainer running this service, this
 # will not work with your setup.
 
-trials: 20
-max_total_time: 82800  # 23 hours, the default time for preemptible experiments.
+trials: 2
+max_total_time: 2100
 cloud_project: fuzzbench
 docker_registry: gcr.io/fuzzbench
 cloud_compute_zone: us-central1-c
@@ -15,7 +15,7 @@ preemptible_runners: true
 
 # This experiment should generate a report that is combined with other public
 # "production" experiments.
-merge_with_nonprivate: true
+merge_with_nonprivate: false
 
 # This experiment should be merged with other reports in later experiments.
 private: false
diff --git a/service/gcbrun_experiment.py b/service/gcbrun_experiment.py
index f19ab493d..fa0313ec8 100644
--- a/service/gcbrun_experiment.py
+++ b/service/gcbrun_experiment.py
@@ -20,6 +20,7 @@
 import logging
 import os
 import sys
+# dummy
 
 # pytype: disable=import-error
 import github  # pylint: disable=import-error