From 928308e2c5d467390e64b1fce819b7863b7193ba Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Tue, 23 Nov 2021 10:47:16 +0100 Subject: [PATCH 01/31] Small refactoring for fm provider --- varats-core/varats/provider/feature/feature_model_provider.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/varats-core/varats/provider/feature/feature_model_provider.py b/varats-core/varats/provider/feature/feature_model_provider.py index 51446fc10..bdfb1afd1 100644 --- a/varats-core/varats/provider/feature/feature_model_provider.py +++ b/varats-core/varats/provider/feature/feature_model_provider.py @@ -57,10 +57,12 @@ def get_feature_model_path( """ project_name = self.project.NAME.lower() + fully_qualified_fm_name = "FeatureModel" + for project_dir in self._get_feature_model_repository_path().iterdir(): if project_dir.name.lower() == project_name: for poss_fm_file in project_dir.iterdir(): - if poss_fm_file.stem == "FeatureModel": + if poss_fm_file.stem == fully_qualified_fm_name: return poss_fm_file return None From 3a70322efee6d9c7adbae135b7623f29ac6d5548 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Tue, 23 Nov 2021 10:47:59 +0100 Subject: [PATCH 02/31] Adds project for FeaturePerfCS repo --- .../perf_tests/feature_perf_cs_collection.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 varats/varats/projects/perf_tests/feature_perf_cs_collection.py diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py new file mode 100644 index 000000000..60737189a --- /dev/null +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -0,0 +1,68 @@ +"""Project file for the feature performance case study collection.""" +import typing as tp + +import benchbuild as bb +from benchbuild.utils.cmd import make, cmake, mkdir +from benchbuild.utils.settings import get_number_of_jobs +from plumbum import local + +from varats.paper_mgmt.paper_config import project_filter_generator +from varats.project.project_domain import ProjectDomains +from varats.project.project_util import ( + wrap_paths_to_binaries, + ProjectBinaryWrapper, + BinaryType, + verify_binaries, +) +from varats.project.varats_project import VProject +from varats.utils.git_util import ShortCommitHash +from varats.utils.settings import bb_cfg + + +class FeaturePerfCSCollection(VProject): + """Test project for feature performance case studies.""" + + NAME = 'FeaturePerfCSCollection' + GROUP = 'perf_tests' + DOMAIN = ProjectDomains.TEST + + SOURCE = [ + bb.source.Git( + remote="https://github.com/se-sic/FeaturePerfCSCollection.git", + local="FeaturePerfCSCollection", + refspec="origin/HEAD", + limit=None, + shallow=False, + version_filter=project_filter_generator("FeaturePerfCSCollection") + ) + ] + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash # pylint: disable=W0613 + ) -> tp.List[ProjectBinaryWrapper]: + return wrap_paths_to_binaries([ + ("build/bin/SingleLocalSimple", BinaryType.EXECUTABLE), + ("build/bin/SingleLocalMultipleRegions", BinaryType.EXECUTABLE) + ]) + + def run_tests(self) -> None: + pass + + def compile(self) -> None: + """Compile the project.""" + feature_perf_source = local.path(self.source_of(self.primary_source)) + + cc_compiler = bb.compiler.cc(self) + cxx_compiler = bb.compiler.cxx(self) + + mkdir("-p", feature_perf_source / "build") + + with local.cwd(feature_perf_source / "build"): + with local.env(CC=str(cc_compiler), CXX=str(cxx_compiler)): + bb.watch(cmake)("-G", "Unix Makefiles", "..") + + bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) + + with local.cwd(feature_perf_source): + verify_binaries(self) From 724d158a8e95e6a9d8a09ae4eacfdde17b48e73c Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Tue, 23 Nov 2021 10:49:18 +0100 Subject: [PATCH 03/31] Cleans up compiler handling for lrzip --- varats/varats/projects/c_projects/lrzip.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/varats/varats/projects/c_projects/lrzip.py b/varats/varats/projects/c_projects/lrzip.py index 43a923e70..14aac864d 100644 --- a/varats/varats/projects/c_projects/lrzip.py +++ b/varats/varats/projects/c_projects/lrzip.py @@ -58,9 +58,11 @@ def compile(self) -> None: self.cflags += ["-fPIC"] - clang = bb.compiler.cc(self) + cc_compiler = bb.compiler.cc(self) + cxx_compiler = bb.compiler.cxx(self) + with local.cwd(lrzip_source): - with local.env(CC=str(clang)): + with local.env(CC=str(cc_compiler), CXX=str(cxx_compiler)): bb.watch(local["./autogen.sh"])() bb.watch(local["./configure"])() bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) From aa3987198744c86a085c8752e08b91046baa12c4 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Tue, 23 Nov 2021 10:56:19 +0100 Subject: [PATCH 04/31] Adds initial draft of feature perf runner --- .../experiments/vara/feature_perf_runner.py | 138 ++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 varats/varats/experiments/vara/feature_perf_runner.py diff --git a/varats/varats/experiments/vara/feature_perf_runner.py b/varats/varats/experiments/vara/feature_perf_runner.py new file mode 100644 index 000000000..856a48d41 --- /dev/null +++ b/varats/varats/experiments/vara/feature_perf_runner.py @@ -0,0 +1,138 @@ +import os +import typing as tp + +from benchbuild import Project +from benchbuild.extensions import compiler, run, time +from benchbuild.utils import actions +from plumbum import local + +from varats.data.reports.empty_report import EmptyReport as EMPTY +from varats.experiment.experiment_util import ( + exec_func_with_pe_error_handler, + ExperimentHandle, + get_varats_result_folder, + wrap_unlimit_stack_size, + create_default_compiler_error_handler, + create_default_analysis_failure_handler, + VersionExperiment, + get_default_compile_error_wrapped, + PEErrorHandler, +) +from varats.experiment.wllvm import ( + get_cached_bc_file_path, + BCFileExtensions, + RunWLLVM, + get_bc_cache_actions, +) +from varats.project.project_util import ProjectBinaryWrapper, BinaryType +from varats.provider.feature.feature_model_provider import FeatureModelProvider +from varats.report.report import ReportSpecification +from varats.report.report import FileStatusExtension as FSE + + +class ExecAndTraceBinary(actions.Step): # type: ignore + """Executes the specified binaries of the project, in specific + configurations, against one or multiple workloads.""" + + NAME = "ExecBinary" + DESCRIPTION = "fobar" # TODO: fix + + def __init__(self, project: Project, experiment_handle: ExperimentHandle): + super().__init__(obj=project, action_fn=self.run_perf_tracing) + self.__experiment_handle = experiment_handle + + def run_perf_tracing(self) -> actions.StepResult: + """Execute the specified binaries of the project, in specific + configurations, against one or multiple workloads.""" + project: Project = self.obj + + print(f"PWD {os.getcwd()}") + + vara_result_folder = get_varats_result_folder(project) + for binary in project.binaries: + if binary.type != BinaryType.EXECUTABLE: + continue + + result_file = self.__experiment_handle.get_file_name( + EMPTY.shorthand(), + project_name=str(project.name), + binary_name=binary.name, + project_revision=project.version_of_primary, + project_uuid=str(project.run_uuid), + extension_type=FSE.SUCCESS + ) + + with local.cwd(local.path(project.source_of_primary)): + print(f"Currenlty at {local.path(project.source_of_primary)}") + print(f"Bin path {binary.path}") + executable = local[f"{binary.path}"] + with local.env( + VARA_TRACE_FILE=f"{vara_result_folder}/{result_file}" + ): + # TODO: figure out how to handle workloads + # executable("/home/vulder/vara-root/Selection_050.png") + + # TODO: figure out how to handle different configs + executable("--slow") + #executable() + + return actions.StepResult.OK + + +class FeaturePerfRunner(VersionExperiment, shorthand="FPR"): + """Test runner for feature performance.""" + + NAME = "RunFeaturePerf" + + REPORT_SPEC = ReportSpecification(EMPTY) + + def actions_for_project( + self, project: Project + ) -> tp.MutableSequence[actions.Step]: + """ + Returns the specified steps to run the project(s) specified in the call + in a fixed order. + + Args: + project: to analyze + """ + + fm_provider = FeatureModelProvider.create_provider_for_project(project) + if fm_provider is None: + # TODO: add log + # TODO: add exception FM not found + raise AssertionError("foo") + + fm_path = fm_provider.get_feature_model_path(project.version_of_primary) + if fm_path is None or not fm_path.exists(): + # TODO: add log + # TODO: add exception FM not found + raise AssertionError("fadf") + + # Sets FM model flags + project.cflags += [ + "-fvara-feature", f"-fvara-fm-path={fm_path.absolute()}" + ] + # Sets vara tracing flags + project.cflags += ["-fsanitize=vara", "-fvara-instr=trace_event"] + + # Add the required runtime extensions to the project(s). + project.runtime_extension = run.RuntimeExtension(project, self) \ + << time.RunWithTime() + + # Add the required compiler extensions to the project(s). + project.compiler_extension = compiler.RunCompiler(project, self) \ + << run.WithTimeout() + + # Add own error handler to compile step. + project.compile = get_default_compile_error_wrapped( + self.get_handle(), project, EMPTY + ) + + analysis_actions = [] + + analysis_actions.append(actions.Compile(project)) + analysis_actions.append(ExecAndTraceBinary(project, self.get_handle())) + analysis_actions.append(actions.Clean(project)) + + return analysis_actions From 9ff98d2d81f43c80faf94fef4ef13134180e0822 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Tue, 23 Nov 2021 11:12:40 +0100 Subject: [PATCH 05/31] Adds exception should a FM not be found --- .../provider/feature/feature_model_provider.py | 13 +++++++++++++ .../varats/experiments/vara/feature_perf_runner.py | 13 ++++++------- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/varats-core/varats/provider/feature/feature_model_provider.py b/varats-core/varats/provider/feature/feature_model_provider.py index bdfb1afd1..961bcbba7 100644 --- a/varats-core/varats/provider/feature/feature_model_provider.py +++ b/varats-core/varats/provider/feature/feature_model_provider.py @@ -9,6 +9,19 @@ from varats.provider.provider import Provider +class FeatureModelNotFound(FileNotFoundError): + """Exception raised when the specified feature model could not be found.""" + + def __init__(self, project: Project, fm_path: tp.Optional[Path]) -> None: + err_msg = f"Could not find feature model for project {project.name}!\n" + if fm_path: + err_msg += f"No file at: {fm_path}." + else: + err_msg += "Got no feature-model path." + + super().__init__(err_msg) + + class FeatureModelProvider(Provider): """Provider for accessing project related FeatureModels.""" diff --git a/varats/varats/experiments/vara/feature_perf_runner.py b/varats/varats/experiments/vara/feature_perf_runner.py index 856a48d41..c8aa7bb86 100644 --- a/varats/varats/experiments/vara/feature_perf_runner.py +++ b/varats/varats/experiments/vara/feature_perf_runner.py @@ -25,7 +25,10 @@ get_bc_cache_actions, ) from varats.project.project_util import ProjectBinaryWrapper, BinaryType -from varats.provider.feature.feature_model_provider import FeatureModelProvider +from varats.provider.feature.feature_model_provider import ( + FeatureModelProvider, + FeatureModelNotFound, +) from varats.report.report import ReportSpecification from varats.report.report import FileStatusExtension as FSE @@ -99,15 +102,11 @@ def actions_for_project( fm_provider = FeatureModelProvider.create_provider_for_project(project) if fm_provider is None: - # TODO: add log - # TODO: add exception FM not found - raise AssertionError("foo") + raise Exception("Could not get FeatureModelProvider!") fm_path = fm_provider.get_feature_model_path(project.version_of_primary) if fm_path is None or not fm_path.exists(): - # TODO: add log - # TODO: add exception FM not found - raise AssertionError("fadf") + raise FeatureModelNotFound(project, fm_path) # Sets FM model flags project.cflags += [ From 73941080c486f3a90b2ec013fd8423acad72fd94 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Wed, 24 Nov 2021 08:58:52 +0100 Subject: [PATCH 06/31] Implements core report discovery --- varats-core/varats/report/__init__.py | 16 ++++++++++++++++ varats/varats/data/discover_reports.py | 2 ++ varats/varats/data/reports/__init__.py | 2 +- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/varats-core/varats/report/__init__.py b/varats-core/varats/report/__init__.py index e69de29bb..90ee7768c 100644 --- a/varats-core/varats/report/__init__.py +++ b/varats-core/varats/report/__init__.py @@ -0,0 +1,16 @@ +"""Auto discover all core reports in subfolders.""" + +import importlib +import pkgutil + + +def discover() -> None: + """Auto import all core varats reports.""" + __all__ = [] + for _, module_name, _ in pkgutil.walk_packages( + __path__, # type: ignore + 'varats.report.' + ): + __all__.append(module_name) + _module = importlib.import_module(module_name) + globals()[module_name] = _module diff --git a/varats/varats/data/discover_reports.py b/varats/varats/data/discover_reports.py index b0ae0ab70..90dfdbbd9 100644 --- a/varats/varats/data/discover_reports.py +++ b/varats/varats/data/discover_reports.py @@ -1,8 +1,10 @@ """This modules handles auto discovering of reports from the tool suite.""" +from varats import report as __CORE_REPORTS__ from varats.data import reports as __REPORTS__ def initialize_reports() -> None: # Discover and initialize all Reports __REPORTS__.discover() + __CORE_REPORTS__.discover() diff --git a/varats/varats/data/reports/__init__.py b/varats/varats/data/reports/__init__.py index 44e4e52d4..b583d76a0 100644 --- a/varats/varats/data/reports/__init__.py +++ b/varats/varats/data/reports/__init__.py @@ -5,7 +5,7 @@ def discover() -> None: - """Auto import all BenchBuild projects.""" + """Auto import all varats reports.""" __all__ = [] for _, module_name, _ in pkgutil.walk_packages( __path__, # type: ignore From 7e87e2ae4599e542d56b6d0b433e6391b4021eae Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Wed, 24 Nov 2021 17:38:30 +0100 Subject: [PATCH 07/31] Small clean up --- tests/data/test_report.py | 11 ++++++++++- varats/varats/tools/bb_config.py | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/data/test_report.py b/tests/data/test_report.py index 288674ae4..b3d4667e6 100644 --- a/tests/data/test_report.py +++ b/tests/data/test_report.py @@ -19,7 +19,7 @@ class TestFileStatusExtension(unittest.TestCase): """Test basic FileStatusExtension functionality.""" def test_status_extension(self): - """""" + """Tests if we convert a FSE to it's string representation.""" self.assertEqual( FileStatusExtension.SUCCESS.get_status_extension(), "success" ) @@ -62,6 +62,11 @@ def test_wrong_status_lookup(self): class TestReportFilename(unittest.TestCase): """Test basic TestReportFilename functionality.""" + correct_UUID: str + raw_filename: str + report_filename: ReportFilename + broken_report_filename: ReportFilename + @classmethod def setUpClass(cls): """Setup file and CommitReport.""" @@ -151,6 +156,10 @@ def test_get_uuid(self): class TestBaseReport(unittest.TestCase): """Test basic BaseReport functionality.""" + success_filename_cr: str + success_filename: str + fail_filename: str + @classmethod def setUpClass(cls): """Setup report file paths.""" diff --git a/varats/varats/tools/bb_config.py b/varats/varats/tools/bb_config.py index 0edc62fca..09505971b 100644 --- a/varats/varats/tools/bb_config.py +++ b/varats/varats/tools/bb_config.py @@ -92,6 +92,7 @@ def create_new_bb_config(varats_cfg: s.Configuration) -> s.Configuration: 'varats.experiments.base.just_compile', 'varats.experiments.vara.blame_report_experiment', 'varats.experiments.vara.commit_report_experiment', + 'varats.experiments.vara.feature_perf_runner', 'varats.experiments.vara.marker_tester', 'varats.experiments.vara.blame_verifier_experiment', 'varats.experiments.vara.phasar_fta', From 2115e2b7637893e78e8a4044fffd588ffcc102cf Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Wed, 24 Nov 2021 17:39:02 +0100 Subject: [PATCH 08/31] Implements report for trace event format --- tests/report/test_tef_report.py | 235 ++++++++++++++++++ varats-core/varats/report/tef_report.py | 130 ++++++++++ .../experiments/vara/feature_perf_runner.py | 10 +- 3 files changed, 370 insertions(+), 5 deletions(-) create mode 100644 tests/report/test_tef_report.py create mode 100644 varats-core/varats/report/tef_report.py diff --git a/tests/report/test_tef_report.py b/tests/report/test_tef_report.py new file mode 100644 index 000000000..fbe2bfd17 --- /dev/null +++ b/tests/report/test_tef_report.py @@ -0,0 +1,235 @@ +"""Test TEFReport.""" + +import json +import unittest +from pathlib import Path +from unittest import mock + +from varats.report.tef_report import TEFReport, TraceEvent, TraceEventType + +TRACE_EVENT_FORMAT_OUTPUT = """{ + "traceEvents": [{ + "name": "Base", + "cat": "Feature", + "ph": "B", + "ts": 1637675320727304236, + "pid": 91098, + "tid": 91098 + }, { + "name": "Foo", + "cat": "Feature", + "ph": "B", + "ts": 1637675320727316656, + "pid": 91098, + "tid": 91098 + }, { + "name": "Foo", + "cat": "Feature", + "ph": "E", + "ts": 1637675325727410375, + "pid": 91098, + "tid": 91098 + }, { + "name": "Bar", + "cat": "Feature", + "ph": "B", + "ts": 1637675328727504858, + "pid": 91098, + "tid": 91098 + }, { + "name": "Bar", + "cat": "Feature", + "ph": "E", + "ts": 1637675331727788401, + "pid": 91098, + "tid": 91098 + }, { + "name": "Foo_2", + "cat": "Feature", + "ph": "B", + "ts": 1637675335727890982, + "pid": 91098, + "tid": 91098 + }, { + "name": "Foo_2", + "cat": "Feature", + "ph": "E", + "ts": 1637675341728002649, + "pid": 91098, + "tid": 91098 + }, { + "name": "Base", + "cat": "Feature", + "ph": "E", + "ts": 1637675341728008439, + "pid": 91098, + "tid": 91098 + } ], + "displayTimeUnit": "ns", + "stackFrames": {} +} +""" + + +class TestTraceEventType(unittest.TestCase): + """Test if we can correclty parse TraceEventTypes.""" + + def test_parse_duration_events(self) -> None: + """Test if we correctly parse duration event types.""" + self.assertEqual( + TraceEventType.parse_event_type("B"), + TraceEventType.DURATION_EVENT_BEGIN + ) + self.assertEqual( + TraceEventType.parse_event_type("E"), + TraceEventType.DURATION_EVENT_END + ) + + def test_parse_async_events(self) -> None: + """Test if we correctly parse async event types.""" + self.assertEqual( + TraceEventType.parse_event_type("b"), + TraceEventType.ASYNC_EVENT_START + ) + self.assertEqual( + TraceEventType.parse_event_type("n"), + TraceEventType.ASYNC_EVENT_INSTANT + ) + self.assertEqual( + TraceEventType.parse_event_type("e"), TraceEventType.ASYNC_EVENT_END + ) + + def test_parse_flow_events(self) -> None: + """Test if we correctly parse flow event types.""" + self.assertEqual( + TraceEventType.parse_event_type("s"), + TraceEventType.FLOW_EVENT_START + ) + self.assertEqual( + TraceEventType.parse_event_type("t"), TraceEventType.FLOW_EVENT_STEP + ) + self.assertEqual( + TraceEventType.parse_event_type("f"), TraceEventType.FLOW_EVENT_END + ) + + def test_parse_other_events(self) -> None: + """Test if we correctly parse other event types.""" + self.assertEqual( + TraceEventType.parse_event_type("X"), TraceEventType.COMPLETE_EVENT + ) + self.assertEqual( + TraceEventType.parse_event_type("i"), TraceEventType.INSTANT_EVENT + ) + self.assertEqual( + TraceEventType.parse_event_type("C"), TraceEventType.COUNTER_EVENT + ) + self.assertEqual( + TraceEventType.parse_event_type("P"), TraceEventType.SAMPLE_EVENT + ) + + def test_fail_at_wrong_event_string(self) -> None: + """Test if we fail should an event type not match.""" + self.assertRaises(LookupError, TraceEventType.parse_event_type, "42") + self.assertRaises(LookupError, TraceEventType.parse_event_type, "I") + self.assertRaises(LookupError, TraceEventType.parse_event_type, "D") + self.assertRaises(LookupError, TraceEventType.parse_event_type, "d") + + +SINGLE_TRACE_EVENT = """{ + "name": "Base", + "cat": "Feature", + "ph": "E", + "ts": 1637675341728008439, + "pid": 91098, + "tid": 91099 +} +""" + + +class TestTraceEvent(unittest.TestCase): + """Test if we can correctly load trace events and parse values.""" + + trace_event: TraceEvent + + @classmethod + def setUpClass(cls): + """Load trace event.""" + cls.trace_event = TraceEvent(json.loads(SINGLE_TRACE_EVENT)) + + def test_name_parsing(self): + """Test if we can correctly parse event names.""" + self.assertEqual(self.trace_event.name, "Base") + self.assertNotEqual(self.trace_event.name, "Foo") + + def test_category_parsing(self): + """Test if we can correctly parse event categories.""" + self.assertEqual(self.trace_event.category, "Feature") + self.assertNotEqual(self.trace_event.name, "Foo") + + def test_event_type_parsing(self): + """Test if we can correctly parse event type.""" + self.assertEqual( + self.trace_event.event_type, TraceEventType.DURATION_EVENT_END + ) + self.assertIsInstance(self.trace_event.event_type, TraceEventType) + self.assertNotEqual( + self.trace_event.event_type, TraceEventType.DURATION_EVENT_BEGIN + ) + + def test_timestamp_parsing(self): + """Test if we can correctly parse event timestamps.""" + self.assertEqual(self.trace_event.timestamp, 1637675341728008439) + self.assertIsInstance(self.trace_event.timestamp, int) + + self.assertNotEqual(self.trace_event.name, 1637675341728008438) + self.assertNotEqual(self.trace_event.name, 0) + + def test_pid_parsing(self): + """Test if we can correctly parse event pid.""" + self.assertEqual(self.trace_event.pid, 91098) + self.assertIsInstance(self.trace_event.pid, int) + + self.assertNotEqual(self.trace_event.name, 91099) + self.assertNotEqual(self.trace_event.name, 91097) + self.assertNotEqual(self.trace_event.name, 0) + + def test_tid_parsing(self): + """Test if we can correctly parse event tid.""" + self.assertEqual(self.trace_event.tid, 91099) + self.assertIsInstance(self.trace_event.tid, int) + + self.assertNotEqual(self.trace_event.name, 91100) + self.assertNotEqual(self.trace_event.name, 91098) + self.assertNotEqual(self.trace_event.name, 0) + + +class TestTEFReportParser(unittest.TestCase): + """Tests if the trace-event-format report can be parsed correctly.""" + + report: TEFReport + + @classmethod + def setUpClass(cls): + """Load and prepare TEF report.""" + with mock.patch( + 'builtins.open', + new=mock.mock_open(read_data=TRACE_EVENT_FORMAT_OUTPUT) + ): + cls.report = TEFReport(Path("fake_file_path")) + + def test_parse_time_unit(self) -> None: + """Test if the time unit field is correclty parsed.""" + self.assertEqual(self.report.display_time_unit, "ns") + self.assertNotEqual(self.report.display_time_unit, "ms") + + def test_parse_trace_events(self) -> None: + """Test if we correctly parse the listed trace events.""" + self.assertEqual(len(self.report.trace_events), 8) + + self.assertEqual(self.report.trace_events[0].name, "Base") + + def test_parse_stack_frames(self) -> None: + """Test if we correctly parse stack frames.""" + # Currently, not implemented so we should get an exception. + with self.assertRaises(NotImplementedError): + _ = self.report.stack_frames diff --git a/varats-core/varats/report/tef_report.py b/varats-core/varats/report/tef_report.py new file mode 100644 index 000000000..ec01e2818 --- /dev/null +++ b/varats-core/varats/report/tef_report.py @@ -0,0 +1,130 @@ +"""Report module to create and handle trace event format files, e.g., created +with chrome tracing.""" + +import json +import typing as tp +from enum import Enum +from pathlib import Path + +from varats.report.report import BaseReport + + +class TraceEventType(Enum): + """Enum to represent the different event types of trace format events, + defined by the Trace Event Format specification.""" + + value: str # pylint: disable=invalid-name + + DURATION_EVENT_BEGIN = 'B' + DURATION_EVENT_END = 'E' + COMPLETE_EVENT = 'X' + INSTANT_EVENT = 'i' + COUNTER_EVENT = 'C' + ASYNC_EVENT_START = 'b' + ASYNC_EVENT_INSTANT = 'n' + ASYNC_EVENT_END = 'e' + FLOW_EVENT_START = 's' + FLOW_EVENT_STEP = 't' + FLOW_EVENT_END = 'f' + SAMPLE_EVENT = 'P' + + @staticmethod + def parse_event_type(raw_event_type: str) -> 'TraceEventType': + """Parses a raw string that represents a trace-format even type and + converts it to the corresponding enum value.""" + for trace_event_type in TraceEventType: + if trace_event_type.value == raw_event_type: + return trace_event_type + + raise LookupError("Could not find correct trace event type") + + def __str__(self) -> str: + return self.value[0] + + +class TraceEvent(): + """Represents a trace event that was captured during the analysis of a + target program.""" + + def __init__(self, json_trace_event: tp.Dict[str, tp.Any]) -> None: + self.__name = json_trace_event["name"] + self.__category = json_trace_event["cat"] + self.__event_type = TraceEventType.parse_event_type( + json_trace_event["ph"] + ) + self.__tracing_clock_timestamp = int(json_trace_event["ts"]) + self.__pid = int(json_trace_event["pid"]) + self.__tid = int(json_trace_event["tid"]) + + @property + def name(self) -> str: + return self.__name + + @property + def category(self) -> str: + return self.__category + + @property + def event_type(self) -> TraceEventType: + return self.__event_type + + @property + def timestamp(self) -> int: + return self.__tracing_clock_timestamp + + @property + def pid(self) -> int: + return self.__pid + + @property + def tid(self) -> int: + return self.__tid + + def __str__(self) -> str: + return f"""{{ + name: {self.name} + cat: {self.category} + ph: {self.event_type} + ts: {self.timestamp} + pid: {self.pid} + tid: {self.tid} +}} +""" + + def __repr__(self) -> str: + return str(self) + + +class TEFReport(BaseReport, shorthand="TEF", file_type="json"): + """Report class to access trace event format files.""" + + def __init__(self, path: Path) -> None: + super().__init__(path) + + with open(self.path, "r", encoding="utf-8") as json_tef_report: + data = json.load(json_tef_report) + + self.__display_time_unit = data["displayTimeUnit"] + self.__trace_events = self._parse_trace_events(data["traceEvents"]) + # Parsing stackFrames is currently not implemented + # x = data["stackFrames"] + + @property + def display_time_unit(self) -> str: + return self.__display_time_unit + + @property + def trace_events(self) -> tp.List[TraceEvent]: + return self.__trace_events + + @property + def stack_frames(self) -> None: + raise NotImplementedError( + "Stack frame parsing is currently not implemented!" + ) + + @staticmethod + def _parse_trace_events( + raw_event_list: tp.List[str] + ) -> tp.List[TraceEvent]: + return [TraceEvent(data_item) for data_item in raw_event_list] diff --git a/varats/varats/experiments/vara/feature_perf_runner.py b/varats/varats/experiments/vara/feature_perf_runner.py index c8aa7bb86..3fbb639dd 100644 --- a/varats/varats/experiments/vara/feature_perf_runner.py +++ b/varats/varats/experiments/vara/feature_perf_runner.py @@ -6,7 +6,6 @@ from benchbuild.utils import actions from plumbum import local -from varats.data.reports.empty_report import EmptyReport as EMPTY from varats.experiment.experiment_util import ( exec_func_with_pe_error_handler, ExperimentHandle, @@ -31,6 +30,7 @@ ) from varats.report.report import ReportSpecification from varats.report.report import FileStatusExtension as FSE +from varats.report.tef_report import TEFReport class ExecAndTraceBinary(actions.Step): # type: ignore @@ -57,7 +57,7 @@ def run_perf_tracing(self) -> actions.StepResult: continue result_file = self.__experiment_handle.get_file_name( - EMPTY.shorthand(), + TEFReport.shorthand(), project_name=str(project.name), binary_name=binary.name, project_revision=project.version_of_primary, @@ -77,7 +77,7 @@ def run_perf_tracing(self) -> actions.StepResult: # TODO: figure out how to handle different configs executable("--slow") - #executable() + # executable() return actions.StepResult.OK @@ -87,7 +87,7 @@ class FeaturePerfRunner(VersionExperiment, shorthand="FPR"): NAME = "RunFeaturePerf" - REPORT_SPEC = ReportSpecification(EMPTY) + REPORT_SPEC = ReportSpecification(TEFReport) def actions_for_project( self, project: Project @@ -125,7 +125,7 @@ def actions_for_project( # Add own error handler to compile step. project.compile = get_default_compile_error_wrapped( - self.get_handle(), project, EMPTY + self.get_handle(), project, TEFReport ) analysis_actions = [] From 9ff38ae4d510940aefa07cc2589d792ffae5c768 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Sun, 26 Dec 2021 16:56:50 +0100 Subject: [PATCH 09/31] Ensure type conversions from json --- varats-core/varats/report/tef_report.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/varats-core/varats/report/tef_report.py b/varats-core/varats/report/tef_report.py index ec01e2818..74ffccae7 100644 --- a/varats-core/varats/report/tef_report.py +++ b/varats-core/varats/report/tef_report.py @@ -47,8 +47,8 @@ class TraceEvent(): target program.""" def __init__(self, json_trace_event: tp.Dict[str, tp.Any]) -> None: - self.__name = json_trace_event["name"] - self.__category = json_trace_event["cat"] + self.__name = str(json_trace_event["name"]) + self.__category = str(json_trace_event["cat"]) self.__event_type = TraceEventType.parse_event_type( json_trace_event["ph"] ) @@ -104,7 +104,7 @@ def __init__(self, path: Path) -> None: with open(self.path, "r", encoding="utf-8") as json_tef_report: data = json.load(json_tef_report) - self.__display_time_unit = data["displayTimeUnit"] + self.__display_time_unit = str(data["displayTimeUnit"]) self.__trace_events = self._parse_trace_events(data["traceEvents"]) # Parsing stackFrames is currently not implemented # x = data["stackFrames"] From a150bc533ba5bd4477022908a8c3952676b06b12 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Sun, 26 Dec 2021 16:59:49 +0100 Subject: [PATCH 10/31] Add doc string and desc to exec step --- varats/varats/experiments/vara/feature_perf_runner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/varats/varats/experiments/vara/feature_perf_runner.py b/varats/varats/experiments/vara/feature_perf_runner.py index 3fbb639dd..b327f6294 100644 --- a/varats/varats/experiments/vara/feature_perf_runner.py +++ b/varats/varats/experiments/vara/feature_perf_runner.py @@ -1,3 +1,5 @@ +"""Module for feature performance experiments that instrument and measure the +execution performance of each binary that is produced by a project.""" import os import typing as tp @@ -38,7 +40,7 @@ class ExecAndTraceBinary(actions.Step): # type: ignore configurations, against one or multiple workloads.""" NAME = "ExecBinary" - DESCRIPTION = "fobar" # TODO: fix + DESCRIPTION = "Executes each binary and caputres white-box performance traces" def __init__(self, project: Project, experiment_handle: ExperimentHandle): super().__init__(obj=project, action_fn=self.run_perf_tracing) From 4bf78a2476010bb8023558e5fc57e368b4751445 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Sun, 26 Dec 2021 17:12:49 +0100 Subject: [PATCH 11/31] Fixes type error --- varats-core/varats/report/tef_report.py | 2 +- varats/varats/experiments/vara/feature_perf_runner.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/varats-core/varats/report/tef_report.py b/varats-core/varats/report/tef_report.py index 74ffccae7..46f78176a 100644 --- a/varats-core/varats/report/tef_report.py +++ b/varats-core/varats/report/tef_report.py @@ -125,6 +125,6 @@ def stack_frames(self) -> None: @staticmethod def _parse_trace_events( - raw_event_list: tp.List[str] + raw_event_list: tp.List[tp.Dict[str, tp.Any]] ) -> tp.List[TraceEvent]: return [TraceEvent(data_item) for data_item in raw_event_list] diff --git a/varats/varats/experiments/vara/feature_perf_runner.py b/varats/varats/experiments/vara/feature_perf_runner.py index b327f6294..e189188ea 100644 --- a/varats/varats/experiments/vara/feature_perf_runner.py +++ b/varats/varats/experiments/vara/feature_perf_runner.py @@ -40,7 +40,8 @@ class ExecAndTraceBinary(actions.Step): # type: ignore configurations, against one or multiple workloads.""" NAME = "ExecBinary" - DESCRIPTION = "Executes each binary and caputres white-box performance traces" + DESCRIPTION = "Executes each binary and caputres white-box " +\ + "performance traces." def __init__(self, project: Project, experiment_handle: ExperimentHandle): super().__init__(obj=project, action_fn=self.run_perf_tracing) From 704fd53e0c027e8e1938995d2910f7e17589505c Mon Sep 17 00:00:00 2001 From: Daniel Gusenburger Date: Wed, 19 Jan 2022 16:30:44 +0100 Subject: [PATCH 12/31] premerge --- requirements.txt | 4 +- varats/setup.py | 4 +- .../vara/compare_traced_untraced.py | 167 ++++++++++++++++++ 3 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 varats/varats/experiments/vara/compare_traced_untraced.py diff --git a/requirements.txt b/requirements.txt index 41123e23a..a88fee757 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,9 +27,9 @@ pyzmq>=19.0.0 requests>=2.23.0 requests_cache>=0.5.2 rich>=1.3.1 -scikit-learn~=0.23.1 +scikit-learn>=0.23.1 seaborn>=0.8.0 -statsmodels~=0.11.1 +statsmodels>=0.11.1 tabulate>=0.8.6 types-PyYAML types-requests diff --git a/varats/setup.py b/varats/setup.py index 33193f175..9be7ff7ab 100644 --- a/varats/setup.py +++ b/varats/setup.py @@ -44,9 +44,9 @@ "pyzmq>=19.0.0", "requests>=2.24.0", "rich>=1.3.1", - "scikit-learn~=0.23.1", + "scikit-learn>=0.23.1", "seaborn>=0.8.0", - "statsmodels~=0.11.1", + "statsmodels>=0.11.1", "tabulate>=0.8.6", "varats-core>=11.0.0", "wllvm>=1.1.4", diff --git a/varats/varats/experiments/vara/compare_traced_untraced.py b/varats/varats/experiments/vara/compare_traced_untraced.py new file mode 100644 index 000000000..be865c437 --- /dev/null +++ b/varats/varats/experiments/vara/compare_traced_untraced.py @@ -0,0 +1,167 @@ +"""Module for feature performance experiments that instrument and measure the +execution performance of each binary that is produced by a project.""" +import os +import typing as tp + +from benchbuild import Project +from benchbuild.extensions import compiler, run, time +from benchbuild.utils import actions +from plumbum import local + +from varats.experiment.experiment_util import ( + ExperimentHandle, + get_varats_result_folder, + VersionExperiment, + get_default_compile_error_wrapped, +) +from varats.project.project_util import BinaryType +from varats.provider.feature.feature_model_provider import ( + FeatureModelProvider, + FeatureModelNotFound, +) +from varats.report.report import ReportSpecification +from varats.report.report import FileStatusExtension as FSE +from varats.report.tef_report import TEFReport + + +class ExecAndTraceBinary(actions.Step): # type: ignore + """Executes the specified binaries of the project, in specific + configurations, against one or multiple workloads.""" + + NAME = "ExecBinary" + DESCRIPTION = "Executes each binary and captures white-box performance traces." + + def __init__(self, project: Project, experiment_handle: ExperimentHandle): + super().__init__(obj=project, action_fn=self.run_perf_tracing) # type: ignore + self.__experiment_handle = experiment_handle + + def run_perf_tracing(self) -> actions.StepResult: + """Execute the specified binaries of the project, in specific + configurations, against one or multiple workloads.""" + project: Project = self.obj + + print(f"PWD {os.getcwd()}") + + vara_result_folder = get_varats_result_folder(project) + for binary in project.binaries: # type: ignore[attr-defined] + if binary.type != BinaryType.EXECUTABLE: + continue + + result_file = self.__experiment_handle.get_file_name( + TEFReport.shorthand(), + project_name=str(project.name), + binary_name=binary.name, + project_revision=project.version_of_primary, + project_uuid=str(project.run_uuid), + extension_type=FSE.SUCCESS, + ) + + with local.cwd(local.path(project.source_of_primary)): + print(f"Currently at {local.path(project.source_of_primary)}") + print(f"Bin path {binary.path}") + executable = local[f"{binary.path}"] + with local.env(VARA_TRACE_FILE=f"{vara_result_folder}/{result_file}"): + executable() + + return actions.StepResult.OK + + +class RunTraced(VersionExperiment, shorthand="RT"): + """Build and run the traced version of the binary""" + + NAME = "RunTraced" + + REPORT_SPEC = ReportSpecification(TEFReport) + + def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.Step]: + """ + Returns the specified steps to run the project(s) specified in the call + in a fixed order. + + Args: + project: to analyze + """ + + fm_provider = FeatureModelProvider.create_provider_for_project(project) + if fm_provider is None: + raise Exception("Could not get FeatureModelProvider!") + + fm_path = fm_provider.get_feature_model_path(project.version_of_primary) + if fm_path is None or not fm_path.exists(): + raise FeatureModelNotFound(project, fm_path) + + # Sets FM model flags + project.cflags += ["-fvara-feature", f"-fvara-fm-path={fm_path.absolute()}"] + # Sets vara tracing flags + project.cflags += ["-fsanitize=vara", "-fvara-instr=trace_event"] + + # Add the required runtime extensions to the project(s). + project.runtime_extension = ( + run.RuntimeExtension(project, self) << time.RunWithTime() + ) + + # Add the required compiler extensions to the project(s). + project.compiler_extension = ( + compiler.RunCompiler(project, self) << run.WithTimeout() + ) + + # Add own error handler to compile step. + project.compile = get_default_compile_error_wrapped( + self.get_handle(), project, TEFReport + ) + + analysis_actions = [] + + analysis_actions.append(actions.Compile(project)) + analysis_actions.append(ExecAndTraceBinary(project, self.get_handle())) + analysis_actions.append(actions.Clean(project)) + + return analysis_actions + + +class RunUntraced(VersionExperiment, shorthand="RU"): + """Build and run the untraced version of the binary""" + + NAME = "RunUntraced" + + REPORT_SPEC = ReportSpecification(TEFReport) + + def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.Step]: + """ + Returns the specified steps to run the project(s) specified in the call + in a fixed order. + + Args: + project: to analyze + """ + + fm_provider = FeatureModelProvider.create_provider_for_project(project) + if fm_provider is None: + raise Exception("Could not get FeatureModelProvider!") + + fm_path = fm_provider.get_feature_model_path(project.version_of_primary) + if fm_path is None or not fm_path.exists(): + raise FeatureModelNotFound(project, fm_path) + + # Add the required runtime extensions to the project(s). + project.runtime_extension = ( + run.RuntimeExtension(project, self) << time.RunWithTime() + ) + + # Add the required compiler extensions to the project(s). + project.compiler_extension = ( + compiler.RunCompiler(project, self) << run.WithTimeout() + ) + + # Add own error handler to compile step. + project.compile = get_default_compile_error_wrapped( + self.get_handle(), project, TEFReport + ) + + analysis_actions = [] + + analysis_actions.append(actions.Compile(project)) + analysis_actions.append(ExecAndTraceBinary(project, self.get_handle())) + analysis_actions.append(actions.Clean(project)) + + return analysis_actions From 004b7ef59a9b5780acb32c8057975009aecc7fb6 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Wed, 19 Jan 2022 22:24:11 +0100 Subject: [PATCH 13/31] Adapts variant calculation to be experiment specific With our new, experiment based reports, we need to analyze a varaint every time it we cannot find a corresponding experiment specific report. Before, a variant was analyzed if we cannot find a corresponding report, independet of the experiment that actually produced the report. --- varats-core/varats/experiment/experiment_util.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/varats-core/varats/experiment/experiment_util.py b/varats-core/varats/experiment/experiment_util.py index c1d7bb4a2..9de8a35ad 100644 --- a/varats-core/varats/experiment/experiment_util.py +++ b/varats-core/varats/experiment/experiment_util.py @@ -421,10 +421,11 @@ def sample(cls, ) bad_revisions = [ - # TODO (se-sic/VaRA#791): clean up usage of report spec - revision.hash for revision, file_status in get_tagged_revisions( + revision.hash for revision, file_status in + get_tagged_experiment_specific_revisions( prj_cls, - getattr(cls, 'REPORT_SPEC').main_report + getattr(cls, 'REPORT_SPEC').main_report, + experiment_type=cls ) if file_status not in fs_good ] From 9213d3f01954a6a95c7e74e2108425e185d07cbe Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Wed, 19 Jan 2022 22:58:29 +0100 Subject: [PATCH 14/31] Ports code to consider all reports not just the main report --- .../varats/experiment/experiment_util.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/varats-core/varats/experiment/experiment_util.py b/varats-core/varats/experiment/experiment_util.py index 9de8a35ad..6f3b0218d 100644 --- a/varats-core/varats/experiment/experiment_util.py +++ b/varats-core/varats/experiment/experiment_util.py @@ -420,14 +420,18 @@ def sample(cls, "Experiment sub class does not implement REPORT_SPEC." ) - bad_revisions = [ - revision.hash for revision, file_status in - get_tagged_experiment_specific_revisions( - prj_cls, - getattr(cls, 'REPORT_SPEC').main_report, - experiment_type=cls - ) if file_status not in fs_good - ] + report_specific_bad_revs = [] + for report_type in cls.report_spec(): + report_specific_bad_revs.append({ + revision.hash for revision, file_status in + get_tagged_experiment_specific_revisions( + prj_cls, report_type, experiment_type=cls + ) if file_status not in fs_good + }) + + bad_revisions = report_specific_bad_revs[0].intersection( + *report_specific_bad_revs[1:] + ) variants = list( filter(lambda var: str(var[0]) not in bad_revisions, variants) From 042ab2edc2dfa501177f0bf832451bb5c4e2d97b Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Wed, 19 Jan 2022 23:02:28 +0100 Subject: [PATCH 15/31] Removes error check --- varats-core/varats/experiment/experiment_util.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/varats-core/varats/experiment/experiment_util.py b/varats-core/varats/experiment/experiment_util.py index 6f3b0218d..14e6cdf27 100644 --- a/varats-core/varats/experiment/experiment_util.py +++ b/varats-core/varats/experiment/experiment_util.py @@ -415,11 +415,6 @@ def sample(cls, for x in fs_whitelist } - if not hasattr(cls, 'REPORT_SPEC'): - raise TypeError( - "Experiment sub class does not implement REPORT_SPEC." - ) - report_specific_bad_revs = [] for report_type in cls.report_spec(): report_specific_bad_revs.append({ From 56060ebf50cee63200d62a962fcc4fae1e779d7a Mon Sep 17 00:00:00 2001 From: Daniel Gusenburger Date: Fri, 21 Jan 2022 10:20:49 +0100 Subject: [PATCH 16/31] first naive implementation of RunTraced and RunUntraced experiments --- .../vara/compare_traced_untraced.py | 123 ++++++++++++++++-- 1 file changed, 115 insertions(+), 8 deletions(-) diff --git a/varats/varats/experiments/vara/compare_traced_untraced.py b/varats/varats/experiments/vara/compare_traced_untraced.py index be865c437..8b7e60854 100644 --- a/varats/varats/experiments/vara/compare_traced_untraced.py +++ b/varats/varats/experiments/vara/compare_traced_untraced.py @@ -3,12 +3,18 @@ import os import typing as tp +from pathlib import Path + from benchbuild import Project from benchbuild.extensions import compiler, run, time from benchbuild.utils import actions +from benchbuild.utils.cmd import time_cmd from plumbum import local from varats.experiment.experiment_util import ( + exec_func_with_pe_error_handler, + get_default_compile_error_wrapped, + create_default_analysis_failure_handler, ExperimentHandle, get_varats_result_folder, VersionExperiment, @@ -22,6 +28,63 @@ from varats.report.report import ReportSpecification from varats.report.report import FileStatusExtension as FSE from varats.report.tef_report import TEFReport +from varats.report.gnu_time_report import TimeReport + +# TODO: Refactor to use a bass class for experiments to avoid code duplication +# where possible + + +class TimeBinary(actions.Step): + """Executes the specified binaries and record the time it took to run""" + + NAME = "TimeBinary" + DESCRIPTION = "Executes each binary and times it" + + def __init__(self, project: Project, experiment_handle: ExperimentHandle): + super().__init__(obj=project, action_fn=self.time_run) # type: ignore + self.__experiment_handle = experiment_handle + + def time_run(self) -> actions.StepResult: + """Execute the specified binaries of the project, in specific + configurations, against one or multiple workloads.""" + project: Project = self.obj + + print(f"PWD {os.getcwd()}") + + vara_result_folder = get_varats_result_folder(project) + + for binary in project.binaries: # type: ignore[attr-defined] + if binary.type != BinaryType.EXECUTABLE: + continue + + result_file = self.__experiment_handle.get_file_name( + TimeReport.shorthand(), + project_name=str(project.name), + binary_name=binary.name, + project_revision=project.version_of_primary, # type: ignore + project_uuid=str(project.run_uuid), + extension_type=FSE.SUCCESS, + ) + + with local.cwd(local.path(project.source_of_primary)): + print(f"Currently at {local.path(project.source_of_primary)}") + print(f"Bin path {binary.path}") + time_cmd = local["/usr/bin/time"] + run_cmd = time_cmd[ + "-o", f"{vara_result_folder}/{result_file}", "-v", binary.path + ] + # REVIEW: Is this correct ? Copied it from JustCompile + exec_func_with_pe_error_handler( + run_cmd, + create_default_analysis_failure_handler( + self.__experiment_handle, + project, + TimeReport, + Path(vara_result_folder), + ), + ) + + return actions.StepResult.OK class ExecAndTraceBinary(actions.Step): # type: ignore @@ -29,7 +92,7 @@ class ExecAndTraceBinary(actions.Step): # type: ignore configurations, against one or multiple workloads.""" NAME = "ExecBinary" - DESCRIPTION = "Executes each binary and captures white-box performance traces." + DESCRIPTION = "Executes each binary and white-box performance traces." def __init__(self, project: Project, experiment_handle: ExperimentHandle): super().__init__(obj=project, action_fn=self.run_perf_tracing) # type: ignore @@ -66,7 +129,7 @@ def run_perf_tracing(self) -> actions.StepResult: return actions.StepResult.OK -class RunTraced(VersionExperiment, shorthand="RT"): +class RunTracedTEF(VersionExperiment, shorthand="RTTEF"): """Build and run the traced version of the binary""" NAME = "RunTraced" @@ -119,12 +182,12 @@ def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.St return analysis_actions -class RunUntraced(VersionExperiment, shorthand="RU"): - """Build and run the untraced version of the binary""" +class RunTracedTime(VersionExperiment, shorthand="RTTIME"): + """Build and run the traced version of the binary""" - NAME = "RunUntraced" + NAME = "RunTracedTime" - REPORT_SPEC = ReportSpecification(TEFReport) + REPORT_SPEC = ReportSpecification(TimeReport) def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.Step]: """ @@ -143,6 +206,11 @@ def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.St if fm_path is None or not fm_path.exists(): raise FeatureModelNotFound(project, fm_path) + # Sets FM model flags + project.cflags += ["-fvara-feature", f"-fvara-fm-path={fm_path.absolute()}"] + # Sets vara tracing flags + project.cflags += ["-fsanitize=vara", "-fvara-instr=trace_event"] + # Add the required runtime extensions to the project(s). project.runtime_extension = ( run.RuntimeExtension(project, self) << time.RunWithTime() @@ -155,13 +223,52 @@ def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.St # Add own error handler to compile step. project.compile = get_default_compile_error_wrapped( - self.get_handle(), project, TEFReport + self.get_handle(), project, TimeReport ) analysis_actions = [] analysis_actions.append(actions.Compile(project)) - analysis_actions.append(ExecAndTraceBinary(project, self.get_handle())) + analysis_actions.append(TimeBinary(project, self.get_handle())) + analysis_actions.append(actions.Clean(project)) + + return analysis_actions + + +class RunUntraced(VersionExperiment, shorthand="RU"): + """Build and run the untraced version of the binary""" + + NAME = "RunUntraced" + + REPORT_SPEC = ReportSpecification(TimeReport) + + def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.Step]: + """ + Returns the specified steps to run the project(s) specified in the call + in a fixed order. + + Args: + project: to analyze + """ + # Add the required runtime extensions to the project(s). + project.runtime_extension = ( + run.RuntimeExtension(project, self) << time.RunWithTime() + ) + + # Add the required compiler extensions to the project(s). + project.compiler_extension = ( + compiler.RunCompiler(project, self) << run.WithTimeout() + ) + + # Add own error handler to compile step. + project.compile = get_default_compile_error_wrapped( + self.get_handle(), project, TimeReport + ) + + analysis_actions = [] + + analysis_actions.append(actions.Compile(project)) + analysis_actions.append(TimeBinary(project, self.get_handle())) analysis_actions.append(actions.Clean(project)) return analysis_actions From a8e0440bb09bc46a37969453c37151caa00aa33d Mon Sep 17 00:00:00 2001 From: Daniel Gusenburger Date: Mon, 24 Jan 2022 11:57:57 +0100 Subject: [PATCH 17/31] implemented common base class for evaluation experiments --- .../vara/compare_traced_untraced.py | 133 +++++++++--------- 1 file changed, 67 insertions(+), 66 deletions(-) diff --git a/varats/varats/experiments/vara/compare_traced_untraced.py b/varats/varats/experiments/vara/compare_traced_untraced.py index 8b7e60854..27a0976d0 100644 --- a/varats/varats/experiments/vara/compare_traced_untraced.py +++ b/varats/varats/experiments/vara/compare_traced_untraced.py @@ -27,6 +27,7 @@ ) from varats.report.report import ReportSpecification from varats.report.report import FileStatusExtension as FSE +from varats.report.report import BaseReport from varats.report.tef_report import TEFReport from varats.report.gnu_time_report import TimeReport @@ -129,12 +130,21 @@ def run_perf_tracing(self) -> actions.StepResult: return actions.StepResult.OK -class RunTracedTEF(VersionExperiment, shorthand="RTTEF"): - """Build and run the traced version of the binary""" +class BaseRunner(VersionExperiment, shorthand="BR"): - NAME = "RunTraced" + NAME = "BaseRunner" + REPORT_SPEC = ReportSpecification(TimeReport) - REPORT_SPEC = ReportSpecification(TEFReport) + def __init__( + self, report_type: tp.Type[BaseReport], analysis_type: tp.Type[actions.Step], *args, **kwargs + ): + super().__init__(*args, **kwargs) + self.cflags = [] + self.report_type = report_type + self.analysis_type = analysis_type + + def set_cflags(self, *flags): + self.cflags = flags def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.Step]: """ @@ -144,19 +154,7 @@ def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.St Args: project: to analyze """ - - fm_provider = FeatureModelProvider.create_provider_for_project(project) - if fm_provider is None: - raise Exception("Could not get FeatureModelProvider!") - - fm_path = fm_provider.get_feature_model_path(project.version_of_primary) - if fm_path is None or not fm_path.exists(): - raise FeatureModelNotFound(project, fm_path) - - # Sets FM model flags - project.cflags += ["-fvara-feature", f"-fvara-fm-path={fm_path.absolute()}"] - # Sets vara tracing flags - project.cflags += ["-fsanitize=vara", "-fvara-instr=trace_event"] + project.cflags.extend(self.cflags) # Add the required runtime extensions to the project(s). project.runtime_extension = ( @@ -170,24 +168,27 @@ def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.St # Add own error handler to compile step. project.compile = get_default_compile_error_wrapped( - self.get_handle(), project, TEFReport + self.get_handle(), project, self.report_type ) analysis_actions = [] analysis_actions.append(actions.Compile(project)) - analysis_actions.append(ExecAndTraceBinary(project, self.get_handle())) + analysis_actions.append(self.analysis_type(project, self.get_handle())) analysis_actions.append(actions.Clean(project)) return analysis_actions -class RunTracedTime(VersionExperiment, shorthand="RTTIME"): +class RunTracedTEF(BaseRunner, shorthand="RTTEF"): """Build and run the traced version of the binary""" - NAME = "RunTracedTime" + NAME = "RunTracedTEF" - REPORT_SPEC = ReportSpecification(TimeReport) + REPORT_SPEC = ReportSpecification(TEFReport) + + def __init__(self, *args, **kwargs): + super().__init__(TEFReport, ExecAndTraceBinary, *args, **kwargs) def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.Step]: """ @@ -206,42 +207,26 @@ def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.St if fm_path is None or not fm_path.exists(): raise FeatureModelNotFound(project, fm_path) - # Sets FM model flags - project.cflags += ["-fvara-feature", f"-fvara-fm-path={fm_path.absolute()}"] - # Sets vara tracing flags - project.cflags += ["-fsanitize=vara", "-fvara-instr=trace_event"] - - # Add the required runtime extensions to the project(s). - project.runtime_extension = ( - run.RuntimeExtension(project, self) << time.RunWithTime() - ) - - # Add the required compiler extensions to the project(s). - project.compiler_extension = ( - compiler.RunCompiler(project, self) << run.WithTimeout() + self.set_cflags( + "-fvara-feature", + f"-fvara-fm-path={fm_path.absolute()}", + "-fsanitize=vara", + "-fvara-instr=trace_event", ) - # Add own error handler to compile step. - project.compile = get_default_compile_error_wrapped( - self.get_handle(), project, TimeReport - ) + return super().actions_for_project(project) - analysis_actions = [] - analysis_actions.append(actions.Compile(project)) - analysis_actions.append(TimeBinary(project, self.get_handle())) - analysis_actions.append(actions.Clean(project)) - - return analysis_actions - - -class RunUntraced(VersionExperiment, shorthand="RU"): - """Build and run the untraced version of the binary""" +class RunTracedTime(BaseRunner, shorthand="RTTIME"): + """Build and run the traced version of the binary""" - NAME = "RunUntraced" + NAME = "RunTracedTime" REPORT_SPEC = ReportSpecification(TimeReport) + def __init__(self, *args, **kwargs): + super().__init__(TimeReport, TimeBinary, *args, **kwargs) + def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.Step]: """ Returns the specified steps to run the project(s) specified in the call @@ -250,25 +235,41 @@ def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.St Args: project: to analyze """ - # Add the required runtime extensions to the project(s). - project.runtime_extension = ( - run.RuntimeExtension(project, self) << time.RunWithTime() - ) - # Add the required compiler extensions to the project(s). - project.compiler_extension = ( - compiler.RunCompiler(project, self) << run.WithTimeout() - ) + fm_provider = FeatureModelProvider.create_provider_for_project(project) + if fm_provider is None: + raise Exception("Could not get FeatureModelProvider!") - # Add own error handler to compile step. - project.compile = get_default_compile_error_wrapped( - self.get_handle(), project, TimeReport + fm_path = fm_provider.get_feature_model_path(project.version_of_primary) + if fm_path is None or not fm_path.exists(): + raise FeatureModelNotFound(project, fm_path) + + self.set_cflags( + "-fvara-feature", + f"-fvara-fm-path={fm_path.absolute()}", + "-fsanitize=vara", + "-fvara-instr=trace_event", ) - analysis_actions = [] + return super().actions_for_project(project) - analysis_actions.append(actions.Compile(project)) - analysis_actions.append(TimeBinary(project, self.get_handle())) - analysis_actions.append(actions.Clean(project)) - return analysis_actions +class RunUntraced(BaseRunner, shorthand="RU"): + """Build and run the untraced version of the binary""" + + NAME = "RunUntraced" + + REPORT_SPEC = ReportSpecification(TimeReport) + + def __init__(self, *args, **kwargs): + super().__init__(TimeReport, TimeBinary, *args, **kwargs) + + def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.Step]: + """ + Returns the specified steps to run the project(s) specified in the call + in a fixed order. + + Args: + project: to analyze + """ + return super().actions_for_project(project) From fa125a12e697549223a9df579c5db3821a2a5d84 Mon Sep 17 00:00:00 2001 From: Daniel Gusenburger Date: Tue, 8 Feb 2022 11:02:54 +0100 Subject: [PATCH 18/31] implemented some of the suggested changes --- .../vara/compare_traced_untraced.py | 100 +++--------------- 1 file changed, 15 insertions(+), 85 deletions(-) diff --git a/varats/varats/experiments/vara/compare_traced_untraced.py b/varats/varats/experiments/vara/compare_traced_untraced.py index 27a0976d0..d293ff294 100644 --- a/varats/varats/experiments/vara/compare_traced_untraced.py +++ b/varats/varats/experiments/vara/compare_traced_untraced.py @@ -8,7 +8,7 @@ from benchbuild import Project from benchbuild.extensions import compiler, run, time from benchbuild.utils import actions -from benchbuild.utils.cmd import time_cmd +from benchbuild.utils.cmd import time as time_cmd from plumbum import local from varats.experiment.experiment_util import ( @@ -28,7 +28,6 @@ from varats.report.report import ReportSpecification from varats.report.report import FileStatusExtension as FSE from varats.report.report import BaseReport -from varats.report.tef_report import TEFReport from varats.report.gnu_time_report import TimeReport # TODO: Refactor to use a bass class for experiments to avoid code duplication @@ -62,7 +61,7 @@ def time_run(self) -> actions.StepResult: TimeReport.shorthand(), project_name=str(project.name), binary_name=binary.name, - project_revision=project.version_of_primary, # type: ignore + project_revision=project.version_of_primary, project_uuid=str(project.run_uuid), extension_type=FSE.SUCCESS, ) @@ -70,7 +69,13 @@ def time_run(self) -> actions.StepResult: with local.cwd(local.path(project.source_of_primary)): print(f"Currently at {local.path(project.source_of_primary)}") print(f"Bin path {binary.path}") - time_cmd = local["/usr/bin/time"] + # TODO: In future versions, we can pass arguments to the binary here + # run_cmd = time_cmd[ + # "-o", + # f"{vara_result_folder}/{result_file}", + # "-v", + # binary["--args", "that", "are", "passed", "to", "the", "binary"], + # ] run_cmd = time_cmd[ "-o", f"{vara_result_folder}/{result_file}", "-v", binary.path ] @@ -88,55 +93,17 @@ def time_run(self) -> actions.StepResult: return actions.StepResult.OK -class ExecAndTraceBinary(actions.Step): # type: ignore - """Executes the specified binaries of the project, in specific - configurations, against one or multiple workloads.""" - - NAME = "ExecBinary" - DESCRIPTION = "Executes each binary and white-box performance traces." - - def __init__(self, project: Project, experiment_handle: ExperimentHandle): - super().__init__(obj=project, action_fn=self.run_perf_tracing) # type: ignore - self.__experiment_handle = experiment_handle - - def run_perf_tracing(self) -> actions.StepResult: - """Execute the specified binaries of the project, in specific - configurations, against one or multiple workloads.""" - project: Project = self.obj - - print(f"PWD {os.getcwd()}") - - vara_result_folder = get_varats_result_folder(project) - for binary in project.binaries: # type: ignore[attr-defined] - if binary.type != BinaryType.EXECUTABLE: - continue - - result_file = self.__experiment_handle.get_file_name( - TEFReport.shorthand(), - project_name=str(project.name), - binary_name=binary.name, - project_revision=project.version_of_primary, - project_uuid=str(project.run_uuid), - extension_type=FSE.SUCCESS, - ) - - with local.cwd(local.path(project.source_of_primary)): - print(f"Currently at {local.path(project.source_of_primary)}") - print(f"Bin path {binary.path}") - executable = local[f"{binary.path}"] - with local.env(VARA_TRACE_FILE=f"{vara_result_folder}/{result_file}"): - executable() - - return actions.StepResult.OK - - class BaseRunner(VersionExperiment, shorthand="BR"): NAME = "BaseRunner" REPORT_SPEC = ReportSpecification(TimeReport) def __init__( - self, report_type: tp.Type[BaseReport], analysis_type: tp.Type[actions.Step], *args, **kwargs + self, + report_type: tp.Type[BaseReport], + analysis_type: tp.Type[actions.Step], + *args, + **kwargs, ): super().__init__(*args, **kwargs) self.cflags = [] @@ -180,44 +147,7 @@ def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.St return analysis_actions -class RunTracedTEF(BaseRunner, shorthand="RTTEF"): - """Build and run the traced version of the binary""" - - NAME = "RunTracedTEF" - - REPORT_SPEC = ReportSpecification(TEFReport) - - def __init__(self, *args, **kwargs): - super().__init__(TEFReport, ExecAndTraceBinary, *args, **kwargs) - - def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.Step]: - """ - Returns the specified steps to run the project(s) specified in the call - in a fixed order. - - Args: - project: to analyze - """ - - fm_provider = FeatureModelProvider.create_provider_for_project(project) - if fm_provider is None: - raise Exception("Could not get FeatureModelProvider!") - - fm_path = fm_provider.get_feature_model_path(project.version_of_primary) - if fm_path is None or not fm_path.exists(): - raise FeatureModelNotFound(project, fm_path) - - self.set_cflags( - "-fvara-feature", - f"-fvara-fm-path={fm_path.absolute()}", - "-fsanitize=vara", - "-fvara-instr=trace_event", - ) - - return super().actions_for_project(project) - - -class RunTracedTime(BaseRunner, shorthand="RTTIME"): +class RunTraced(BaseRunner, shorthand="RTTIME"): """Build and run the traced version of the binary""" NAME = "RunTracedTime" From 375a0a9cb3625890ae76865a30604b9a3382de4c Mon Sep 17 00:00:00 2001 From: Daniel Gusenburger Date: Mon, 11 Apr 2022 15:50:10 +0200 Subject: [PATCH 19/31] fix: now passing correct CLI args to clang in RunTimeTraced to properly select policy at runtime --- varats/varats/experiments/vara/compare_traced_untraced.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/varats/varats/experiments/vara/compare_traced_untraced.py b/varats/varats/experiments/vara/compare_traced_untraced.py index d293ff294..c9a92f9e0 100644 --- a/varats/varats/experiments/vara/compare_traced_untraced.py +++ b/varats/varats/experiments/vara/compare_traced_untraced.py @@ -1,4 +1,4 @@ -"""Module for feature performance experiments that instrument and measure the +"""Module for feature performance eperiments that instrument and measure the execution performance of each binary that is produced by a project.""" import os import typing as tp @@ -30,7 +30,7 @@ from varats.report.report import BaseReport from varats.report.gnu_time_report import TimeReport -# TODO: Refactor to use a bass class for experiments to avoid code duplication +# TODO: Refactor to use a base class for experiments to avoid code duplication # where possible @@ -179,6 +179,8 @@ def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.St f"-fvara-fm-path={fm_path.absolute()}", "-fsanitize=vara", "-fvara-instr=trace_event", + "-mllvm", + "--vara-optimizer-policy=naive", ) return super().actions_for_project(project) From 2858825d204850f9e4cffd24a5816b6e614b6b57 Mon Sep 17 00:00:00 2001 From: Daniel Gusenburger Date: Fri, 15 Apr 2022 14:44:09 +0200 Subject: [PATCH 20/31] fix: removed type ignore comments --- .../experiments/vara/compare_traced_untraced.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/varats/varats/experiments/vara/compare_traced_untraced.py b/varats/varats/experiments/vara/compare_traced_untraced.py index c9a92f9e0..99ed273a8 100644 --- a/varats/varats/experiments/vara/compare_traced_untraced.py +++ b/varats/varats/experiments/vara/compare_traced_untraced.py @@ -29,9 +29,7 @@ from varats.report.report import FileStatusExtension as FSE from varats.report.report import BaseReport from varats.report.gnu_time_report import TimeReport - -# TODO: Refactor to use a base class for experiments to avoid code duplication -# where possible +from varats.utils.git_util import ShortCommitHash class TimeBinary(actions.Step): @@ -41,7 +39,7 @@ class TimeBinary(actions.Step): DESCRIPTION = "Executes each binary and times it" def __init__(self, project: Project, experiment_handle: ExperimentHandle): - super().__init__(obj=project, action_fn=self.time_run) # type: ignore + super().__init__(obj=project, action_fn=self.time_run) self.__experiment_handle = experiment_handle def time_run(self) -> actions.StepResult: @@ -53,7 +51,7 @@ def time_run(self) -> actions.StepResult: vara_result_folder = get_varats_result_folder(project) - for binary in project.binaries: # type: ignore[attr-defined] + for binary in project.binaries: if binary.type != BinaryType.EXECUTABLE: continue @@ -61,7 +59,7 @@ def time_run(self) -> actions.StepResult: TimeReport.shorthand(), project_name=str(project.name), binary_name=binary.name, - project_revision=project.version_of_primary, + project_revision=ShortCommitHash(project.version_of_primary), project_uuid=str(project.run_uuid), extension_type=FSE.SUCCESS, ) @@ -69,7 +67,8 @@ def time_run(self) -> actions.StepResult: with local.cwd(local.path(project.source_of_primary)): print(f"Currently at {local.path(project.source_of_primary)}") print(f"Bin path {binary.path}") - # TODO: In future versions, we can pass arguments to the binary here + + # TODO(d.gusenburger): In future versions, we can pass arguments to the binary here # run_cmd = time_cmd[ # "-o", # f"{vara_result_folder}/{result_file}", @@ -79,6 +78,7 @@ def time_run(self) -> actions.StepResult: run_cmd = time_cmd[ "-o", f"{vara_result_folder}/{result_file}", "-v", binary.path ] + # REVIEW: Is this correct ? Copied it from JustCompile exec_func_with_pe_error_handler( run_cmd, @@ -179,6 +179,7 @@ def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.St f"-fvara-fm-path={fm_path.absolute()}", "-fsanitize=vara", "-fvara-instr=trace_event", + # Temporary. Instructions to enable instrumentation placement optimization "-mllvm", "--vara-optimizer-policy=naive", ) From ed9a39cfb2725e0a57c42006e00cab7dc9347d9e Mon Sep 17 00:00:00 2001 From: Daniel Gusenburger Date: Fri, 20 Jan 2023 12:09:20 +0100 Subject: [PATCH 21/31] updated experiments --- .../varats/experiments/base/time_workloads.py | 1 + .../vara/compare_traced_untraced.py | 253 +++++++++--------- .../perf_tests/feature_perf_cs_collection.py | 15 ++ varats/varats/tools/bb_config.py | 2 +- 4 files changed, 140 insertions(+), 131 deletions(-) diff --git a/varats/varats/experiments/base/time_workloads.py b/varats/varats/experiments/base/time_workloads.py index 60242728b..19d2c70af 100644 --- a/varats/varats/experiments/base/time_workloads.py +++ b/varats/varats/experiments/base/time_workloads.py @@ -94,6 +94,7 @@ def actions_for_project( binary = project.binaries[0] measurement_repetitions = 2 + result_filepath = create_new_success_result_filepath( self.get_handle(), self.get_handle().report_spec().main_report, project, binary diff --git a/varats/varats/experiments/vara/compare_traced_untraced.py b/varats/varats/experiments/vara/compare_traced_untraced.py index 99ed273a8..90323fb70 100644 --- a/varats/varats/experiments/vara/compare_traced_untraced.py +++ b/varats/varats/experiments/vara/compare_traced_untraced.py @@ -1,119 +1,94 @@ """Module for feature performance eperiments that instrument and measure the execution performance of each binary that is produced by a project.""" -import os import typing as tp -from pathlib import Path - -from benchbuild import Project from benchbuild.extensions import compiler, run, time from benchbuild.utils import actions -from benchbuild.utils.cmd import time as time_cmd -from plumbum import local from varats.experiment.experiment_util import ( - exec_func_with_pe_error_handler, - get_default_compile_error_wrapped, - create_default_analysis_failure_handler, - ExperimentHandle, - get_varats_result_folder, - VersionExperiment, - get_default_compile_error_wrapped, -) -from varats.project.project_util import BinaryType -from varats.provider.feature.feature_model_provider import ( - FeatureModelProvider, - FeatureModelNotFound, + create_new_success_result_filepath, get_default_compile_error_wrapped, + get_default_compile_error_wrapped, ZippedExperimentSteps ) from varats.report.report import ReportSpecification -from varats.report.report import FileStatusExtension as FSE -from varats.report.report import BaseReport from varats.report.gnu_time_report import TimeReport -from varats.utils.git_util import ShortCommitHash +from varats.experiments.vara.feature_experiment import FeatureExperiment +from varats.experiments.base.time_workloads import TimeProjectWorkloads + +from varats.project.varats_project import VProject + + +class RunTracedUnoptimized(FeatureExperiment, shorthand="RTUnopt"): + """Build and run the traced version of the binary""" + NAME = "RunTracedUnoptimized" + REPORT_SPEC = ReportSpecification(TimeReport) -class TimeBinary(actions.Step): - """Executes the specified binaries and record the time it took to run""" + def actions_for_project( + self, project: VProject + ) -> tp.MutableSequence[actions.Step]: + """ + Returns the specified steps to run the project(s) specified in the call + in a fixed order. - NAME = "TimeBinary" - DESCRIPTION = "Executes each binary and times it" + Args: + project: to analyze + """ + instr_type = "trace_event" # trace_event - def __init__(self, project: Project, experiment_handle: ExperimentHandle): - super().__init__(obj=project, action_fn=self.time_run) - self.__experiment_handle = experiment_handle + project.cflags += self.get_vara_feature_cflags(project) - def time_run(self) -> actions.StepResult: - """Execute the specified binaries of the project, in specific - configurations, against one or multiple workloads.""" - project: Project = self.obj + project.cflags += self.get_vara_tracing_cflags(instr_type) - print(f"PWD {os.getcwd()}") + project.ldflags += self.get_vara_tracing_ldflags() - vara_result_folder = get_varats_result_folder(project) + # Add the required runtime extensions to the project(s). + project.runtime_extension = ( + run.RuntimeExtension(project, self) << time.RunWithTime() + ) + + # Add the required compiler extensions to the project(s). + project.compiler_extension = ( + compiler.RunCompiler(project, self) << run.WithTimeout() + ) + + # Add own error handler to compile step. + project.compile = get_default_compile_error_wrapped( + self.get_handle(), project, TimeReport + ) + + measurement_reps = 5 + + analysis_actions = [] + + analysis_actions.append(actions.Compile(project)) for binary in project.binaries: - if binary.type != BinaryType.EXECUTABLE: - continue - - result_file = self.__experiment_handle.get_file_name( - TimeReport.shorthand(), - project_name=str(project.name), - binary_name=binary.name, - project_revision=ShortCommitHash(project.version_of_primary), - project_uuid=str(project.run_uuid), - extension_type=FSE.SUCCESS, + result_filepath = create_new_success_result_filepath( + self.get_handle(), + self.get_handle().report_spec().main_report, project, binary ) - - with local.cwd(local.path(project.source_of_primary)): - print(f"Currently at {local.path(project.source_of_primary)}") - print(f"Bin path {binary.path}") - - # TODO(d.gusenburger): In future versions, we can pass arguments to the binary here - # run_cmd = time_cmd[ - # "-o", - # f"{vara_result_folder}/{result_file}", - # "-v", - # binary["--args", "that", "are", "passed", "to", "the", "binary"], - # ] - run_cmd = time_cmd[ - "-o", f"{vara_result_folder}/{result_file}", "-v", binary.path - ] - - # REVIEW: Is this correct ? Copied it from JustCompile - exec_func_with_pe_error_handler( - run_cmd, - create_default_analysis_failure_handler( - self.__experiment_handle, - project, - TimeReport, - Path(vara_result_folder), - ), + analysis_actions.append( + ZippedExperimentSteps( + result_filepath, [ + TimeProjectWorkloads(project, num, binary) + for num in range(measurement_reps) + ] ) + ) + analysis_actions.append(actions.Clean(project)) - return actions.StepResult.OK + return analysis_actions -class BaseRunner(VersionExperiment, shorthand="BR"): +class RunTracedNaive(FeatureExperiment, shorthand="RTNaive"): + """Build and run the traced version of the binary""" - NAME = "BaseRunner" + NAME = "RunTracedNaiveOptimization" REPORT_SPEC = ReportSpecification(TimeReport) - def __init__( - self, - report_type: tp.Type[BaseReport], - analysis_type: tp.Type[actions.Step], - *args, - **kwargs, - ): - super().__init__(*args, **kwargs) - self.cflags = [] - self.report_type = report_type - self.analysis_type = analysis_type - - def set_cflags(self, *flags): - self.cflags = flags - - def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.Step]: + def actions_for_project( + self, project: VProject + ) -> tp.MutableSequence[actions.Step]: """ Returns the specified steps to run the project(s) specified in the call in a fixed order. @@ -121,7 +96,15 @@ def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.St Args: project: to analyze """ - project.cflags.extend(self.cflags) + instr_type = "trace_event" # trace_event + + project.cflags += self.get_vara_feature_cflags(project) + + project.cflags += self.get_vara_tracing_cflags(instr_type) + + project.cflags += ["-mllvm", "--vara-optimizer-policy=naive"] + + project.ldflags += self.get_vara_tracing_ldflags() # Add the required runtime extensions to the project(s). project.runtime_extension = ( @@ -135,29 +118,42 @@ def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.St # Add own error handler to compile step. project.compile = get_default_compile_error_wrapped( - self.get_handle(), project, self.report_type + self.get_handle(), project, TimeReport ) + measurement_reps = 5 + analysis_actions = [] analysis_actions.append(actions.Compile(project)) - analysis_actions.append(self.analysis_type(project, self.get_handle())) + for binary in project.binaries: + result_filepath = create_new_success_result_filepath( + self.get_handle(), + self.get_handle().report_spec().main_report, project, binary + ) + analysis_actions.append( + ZippedExperimentSteps( + result_filepath, [ + TimeProjectWorkloads(project, num, binary) + for num in range(measurement_reps) + ] + ) + ) analysis_actions.append(actions.Clean(project)) return analysis_actions -class RunTraced(BaseRunner, shorthand="RTTIME"): - """Build and run the traced version of the binary""" +class RunUntraced(FeatureExperiment, shorthand="RU"): + """Build and run the untraced version of the binary""" - NAME = "RunTracedTime" + NAME = "RunUntraced" REPORT_SPEC = ReportSpecification(TimeReport) - def __init__(self, *args, **kwargs): - super().__init__(TimeReport, TimeBinary, *args, **kwargs) - - def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.Step]: + def actions_for_project( + self, project: VProject + ) -> tp.MutableSequence[actions.Step]: """ Returns the specified steps to run the project(s) specified in the call in a fixed order. @@ -165,44 +161,41 @@ def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.St Args: project: to analyze """ - - fm_provider = FeatureModelProvider.create_provider_for_project(project) - if fm_provider is None: - raise Exception("Could not get FeatureModelProvider!") - - fm_path = fm_provider.get_feature_model_path(project.version_of_primary) - if fm_path is None or not fm_path.exists(): - raise FeatureModelNotFound(project, fm_path) - - self.set_cflags( - "-fvara-feature", - f"-fvara-fm-path={fm_path.absolute()}", - "-fsanitize=vara", - "-fvara-instr=trace_event", - # Temporary. Instructions to enable instrumentation placement optimization - "-mllvm", - "--vara-optimizer-policy=naive", + # Add the required runtime extensions to the project(s). + project.runtime_extension = ( + run.RuntimeExtension(project, self) << time.RunWithTime() ) - return super().actions_for_project(project) + # Add the required compiler extensions to the project(s). + project.compiler_extension = ( + compiler.RunCompiler(project, self) << run.WithTimeout() + ) + # Add own error handler to compile step. + project.compile = get_default_compile_error_wrapped( + self.get_handle(), project, TimeReport + ) -class RunUntraced(BaseRunner, shorthand="RU"): - """Build and run the untraced version of the binary""" + measurement_reps = 5 - NAME = "RunUntraced" + analysis_actions = [] - REPORT_SPEC = ReportSpecification(TimeReport) + analysis_actions.append(actions.Compile(project)) - def __init__(self, *args, **kwargs): - super().__init__(TimeReport, TimeBinary, *args, **kwargs) + for binary in project.binaries: + result_filepath = create_new_success_result_filepath( + self.get_handle(), + self.get_handle().report_spec().main_report, project, binary + ) + analysis_actions.append( + ZippedExperimentSteps( + result_filepath, [ + TimeProjectWorkloads(project, num, binary) + for num in range(measurement_reps) + ] + ) + ) - def actions_for_project(self, project: Project) -> tp.MutableSequence[actions.Step]: - """ - Returns the specified steps to run the project(s) specified in the call - in a fixed order. + analysis_actions.append(actions.Clean(project)) - Args: - project: to analyze - """ - return super().actions_for_project(project) + return analysis_actions diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py index ef50e68d9..c2c865685 100644 --- a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -53,6 +53,21 @@ class FeaturePerfCSCollection(VProject): SourceRoot("FeaturePerfCSCollection") / RSBinary("MultiSharedMultipleRegions"), label="MSMR-no-input" + ), + Command( + SourceRoot("FeaturePerfCSCollection") / + RSBinary("SimpleSleepLoop"), "--iterations", "10000", "--sleepns", "10000", + label="SSL-10K-10K" + ), + Command( + SourceRoot("FeaturePerfCSCollection") / + RSBinary("SimpleBusyLoop"), "--iterations", "10000", "--count_to", "1000000", + label="SBL-10K-1M" + ), + Command( + SourceRoot("FeaturePerfCSCollection") / + RSBinary("SingleLocalMultipleRegions"), + label="SLMR-not-slow" ) ] } diff --git a/varats/varats/tools/bb_config.py b/varats/varats/tools/bb_config.py index 6d4085b8e..5da783597 100644 --- a/varats/varats/tools/bb_config.py +++ b/varats/varats/tools/bb_config.py @@ -110,7 +110,6 @@ def create_new_bb_config( projects_conf.value[:] = [] projects_conf.value[:] += [ 'varats.experiments.base.just_compile', - 'varats.experiments.base.time_workload', 'varats.experiments.base.time_workloads', 'varats.experiments.phasar.global_analysis_compare', 'varats.experiments.phasar.ide_linear_constant_experiment', @@ -119,6 +118,7 @@ def create_new_bb_config( 'varats.experiments.vara.blame_report_experiment', 'varats.experiments.vara.blame_verifier_experiment', 'varats.experiments.vara.commit_report_experiment', + 'varats.experiments.vara.compare_traced_untraced', 'varats.experiments.vara.feature_perf_runner', 'varats.experiments.vara.instrumentation_point_printer', 'varats.experiments.vara.instrumentation_stats', From 125970a4c535e408c97c8e7a8a7ef3af7d58e5c8 Mon Sep 17 00:00:00 2001 From: Daniel Gusenburger Date: Mon, 10 Apr 2023 10:37:15 +0200 Subject: [PATCH 22/31] added workloads to bzip2 and lrzip, rewrote instrumentation stats to use bb workloads --- .../vara/compare_traced_untraced.py | 192 ++++-------------- .../experiments/vara/instrumentation_stats.py | 77 +++---- varats/varats/projects/c_projects/bzip2.py | 27 +++ varats/varats/projects/c_projects/lrzip.py | 2 + .../perf_tests/feature_perf_cs_collection.py | 36 ++-- 5 files changed, 126 insertions(+), 208 deletions(-) diff --git a/varats/varats/experiments/vara/compare_traced_untraced.py b/varats/varats/experiments/vara/compare_traced_untraced.py index 90323fb70..5b66ec9f0 100644 --- a/varats/varats/experiments/vara/compare_traced_untraced.py +++ b/varats/varats/experiments/vara/compare_traced_untraced.py @@ -1,201 +1,93 @@ -"""Module for feature performance eperiments that instrument and measure the +"""Module for feature performance experiments that instrument and measure the execution performance of each binary that is produced by a project.""" +from abc import abstractmethod import typing as tp -from benchbuild.extensions import compiler, run, time from benchbuild.utils import actions from varats.experiment.experiment_util import ( - create_new_success_result_filepath, get_default_compile_error_wrapped, - get_default_compile_error_wrapped, ZippedExperimentSteps + create_new_success_result_filepath, ZippedExperimentSteps ) from varats.report.report import ReportSpecification -from varats.report.gnu_time_report import TimeReport -from varats.experiments.vara.feature_experiment import FeatureExperiment from varats.experiments.base.time_workloads import TimeProjectWorkloads +from varats.report.gnu_time_report import WLTimeReportAggregate from varats.project.varats_project import VProject +from varats.experiments.vara.dynamic_overhead_analysis import Runner -class RunTracedUnoptimized(FeatureExperiment, shorthand="RTUnopt"): - """Build and run the traced version of the binary""" - - NAME = "RunTracedUnoptimized" - REPORT_SPEC = ReportSpecification(TimeReport) - - def actions_for_project( - self, project: VProject - ) -> tp.MutableSequence[actions.Step]: - """ - Returns the specified steps to run the project(s) specified in the call - in a fixed order. - - Args: - project: to analyze - """ - instr_type = "trace_event" # trace_event - - project.cflags += self.get_vara_feature_cflags(project) - - project.cflags += self.get_vara_tracing_cflags(instr_type) - - project.ldflags += self.get_vara_tracing_ldflags() - - # Add the required runtime extensions to the project(s). - project.runtime_extension = ( - run.RuntimeExtension(project, self) << time.RunWithTime() - ) - - # Add the required compiler extensions to the project(s). - project.compiler_extension = ( - compiler.RunCompiler(project, self) << run.WithTimeout() - ) - - # Add own error handler to compile step. - project.compile = get_default_compile_error_wrapped( - self.get_handle(), project, TimeReport - ) +MEASUREMENT_REPS = 5 - measurement_reps = 5 - analysis_actions = [] +class RunUntraced(Runner, shorthand="RU"): + """Build and run the untraced version of the binary""" - analysis_actions.append(actions.Compile(project)) + NAME = "RunUntraced" + REPORT_SPEC = ReportSpecification(WLTimeReportAggregate) + def get_analysis_actions( + self, project: VProject + ) -> tp.MutableSequence[actions.Step]: + actions = [] for binary in project.binaries: result_filepath = create_new_success_result_filepath( self.get_handle(), self.get_handle().report_spec().main_report, project, binary ) - analysis_actions.append( + actions.append( ZippedExperimentSteps( result_filepath, [ TimeProjectWorkloads(project, num, binary) - for num in range(measurement_reps) + for num in range(MEASUREMENT_REPS) ] ) ) - analysis_actions.append(actions.Clean(project)) + return actions - return analysis_actions - -class RunTracedNaive(FeatureExperiment, shorthand="RTNaive"): +class RunTraced(RunUntraced, shorthand="RT"): """Build and run the traced version of the binary""" - NAME = "RunTracedNaiveOptimization" - REPORT_SPEC = ReportSpecification(TimeReport) + NAME = "RunTraced" - def actions_for_project( - self, project: VProject - ) -> tp.MutableSequence[actions.Step]: - """ - Returns the specified steps to run the project(s) specified in the call - in a fixed order. + @property + @abstractmethod + def optimizer_policy(self): + return "none" - Args: - project: to analyze - """ + def set_vara_flags(self, project: VProject) -> VProject: instr_type = "trace_event" # trace_event project.cflags += self.get_vara_feature_cflags(project) project.cflags += self.get_vara_tracing_cflags(instr_type) - project.cflags += ["-mllvm", "--vara-optimizer-policy=naive"] + project.cflags += [ + "-mllvm", f"-vara-optimizer-policy={self.optimizer_policy}" + ] project.ldflags += self.get_vara_tracing_ldflags() - # Add the required runtime extensions to the project(s). - project.runtime_extension = ( - run.RuntimeExtension(project, self) << time.RunWithTime() - ) - - # Add the required compiler extensions to the project(s). - project.compiler_extension = ( - compiler.RunCompiler(project, self) << run.WithTimeout() - ) + return project - # Add own error handler to compile step. - project.compile = get_default_compile_error_wrapped( - self.get_handle(), project, TimeReport - ) - measurement_reps = 5 - - analysis_actions = [] - - analysis_actions.append(actions.Compile(project)) - for binary in project.binaries: - result_filepath = create_new_success_result_filepath( - self.get_handle(), - self.get_handle().report_spec().main_report, project, binary - ) - analysis_actions.append( - ZippedExperimentSteps( - result_filepath, [ - TimeProjectWorkloads(project, num, binary) - for num in range(measurement_reps) - ] - ) - ) - analysis_actions.append(actions.Clean(project)) - - return analysis_actions - - -class RunUntraced(FeatureExperiment, shorthand="RU"): - """Build and run the untraced version of the binary""" - - NAME = "RunUntraced" - - REPORT_SPEC = ReportSpecification(TimeReport) - - def actions_for_project( - self, project: VProject - ) -> tp.MutableSequence[actions.Step]: - """ - Returns the specified steps to run the project(s) specified in the call - in a fixed order. - - Args: - project: to analyze - """ - # Add the required runtime extensions to the project(s). - project.runtime_extension = ( - run.RuntimeExtension(project, self) << time.RunWithTime() - ) - - # Add the required compiler extensions to the project(s). - project.compiler_extension = ( - compiler.RunCompiler(project, self) << run.WithTimeout() - ) - - # Add own error handler to compile step. - project.compile = get_default_compile_error_wrapped( - self.get_handle(), project, TimeReport - ) +class RunTracedNaive(RunTraced, shorthand=RunTraced.SHORTHAND + "N"): + """Build and run the traced version of the binary""" - measurement_reps = 5 + NAME = "RunTracedNaive" - analysis_actions = [] + @property + @abstractmethod + def optimizer_policy(self): + return "naive" - analysis_actions.append(actions.Compile(project)) - for binary in project.binaries: - result_filepath = create_new_success_result_filepath( - self.get_handle(), - self.get_handle().report_spec().main_report, project, binary - ) - analysis_actions.append( - ZippedExperimentSteps( - result_filepath, [ - TimeProjectWorkloads(project, num, binary) - for num in range(measurement_reps) - ] - ) - ) +class RunTracedAlternating(RunTraced, shorthand=RunTraced.SHORTHAND + "A"): + """Build and run the traced version of the binary""" - analysis_actions.append(actions.Clean(project)) + NAME = "RunTracedAlternating" - return analysis_actions + @property + @abstractmethod + def optimizer_policy(self): + return "alternating" diff --git a/varats/varats/experiments/vara/instrumentation_stats.py b/varats/varats/experiments/vara/instrumentation_stats.py index 4bde4c8a7..1e907b84e 100644 --- a/varats/varats/experiments/vara/instrumentation_stats.py +++ b/varats/varats/experiments/vara/instrumentation_stats.py @@ -12,6 +12,8 @@ from plumbum import BG, FG, local from plumbum.commands.modifiers import Future +from varats.experiment.workload_util import WorkloadCategory, workload_commands + from varats.data.reports.usdt_stats_report import VaraInstrumentationStatsReport from varats.experiment.experiment_util import ( ExperimentHandle, @@ -58,59 +60,46 @@ def run(self) -> actions.StepResult: if binary.type != BinaryType.EXECUTABLE: continue - # Get workload to use. - # TODO (se-sic/VaRA#841): refactor to bb workloads if possible - workload_provider = WorkloadProvider.create_provider_for_project( - self.project - ) - if not workload_provider: - print( - f"No workload provider for project={self.project.name}. " \ - "Skipping." - ) - return actions.StepResult.CAN_CONTINUE - workload = workload_provider.get_workload_for_binary(binary.name) - if workload is None: - print( - f"No workload for project={self.project.name} " \ - f"binary={binary.name}. Skipping." + for workload in workload_commands( + self.project, binary, + [WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL] + ): + # Assemble Path for report. + report_file_name = create_new_success_result_filepath( + self.__experiment_handle, VaraInstrumentationStatsReport, + self.project, binary ) - continue - # Assemble Path for report. - report_file_name = create_new_success_result_filepath( - self.__experiment_handle, VaraInstrumentationStatsReport, - self.project, binary - ) + report_file = Path(vara_result_folder, str(report_file_name)) - report_file = Path(vara_result_folder, str(report_file_name)) + # Execute binary. + with local.cwd(self.project.builddir): + run_cmd = workload.command.as_plumbum(project=self.project) - # Execute binary. - with local.cwd(self.project.source_of_primary): - run_cmd = binary[workload] - - # attach bpftrace to binary to allow tracing it via USDT - bpftrace_script = Path( - VaRA.install_location(), - "share/vara/perf_bpf_tracing/UsdtExecutionStats.bt" - ) + # attach bpftrace to binary to allow tracing it via USDT + bpftrace_script = Path( + VaRA.install_location(), + "share/vara/perf_bpf_tracing/UsdtExecutionStats.bt" + ) - # Assertion: Can be run without sudo password prompt. To - # guarentee this, add an entry to /etc/sudoers. - bpftrace_cmd = bpftrace["-o", report_file, bpftrace_script, - binary.path] + # Assertion: Can be run without sudo password prompt. To + # guarentee this, add an entry to /etc/sudoers. + # bpftrace_cmd = bpftrace["-o", report_file, bpftrace_script, + # binary.path] + bpftrace_cmd = bpftrace["-o", report_file, bpftrace_script, + workload.path] - bpftrace_runner: Future - with local.as_root(): - bpftrace_runner = bpftrace_cmd & BG + bpftrace_runner: Future + with local.as_root(): + bpftrace_runner = bpftrace_cmd & BG - sleep(3) # give bpftrace time to start up + sleep(3) # give bpftrace time to start up - # Run. - run_cmd & FG # pylint: disable=W0104 + # Run. + run_cmd & FG # pylint: disable=W0104 - # Wait for bpftrace running in background to exit. - bpftrace_runner.wait() + # Wait for bpftrace running in background to exit. + bpftrace_runner.wait() return actions.StepResult.OK diff --git a/varats/varats/projects/c_projects/bzip2.py b/varats/varats/projects/c_projects/bzip2.py index 89af8eb03..b758aaae0 100644 --- a/varats/varats/projects/c_projects/bzip2.py +++ b/varats/varats/projects/c_projects/bzip2.py @@ -3,6 +3,7 @@ import benchbuild as bb from benchbuild.utils.cmd import mkdir, cmake +from benchbuild.source import HTTP from benchbuild.utils.settings import get_number_of_jobs from plumbum import local @@ -18,6 +19,9 @@ from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap from varats.utils.settings import bb_cfg +from varats.experiment.workload_util import RSBinary, WorkloadCategory +from benchbuild.command import Command, SourceRoot, WorkloadSet + class Bzip2(VProject): """Compression and decompression tool bzip2 (fetched by Git)""" @@ -34,9 +38,32 @@ class Bzip2(VProject): refspec="origin/HEAD", limit=None, shallow=False + ), + HTTP( + local="countries-land-1km.geo.json", + remote={ + "1.0": + "https://github.com/simonepri/geo-maps/releases/" + "download/v0.6.0/countries-land-1km.geo.json" + } ) ] + WORKLOADS = { + WorkloadSet(WorkloadCategory.EXAMPLE): [ + Command( + SourceRoot("bzip2") / RSBinary("bzip2"), + "--compress", + "--best", + "--verbose", + "--keep", + "--force", + "countries-land-1km.geo.json", + label="BZ2-Countries" + ), + ] + } + @staticmethod def binaries_for_revision( revision: ShortCommitHash diff --git a/varats/varats/projects/c_projects/lrzip.py b/varats/varats/projects/c_projects/lrzip.py index ce1cccb48..e9ddca593 100644 --- a/varats/varats/projects/c_projects/lrzip.py +++ b/varats/varats/projects/c_projects/lrzip.py @@ -7,6 +7,7 @@ from benchbuild.utils.cmd import make from benchbuild.utils.settings import get_number_of_jobs from plumbum import local +from varats.project.sources import FeatureSource from varats.containers.containers import get_base_image, ImageBase from varats.experiment.workload_util import RSBinary, WorkloadCategory @@ -39,6 +40,7 @@ class Lrzip(VProject): limit=None, shallow=False ), + FeatureSource(), # TODO: auto unzipper for BB? HTTP( local="countries-land-1km.geo.json", diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py index c2c865685..1b67a55f9 100644 --- a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -54,15 +54,23 @@ class FeaturePerfCSCollection(VProject): RSBinary("MultiSharedMultipleRegions"), label="MSMR-no-input" ), + # Command( + # SourceRoot("FeaturePerfCSCollection") / + # RSBinary("SimpleSleepLoop"), + # "--iterations", + # "10000", + # "--sleepns", + # "10000", + # label="SSL-10K-10K" + # ), Command( SourceRoot("FeaturePerfCSCollection") / - RSBinary("SimpleSleepLoop"), "--iterations", "10000", "--sleepns", "10000", - label="SSL-10K-10K" - ), - Command( - SourceRoot("FeaturePerfCSCollection") / - RSBinary("SimpleBusyLoop"), "--iterations", "10000", "--count_to", "1000000", - label="SBL-10K-1M" + RSBinary("SimpleBusyLoop"), + "--iterations", + "100", + "--count_to", + "1000", + label="SBL-100-1000" ), Command( SourceRoot("FeaturePerfCSCollection") / @@ -88,13 +96,13 @@ def binaries_for_revision( BinaryType.EXECUTABLE, only_valid_in=RevisionRange("162db88346", "master") ) - binary_map.specify_binary( - "build/bin/SimpleSleepLoop", - BinaryType.EXECUTABLE, - only_valid_in=RevisionRange( - "c77bca4c6888970fb721069c82455137943ccf49", "master" - ) - ) + # binary_map.specify_binary( + # "build/bin/SimpleSleepLoop", + # BinaryType.EXECUTABLE, + # only_valid_in=RevisionRange( + # "c77bca4c6888970fb721069c82455137943ccf49", "master" + # ) + # ) binary_map.specify_binary( "build/bin/SimpleBusyLoop", BinaryType.EXECUTABLE, From 4f018c1721122728257910631f48897b753730a6 Mon Sep 17 00:00:00 2001 From: Daniel Gusenburger Date: Mon, 10 Apr 2023 10:37:28 +0200 Subject: [PATCH 23/31] added dynamic overhead analysis (WIP) --- .../data/reports/dynamic_overhead_report.py | 53 +++++ .../vara/dynamic_overhead_analysis.py | 182 ++++++++++++++++++ varats/varats/plots/compare_traced.py | 91 +++++++++ varats/varats/plots/dynamic_overhead_plot.py | 70 +++++++ 4 files changed, 396 insertions(+) create mode 100644 varats/varats/data/reports/dynamic_overhead_report.py create mode 100644 varats/varats/experiments/vara/dynamic_overhead_analysis.py create mode 100644 varats/varats/plots/compare_traced.py create mode 100644 varats/varats/plots/dynamic_overhead_plot.py diff --git a/varats/varats/data/reports/dynamic_overhead_report.py b/varats/varats/data/reports/dynamic_overhead_report.py new file mode 100644 index 000000000..58dd4b2a7 --- /dev/null +++ b/varats/varats/data/reports/dynamic_overhead_report.py @@ -0,0 +1,53 @@ +from varats.report.report import BaseReport +from pathlib import Path +from collections import defaultdict + + +class DynamicOverheadReport( + BaseReport, shorthand="DynOverhead", file_type="txt" +): + + class RegionCounter: + + def __init__(self): + self.__in = 0 + self.__out = 0 + + def enter(self): + self.__in += 1 + + def leave(self): + self.__out += 1 + + def isvalid(self): + return self.__in == self.__out + + def count_visited(self): + return self.__in + + def __init__(self, path: Path): + super().__init__(path) + self.__entries = defaultdict(DynamicOverheadReport.RegionCounter) + + for line in open(path, "r"): + try: + command, id = line.split() + if command == "Entering": + self.__entries[id].enter() + elif command == "Leaving": + self.__entries[id].leave() + except ValueError: + continue + + self.__total_region_count = 0 + + # Generate report + for region in self.__entries.values(): + if region.isvalid(): + self.__total_region_count += region.count_visited() + + def isvalid(self) -> bool: + return all(v.isvalid() for v in self.__entries.values()) + + def regions_visited(self): + return self.__total_region_count diff --git a/varats/varats/experiments/vara/dynamic_overhead_analysis.py b/varats/varats/experiments/vara/dynamic_overhead_analysis.py new file mode 100644 index 000000000..05cbec31f --- /dev/null +++ b/varats/varats/experiments/vara/dynamic_overhead_analysis.py @@ -0,0 +1,182 @@ +from abc import abstractmethod +import typing as tp +import textwrap + +from plumbum import local + +from benchbuild import Project +from benchbuild.extensions import compiler, run, time +from benchbuild.utils import actions + +from varats.experiment.experiment_util import ( + create_new_success_result_filepath, get_default_compile_error_wrapped, + ExperimentHandle, exec_func_with_pe_error_handler, + create_default_analysis_failure_handler +) +from varats.experiment.workload_util import ( + workload_commands, + WorkloadCategory, +) +from varats.report.report import ReportSpecification +from varats.experiment.workload_util import WorkloadCategory, workload_commands +from varats.experiments.vara.feature_experiment import FeatureExperiment, ProjectStep +from varats.data.reports.dynamic_overhead_report import DynamicOverheadReport +from varats.provider.workload.workload_provider import WorkloadProvider + +from varats.project.project_util import ProjectBinaryWrapper + +from varats.project.varats_project import VProject + +from varats.ts_utils.cli_util import tee + + +class Runner(FeatureExperiment, shorthand="R"): + + @abstractmethod + def set_vara_flags(self, project: VProject) -> VProject: + return project + + @abstractmethod + def get_analysis_actions( + self, project: VProject + ) -> tp.MutableSequence[actions.Step]: + return [] + + def actions_for_project( + self, project: VProject + ) -> tp.MutableSequence[actions.Step]: + self.set_vara_flags(project) + + # Add the required runtime extensions to the project(s). + project.runtime_extension = ( + run.RuntimeExtension(project, self) << time.RunWithTime() + ) + + # Add the required compiler extensions to the project(s). + project.compiler_extension = ( + compiler.RunCompiler(project, self) << run.WithTimeout() + ) + + # Add own error handler to compile step. + project.compile = get_default_compile_error_wrapped( + self.get_handle(), project, self.REPORT_SPEC.main_report + ) + + analysis_actions = [] + + analysis_actions.append(actions.Compile(project)) + + analysis_actions.extend(self.get_analysis_actions(project)) + + analysis_actions.append(actions.Clean(project)) + + return analysis_actions + + +class DynamicOverheadAnalysis(ProjectStep): + + def __init__( + self, project: Project, experiment: ExperimentHandle, + binary: ProjectBinaryWrapper + ): + super().__init__(project=project) + self.__binary = binary + self.__experiment_handle = experiment + + def __call__(self) -> actions.StepResult: + return self.analyze() + + def __str__(self, indent: int = 0) -> str: + return textwrap.indent( + f"* Compute dynamic overhead for binary {self.__binary.path}", + indent * " " + ) + + def analyze(self) -> actions.StepResult: + """Only create a report file.""" + with local.cwd(self.project.builddir): + for prj_command in workload_commands( + self.project, self.__binary, + [WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL] + ): + result_file = create_new_success_result_filepath( + self.__experiment_handle, DynamicOverheadReport, + self.project, self.__binary + ) + pb_cmd = prj_command.command.as_plumbum(project=self.project) + print("##################", pb_cmd) + + run_cmd = pb_cmd > str(result_file) + + run_cmd() + + # exec_func_with_pe_error_handler( + # run_cmd, + # create_default_analysis_failure_handler( + # self.__experiment_handle, self.project, + # DynamicOverheadReport + # ) + # ) + + return actions.StepResult.OK + + +class DynamicOverheadRunner(Runner, shorthand="CDO"): + NAME = "ComputeDynamicOverhead" + REPORT_SPEC = ReportSpecification(DynamicOverheadReport) + + @property + @abstractmethod + def optimizer_policy(self) -> str: + return "none" + + def get_analysis_actions( + self, project: VProject + ) -> tp.MutableSequence[actions.Step]: + + return [ + DynamicOverheadAnalysis(project, self.get_handle(), binary) + for binary in project.binaries + if binary.type == BinaryType.EXECUTABLE + ] + + def __str__(self, indent: int = 0) -> str: + return textwrap.indent( + f"* Compute dynamic overhead with policy {self.optimizer_policy} for binary {self.__binary.path}", + indent * " " + ) + + def set_vara_flags(self, project: VProject) -> VProject: + instr_type = "print" + + project.cflags += self.get_vara_feature_cflags(project) + + project.cflags += self.get_vara_tracing_cflags(instr_type) + + project.cflags += [ + "-mllvm", f"-vara-optimizer-policy={self.optimizer_policy}" + ] + + project.ldflags += self.get_vara_tracing_ldflags() + + return project + + +class DynamicOverheadOptimizedNaive( + DynamicOverheadRunner, shorthand=DynamicOverheadRunner.SHORTHAND + "N" +): + NAME = "ComputeDynamicOverheadNaive" + + @property + def optimizer_policy(self) -> str: + return "naive" + + +class DynamicOverheadOptimizedAlternating( + DynamicOverheadRunner, shorthand=DynamicOverheadRunner.SHORTHAND + "A" +): + NAME = "ComputeDynamicOverheadAlternating" + + @property + def optimizer_policy(self) -> str: + return "alternating" diff --git a/varats/varats/plots/compare_traced.py b/varats/varats/plots/compare_traced.py new file mode 100644 index 000000000..921a25779 --- /dev/null +++ b/varats/varats/plots/compare_traced.py @@ -0,0 +1,91 @@ +"""Example table that uses different workloads and visualizes the time it took +to run them.""" +import typing as tp + +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sns +import numpy as np + +from varats.paper.paper_config import get_loaded_paper_config +from varats.paper_mgmt.case_study import get_case_study_file_name_filter +from varats.plot.plot import Plot +from varats.plot.plots import PlotGenerator +from varats.report.gnu_time_report import WLTimeReportAggregate +from varats.revision.revisions import get_processed_revisions_files +from varats.ts_utils.click_param_types import REQUIRE_MULTI_EXPERIMENT_TYPE +from varats.utils.git_util import FullCommitHash + +# TODO: Is there a better way to include revisions of all workloads than to use +# only_newest=False ? +# Maybe the result files are not defined correctly. We should be able to find +# the revision files for all workloads with only_newest=True... + + +class CompareRuntimesPlot(Plot, plot_name="compare_runtimes"): + + def plot(self, view_mode: bool) -> None: + case_studies = get_loaded_paper_config().get_all_case_studies() + + df = pd.DataFrame() + + for case_study in case_studies: + project_name = case_study.project_name + + for experiment in self.plot_kwargs["experiment_type"]: + report_files = get_processed_revisions_files( + project_name, + experiment, + WLTimeReportAggregate, + get_case_study_file_name_filter(case_study), + only_newest=False + ) + + for report_filepath in report_files: + agg_time_report = WLTimeReportAggregate( + report_filepath.full_path() + ) + report_file = agg_time_report.filename + + for workload_name in agg_time_report.workload_names(): + for wall_clock_time in \ + agg_time_report.measurements_wall_clock_time( + workload_name + ): + new_row = { + "Binary": + report_file.binary_name, + "Experiment": + experiment.NAME, + "Mean wall time (msecs)": + wall_clock_time * 1000, + } + + df = df.append(new_row, ignore_index=True) + + fig, ax = plt.subplots() + fig.set_size_inches(11.7, 8.27) + sns.barplot( + x="Binary", + y="Mean wall time (msecs)", + hue="Experiment", + estimator=np.mean, + data=df, + ax=ax, + ) + sns.despine() + + def calc_missing_revisions( + self, boundary_gradient: float + ) -> tp.Set[FullCommitHash]: + raise NotImplementedError + + +class CompareRuntimesPlotGenerator( + PlotGenerator, + generator_name="compare-runtimes", + options=[REQUIRE_MULTI_EXPERIMENT_TYPE] +): + + def generate(self) -> tp.List[Plot]: + return [CompareRuntimesPlot(self.plot_config, **self.plot_kwargs)] diff --git a/varats/varats/plots/dynamic_overhead_plot.py b/varats/varats/plots/dynamic_overhead_plot.py new file mode 100644 index 000000000..1fa9fa251 --- /dev/null +++ b/varats/varats/plots/dynamic_overhead_plot.py @@ -0,0 +1,70 @@ +import typing as tp + +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sns + +from varats.paper.paper_config import get_loaded_paper_config +from varats.paper_mgmt.case_study import get_case_study_file_name_filter +from varats.plot.plot import Plot +from varats.plot.plots import PlotGenerator +from varats.data.reports.dynamic_overhead_report import DynamicOverheadReport +from varats.revision.revisions import get_processed_revisions_files +from varats.ts_utils.click_param_types import REQUIRE_MULTI_EXPERIMENT_TYPE +from varats.utils.git_util import FullCommitHash + + +class DynamicOverheadPlot(Plot, plot_name="dynamic_overhead"): + + def plot(self, view_mode: bool) -> None: + case_studies = get_loaded_paper_config().get_all_case_studies() + + df = pd.DataFrame() + + for case_study in case_studies: + project_name = case_study.project_name + + for experiment in self.plot_kwargs["experiment_type"]: + report_files = get_processed_revisions_files( + project_name, + experiment, + DynamicOverheadReport, + get_case_study_file_name_filter(case_study), + only_newest=False + ) + + for report_filepath in report_files: + report = DynamicOverheadReport(report_filepath.full_path()) + + new_row = { + "Name": report.filename.binary_name, + "Visited regions": report.regions_visited(), + } + + df = df.append(new_row, ignore_index=True) + + fig, ax = plt.subplots() + fig.set_size_inches(11.7, 8.27) + sns.barplot( + x="Name", + y="Visited regions", + hue="Name", + data=df, + ax=ax, + ) + sns.despine() + + def calc_missing_revisions( + self, boundary_gradient: float + ) -> tp.Set[FullCommitHash]: + raise NotImplementedError + + +class DynamicOverheadPlotGenerator( + PlotGenerator, + generator_name="dynamic-overhead", + options=[REQUIRE_MULTI_EXPERIMENT_TYPE] +): + + def generate(self) -> tp.List[Plot]: + return [DynamicOverheadPlot(self.plot_config, **self.plot_kwargs)] From 9b3d995161c5d536ed6d13f7ed9af9fd537fc3e6 Mon Sep 17 00:00:00 2001 From: Daniel Gusenburger Date: Thu, 4 May 2023 11:59:48 +0200 Subject: [PATCH 24/31] implemented multicompileexperiment --- varats-core/varats/report/gnu_time_report.py | 2 +- .../varats/experiments/base/time_workloads.py | 22 +- .../vara/compare_traced_untraced.py | 80 +++++--- .../vara/dynamic_overhead_analysis.py | 189 ++---------------- .../experiments/vara/feature_experiment.py | 15 +- .../experiments/vara/feature_tracing_stats.py | 87 ++++---- .../vara/instrumentation_verifier.py | 3 +- .../vara/multi_compile_experiment.py | 140 +++++++++++++ varats/varats/plot/plot.py | 6 +- varats/varats/plots/compare_traced.py | 7 +- varats/varats/plots/compare_traced_budget.py | 90 +++++++++ varats/varats/plots/compare_traced_cs.py | 92 +++++++++ varats/varats/plots/dynamic_overhead_plot.py | 5 +- ...n_verifier_compare_experiments_overview.py | 164 +++++++++++++++ varats/varats/projects/c_projects/gzip.py | 6 +- varats/varats/projects/c_projects/picosat.py | 67 ++++++- .../tables/instrumentation_verifier_table.py | 52 ++--- varats/varats/tools/driver_run.py | 9 +- 18 files changed, 758 insertions(+), 278 deletions(-) create mode 100644 varats/varats/experiments/vara/multi_compile_experiment.py create mode 100644 varats/varats/plots/compare_traced_budget.py create mode 100644 varats/varats/plots/compare_traced_cs.py create mode 100644 varats/varats/plots/instrumentation_verifier_compare_experiments_overview.py diff --git a/varats-core/varats/report/gnu_time_report.py b/varats-core/varats/report/gnu_time_report.py index 88200ba52..124929602 100644 --- a/varats-core/varats/report/gnu_time_report.py +++ b/varats-core/varats/report/gnu_time_report.py @@ -284,7 +284,7 @@ def summary(self) -> str: class WLTimeReportAggregate( WorkloadSpecificReportAggregate[TimeReport], - shorthand=TimeReport.SHORTHAND + ReportAggregate.SHORTHAND, + shorthand="WL" + TimeReport.SHORTHAND + ReportAggregate.SHORTHAND, file_type=ReportAggregate.FILE_TYPE ): """Context Manager for parsing multiple time reports stored inside a zip diff --git a/varats/varats/experiments/base/time_workloads.py b/varats/varats/experiments/base/time_workloads.py index 19d2c70af..26c4f6ee7 100644 --- a/varats/varats/experiments/base/time_workloads.py +++ b/varats/varats/experiments/base/time_workloads.py @@ -1,6 +1,7 @@ """Implements an experiment that times the execution of all project binaries.""" import typing as tp +import textwrap from pathlib import Path from benchbuild import Project @@ -36,11 +37,16 @@ class TimeProjectWorkloads(actions.ProjectStep): # type: ignore project: VProject def __init__( - self, project: Project, num: int, binary: ProjectBinaryWrapper + self, + project: Project, + num: int, + binary: ProjectBinaryWrapper, + categories: tp.List[WorkloadCategory] = [WorkloadCategory.EXAMPLE] ): super().__init__(project=project) self.__num = num self.__binary = binary + self.__workload_categories = categories def __call__(self, tmp_dir: Path) -> actions.StepResult: return self.analyze(tmp_dir) @@ -50,8 +56,9 @@ def analyze(self, tmp_dir: Path) -> actions.StepResult: with local.cwd(self.project.builddir): for prj_command in workload_commands( - self.project, self.__binary, [WorkloadCategory.EXAMPLE] + self.project, self.__binary, self.__workload_categories ): + print("Running workload") pb_cmd = prj_command.command.as_plumbum(project=self.project) run_report_name = tmp_dir / create_workload_specific_filename( @@ -60,11 +67,20 @@ def analyze(self, tmp_dir: Path) -> actions.StepResult: run_cmd = time['-v', '-o', f'{run_report_name}', pb_cmd] + print("\t", run_cmd) + with cleanup(prj_command): - run_cmd() + run_cmd(retcode=self.__binary.valid_exit_codes) return actions.StepResult.OK + def __str__(self, indent: int = 0) -> str: + return textwrap.indent( + f"* Run workloads of categories {', '.join(str(x) for x in self.__workload_categories)} " + f"for binary {self.__binary.name} ({self.__num})", + indent * " " + ) + class TimeWorkloads(VersionExperiment, shorthand="TWL"): """Generates time report files.""" diff --git a/varats/varats/experiments/vara/compare_traced_untraced.py b/varats/varats/experiments/vara/compare_traced_untraced.py index 5b66ec9f0..3aeee9b46 100644 --- a/varats/varats/experiments/vara/compare_traced_untraced.py +++ b/varats/varats/experiments/vara/compare_traced_untraced.py @@ -8,27 +8,32 @@ from varats.experiment.experiment_util import ( create_new_success_result_filepath, ZippedExperimentSteps ) +from varats.experiment.workload_util import WorkloadCategory from varats.report.report import ReportSpecification from varats.experiments.base.time_workloads import TimeProjectWorkloads from varats.report.gnu_time_report import WLTimeReportAggregate from varats.project.varats_project import VProject -from varats.experiments.vara.dynamic_overhead_analysis import Runner +from varats.experiments.vara.dynamic_overhead_analysis import OptimizerPolicyType +from varats.experiments.vara.feature_experiment import FeatureExperiment, FeatureInstrType +from varats.experiments.vara.multi_compile_experiment import VaryingStartingBudgetExperiment -MEASUREMENT_REPS = 5 +MEASUREMENT_REPS = 1 -class RunUntraced(Runner, shorthand="RU"): +class RunUntraced(FeatureExperiment, shorthand="RU"): """Build and run the untraced version of the binary""" NAME = "RunUntraced" + REPORT_SPEC = ReportSpecification(WLTimeReportAggregate) - def get_analysis_actions( + def actions_for_project( self, project: VProject ) -> tp.MutableSequence[actions.Step]: actions = [] + for binary in project.binaries: result_filepath = create_new_success_result_filepath( self.get_handle(), @@ -37,38 +42,67 @@ def get_analysis_actions( actions.append( ZippedExperimentSteps( result_filepath, [ - TimeProjectWorkloads(project, num, binary) - for num in range(MEASUREMENT_REPS) + TimeProjectWorkloads( + project, + num, + binary, + categories=[ + WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL + ] + ) for num in range(MEASUREMENT_REPS) ] ) ) - return actions + + return self.get_common_tracing_actions( + project, FeatureInstrType.NONE, actions, save_temps=True + ) -class RunTraced(RunUntraced, shorthand="RT"): +class RunTraced(VaryingStartingBudgetExperiment, shorthand="RT"): """Build and run the traced version of the binary""" NAME = "RunTraced" + REPORT_SPEC = ReportSpecification(WLTimeReportAggregate) @property @abstractmethod - def optimizer_policy(self): - return "none" - - def set_vara_flags(self, project: VProject) -> VProject: - instr_type = "trace_event" # trace_event + def optimizer_policy(self) -> OptimizerPolicyType: + return OptimizerPolicyType.NONE - project.cflags += self.get_vara_feature_cflags(project) - - project.cflags += self.get_vara_tracing_cflags(instr_type) + def actions_for_project( + self, project: VProject + ) -> tp.MutableSequence[actions.Step]: project.cflags += [ - "-mllvm", f"-vara-optimizer-policy={self.optimizer_policy}" + "-mllvm", + f"-vara-optimizer-policy={self.optimizer_policy.value}", ] - project.ldflags += self.get_vara_tracing_ldflags() + actions = [] + for binary in project.binaries: + result_filepath = create_new_success_result_filepath( + self.get_handle(), + self.get_handle().report_spec().main_report, project, binary + ) + actions.append( + ZippedExperimentSteps( + result_filepath, [ + TimeProjectWorkloads( + project, + num, + binary, + categories=[ + WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL + ] + ) for num in range(MEASUREMENT_REPS) + ] + ) + ) - return project + return self.get_common_tracing_actions( + project, FeatureInstrType.TEF, actions, save_temps=True + ) class RunTracedNaive(RunTraced, shorthand=RunTraced.SHORTHAND + "N"): @@ -78,8 +112,8 @@ class RunTracedNaive(RunTraced, shorthand=RunTraced.SHORTHAND + "N"): @property @abstractmethod - def optimizer_policy(self): - return "naive" + def optimizer_policy(self) -> OptimizerPolicyType: + return OptimizerPolicyType.NAIVE class RunTracedAlternating(RunTraced, shorthand=RunTraced.SHORTHAND + "A"): @@ -89,5 +123,5 @@ class RunTracedAlternating(RunTraced, shorthand=RunTraced.SHORTHAND + "A"): @property @abstractmethod - def optimizer_policy(self): - return "alternating" + def optimizer_policy(self) -> OptimizerPolicyType: + return OptimizerPolicyType.ALTERNATING diff --git a/varats/varats/experiments/vara/dynamic_overhead_analysis.py b/varats/varats/experiments/vara/dynamic_overhead_analysis.py index 05cbec31f..277a1beeb 100644 --- a/varats/varats/experiments/vara/dynamic_overhead_analysis.py +++ b/varats/varats/experiments/vara/dynamic_overhead_analysis.py @@ -1,182 +1,33 @@ -from abc import abstractmethod -import typing as tp -import textwrap - -from plumbum import local - -from benchbuild import Project -from benchbuild.extensions import compiler, run, time -from benchbuild.utils import actions - -from varats.experiment.experiment_util import ( - create_new_success_result_filepath, get_default_compile_error_wrapped, - ExperimentHandle, exec_func_with_pe_error_handler, - create_default_analysis_failure_handler -) -from varats.experiment.workload_util import ( - workload_commands, - WorkloadCategory, -) -from varats.report.report import ReportSpecification -from varats.experiment.workload_util import WorkloadCategory, workload_commands -from varats.experiments.vara.feature_experiment import FeatureExperiment, ProjectStep -from varats.data.reports.dynamic_overhead_report import DynamicOverheadReport -from varats.provider.workload.workload_provider import WorkloadProvider - -from varats.project.project_util import ProjectBinaryWrapper +from enum import Enum +from typing import MutableSequence +from varats.experiment.experiment_util import Step from varats.project.varats_project import VProject -from varats.ts_utils.cli_util import tee - - -class Runner(FeatureExperiment, shorthand="R"): - - @abstractmethod - def set_vara_flags(self, project: VProject) -> VProject: - return project - - @abstractmethod - def get_analysis_actions( - self, project: VProject - ) -> tp.MutableSequence[actions.Step]: - return [] - - def actions_for_project( - self, project: VProject - ) -> tp.MutableSequence[actions.Step]: - self.set_vara_flags(project) - - # Add the required runtime extensions to the project(s). - project.runtime_extension = ( - run.RuntimeExtension(project, self) << time.RunWithTime() - ) - - # Add the required compiler extensions to the project(s). - project.compiler_extension = ( - compiler.RunCompiler(project, self) << run.WithTimeout() - ) - - # Add own error handler to compile step. - project.compile = get_default_compile_error_wrapped( - self.get_handle(), project, self.REPORT_SPEC.main_report - ) - - analysis_actions = [] - - analysis_actions.append(actions.Compile(project)) - - analysis_actions.extend(self.get_analysis_actions(project)) - - analysis_actions.append(actions.Clean(project)) - - return analysis_actions - - -class DynamicOverheadAnalysis(ProjectStep): - - def __init__( - self, project: Project, experiment: ExperimentHandle, - binary: ProjectBinaryWrapper - ): - super().__init__(project=project) - self.__binary = binary - self.__experiment_handle = experiment - - def __call__(self) -> actions.StepResult: - return self.analyze() - - def __str__(self, indent: int = 0) -> str: - return textwrap.indent( - f"* Compute dynamic overhead for binary {self.__binary.path}", - indent * " " - ) - - def analyze(self) -> actions.StepResult: - """Only create a report file.""" - with local.cwd(self.project.builddir): - for prj_command in workload_commands( - self.project, self.__binary, - [WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL] - ): - result_file = create_new_success_result_filepath( - self.__experiment_handle, DynamicOverheadReport, - self.project, self.__binary - ) - pb_cmd = prj_command.command.as_plumbum(project=self.project) - print("##################", pb_cmd) - - run_cmd = pb_cmd > str(result_file) - - run_cmd() - - # exec_func_with_pe_error_handler( - # run_cmd, - # create_default_analysis_failure_handler( - # self.__experiment_handle, self.project, - # DynamicOverheadReport - # ) - # ) - - return actions.StepResult.OK - - -class DynamicOverheadRunner(Runner, shorthand="CDO"): - NAME = "ComputeDynamicOverhead" - REPORT_SPEC = ReportSpecification(DynamicOverheadReport) - - @property - @abstractmethod - def optimizer_policy(self) -> str: - return "none" - - def get_analysis_actions( - self, project: VProject - ) -> tp.MutableSequence[actions.Step]: - - return [ - DynamicOverheadAnalysis(project, self.get_handle(), binary) - for binary in project.binaries - if binary.type == BinaryType.EXECUTABLE - ] - - def __str__(self, indent: int = 0) -> str: - return textwrap.indent( - f"* Compute dynamic overhead with policy {self.optimizer_policy} for binary {self.__binary.path}", - indent * " " - ) - - def set_vara_flags(self, project: VProject) -> VProject: - instr_type = "print" - - project.cflags += self.get_vara_feature_cflags(project) - - project.cflags += self.get_vara_tracing_cflags(instr_type) - - project.cflags += [ - "-mllvm", f"-vara-optimizer-policy={self.optimizer_policy}" - ] +from varats.experiments.vara.instrumentation_verifier import RunInstrVerifier - project.ldflags += self.get_vara_tracing_ldflags() - return project +class OptimizerPolicyType(Enum): + NONE = "none" + NAIVE = "naive" + ALTERNATING = "alternating" -class DynamicOverheadOptimizedNaive( - DynamicOverheadRunner, shorthand=DynamicOverheadRunner.SHORTHAND + "N" +class RunInstrVerifierNaive( + RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "N" ): - NAME = "ComputeDynamicOverheadNaive" + NAME = "RunInstrVerifierNaive" - @property - def optimizer_policy(self) -> str: - return "naive" + def actions_for_project(self, project: VProject) -> MutableSequence[Step]: + project.cflags += ["-mllvm", f"-vara-optimizer-policy=naive"] + return super().actions_for_project(project) -class DynamicOverheadOptimizedAlternating( - DynamicOverheadRunner, shorthand=DynamicOverheadRunner.SHORTHAND + "A" +class RunInstrVerifierAlternating( + RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "A" ): - NAME = "ComputeDynamicOverheadAlternating" + NAME = "RunInstrVerifierAlternating" - @property - def optimizer_policy(self) -> str: - return "alternating" + def actions_for_project(self, project: VProject) -> MutableSequence[Step]: + project.cflags += ["-mllvm", f"-vara-optimizer-policy=alternating"] + return super().actions_for_project(project) diff --git a/varats/varats/experiments/vara/feature_experiment.py b/varats/varats/experiments/vara/feature_experiment.py index 52d2ffa16..d6bca45c6 100644 --- a/varats/varats/experiments/vara/feature_experiment.py +++ b/varats/varats/experiments/vara/feature_experiment.py @@ -184,8 +184,10 @@ def get_vara_tracing_cflags( if instr_type != FeatureInstrType.NONE: c_flags += ["-fsanitize=vara", f"-fvara-instr={instr_type.value}"] c_flags += [ - "-flto", "-fuse-ld=lld", "-flegacy-pass-manager", - "-fno-omit-frame-pointer" + "-flto", + "-fuse-ld=lld", + "-flegacy-pass-manager", + "-fno-omit-frame-pointer", ] if instruction_threshold is not None: # For test projects, do not exclude small regions @@ -193,6 +195,7 @@ def get_vara_tracing_cflags( instruction_threshold = 1 c_flags += [f"-fvara-instruction-threshold={instruction_threshold}"] + if save_temps: c_flags += ["-Wl,-plugin-opt=save-temps"] @@ -220,11 +223,15 @@ def __init__( self, project: VProject, experiment_handle: ExperimentHandle, - report_file_ending: str = "json" + report_file_ending: str = "json", + workload_categories: tp.List[WorkloadCategory] = [ + WorkloadCategory.EXAMPLE + ] ): super().__init__(project=project) self.__experiment_handle = experiment_handle self.__report_file_ending = report_file_ending + self.__workload_categories = workload_categories def __call__(self) -> StepResult: return self.run_traced_code() @@ -250,7 +257,7 @@ def run_traced_code(self) -> StepResult: with local.cwd(local.path(self.project.builddir)): with ZippedReportFolder(result_filepath.full_path()) as tmp_dir: for prj_command in workload_commands( - self.project, binary, [WorkloadCategory.EXAMPLE] + self.project, binary, self.__workload_categories ): local_tracefile_path = Path( tmp_dir diff --git a/varats/varats/experiments/vara/feature_tracing_stats.py b/varats/varats/experiments/vara/feature_tracing_stats.py index 873236204..3273facc3 100644 --- a/varats/varats/experiments/vara/feature_tracing_stats.py +++ b/varats/varats/experiments/vara/feature_tracing_stats.py @@ -11,6 +11,8 @@ from benchbuild.utils import actions from benchbuild.utils.cmd import bpftrace, sudo from plumbum import BG + +from plumbum import local from plumbum.commands.modifiers import Future from varats.data.reports.feature_tracing_stats_report import ( @@ -57,50 +59,51 @@ def __call__(self) -> actions.StepResult: continue # get workload to use - workloads = workload_commands( - self.project, binary, [WorkloadCategory.MEDIUM] - ) - if len(workloads) == 0: - print( - f"No workload for project={self.project.name} " - f"binary={binary.name}. Skipping." + with local.cwd(local.path(self.project.builddir)): + workloads = workload_commands( + self.project, binary, [WorkloadCategory.SMALL, WorkloadCategory.EXAMPLE] ) - continue - if len(workloads) > 1: - raise RuntimeError( - "Currently, only a single workload is supported. " - f"project={self.project.name} binary={binary.name}" + if len(workloads) == 0: + print( + f"No workload for project={self.project.name} " + f"binary={binary.name}. Skipping." + ) + continue + if len(workloads) > 1: + raise RuntimeError( + "Currently, only a single workload is supported. " + f"project={self.project.name} binary={binary.name}" + ) + workload = workloads[0] + + # report path + stats_report = create_new_success_result_filepath( + self.__experiment_handle, FeatureTracingStatsReport, + self.project, binary ) - workload = workloads[0] - - # report path - stats_report = create_new_success_result_filepath( - self.__experiment_handle, FeatureTracingStatsReport, - self.project, binary - ) - - # attach bpftrace script - bpftrace_script = Path( - VaRA.install_location(), - "share/vara/perf_bpf_tracing/UsdtExecutionStats.bt" - ) - - # assertion: Can be run without sudo password prompt - bpftrace_cmd = bpftrace["-f", "json", "-o", stats_report, - bpftrace_script, - self.project.source_of_primary / - binary.path] - bpftrace_cmd = sudo[bpftrace_cmd] - bpftrace_runner: Future = bpftrace_cmd & BG - sleep(3) # give bpftrace time to start up - - # execute binary with workload - run_cmd = workload.command.as_plumbum(project=self.project) - with cleanup(workload): - bb.watch(run_cmd)() - - # Wait for bpftrace running in background to exit. - bpftrace_runner.wait() + + # attach bpftrace script + bpftrace_script = Path( + VaRA.install_location(), + "share/vara/perf_bpf_tracing/UsdtExecutionStats.bt" + ) + + # assertion: Can be run without sudo password prompt + bpftrace_cmd = bpftrace["-f", "json", "-o", stats_report, + bpftrace_script, + self.project.source_of_primary / + binary.path] + bpftrace_cmd = sudo[bpftrace_cmd] + bpftrace_runner: Future = bpftrace_cmd & BG + sleep(3) # give bpftrace time to start up + + # execute binary with workload + run_cmd = workload.command.as_plumbum(project=self.project) + with cleanup(workload): + bb.watch(run_cmd)(retcode=binary.valid_exit_codes) + + # Wait for bpftrace running in background to exit. + bpftrace_runner.wait() return actions.StepResult.OK diff --git a/varats/varats/experiments/vara/instrumentation_verifier.py b/varats/varats/experiments/vara/instrumentation_verifier.py index ed54cf05f..f278dc4fb 100644 --- a/varats/varats/experiments/vara/instrumentation_verifier.py +++ b/varats/varats/experiments/vara/instrumentation_verifier.py @@ -4,6 +4,7 @@ from benchbuild.extensions import compiler, run from benchbuild.utils import actions +from varats.experiment.workload_util import WorkloadCategory from varats.data.reports.instrumentation_verifier_report import ( InstrVerifierReport, @@ -69,7 +70,7 @@ def actions_for_project( analysis_actions.append(actions.Compile(project)) analysis_actions.append( RunVaRATracedWorkloads( - project, self.get_handle(), report_file_ending="ivr" + project, self.get_handle(), report_file_ending="ivr", workload_categories=[WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL] ) ) analysis_actions.append(actions.Clean(project)) diff --git a/varats/varats/experiments/vara/multi_compile_experiment.py b/varats/varats/experiments/vara/multi_compile_experiment.py new file mode 100644 index 000000000..111fff951 --- /dev/null +++ b/varats/varats/experiments/vara/multi_compile_experiment.py @@ -0,0 +1,140 @@ +"""Base class experiment and utilities for experiments that work with +features.""" +import typing as tp +from abc import abstractmethod + +from benchbuild.utils.actions import ( + Step, +) +from benchbuild.project import build_dir +import benchbuild.utils.actions as actns +from varats.experiment.experiment_util import Project +from varats.project.varats_project import VProject +from varats.report.report import ReportSpecification +from varats.experiments.vara.feature_experiment import FeatureExperiment +from benchbuild.experiment import Actions + + +class Flags: + + def __init__( + self, + cflags: tp.List[str] | None = None, + ldflags: tp.List[str] | None = None, + result_folder_name: str | None = None + ): + self.__cflags = cflags or [] + self.__ldflags = ldflags or [] + self.__result_folder_name = result_folder_name + + @property + def cflags(self) -> tp.List[str]: + return self.__cflags + + @property + def ldflags(self) -> tp.List[str]: + return self.__ldflags + + @property + def result_folder_name(self) -> str | None: + return self.__result_folder_name + + def __str__(self): + return f"Flags(cflags={self.cflags}, ldflags={self.ldflags}, result_folder_name={self.result_folder_name})" + + __repr__ = __str__ + + +class MultiCompileExperiment(FeatureExperiment, shorthand=""): + """Base class experiment for feature specific experiments.""" + + NAME = "MultiCompileExperiment" + + REPORT_SPEC = ReportSpecification() + + @abstractmethod + def actions_for_project(self, + project: VProject) -> tp.MutableSequence[Step]: + """Get the actions a project wants to run.""" + + def get_flags(self) -> tp.List[Flags]: + """Get a list of flags that should be changed for every compilation attempt""" + return [Flags()] + + def actions(self) -> Actions: + actions: Actions = [] + + # TODO: Add additional echo to provide more information about the current run + def new_actions(self, proj: Project, flags: Flags) -> Actions: + atomic_actions: Actions = [ + tp.cast(Step, actns.Clean(proj)), + actns.MakeBuildDir(proj), + actns.Echo( + message=f"Selected {proj.name} with version {version_str}" + ), + ] + if flags.cflags: + atomic_actions.append( + actns.Echo(message=f"Set additional cflags {flags.cflags}") + ) + if flags.ldflags: + atomic_actions.append( + actns.Echo( + message=f"Set additional ldflags {flags.ldflags}" + ) + ) + if flags.result_folder_name: + atomic_actions.append( + actns.Echo( + message= + f"Set result folder name override {flags.result_folder_name}" + ) + ) + atomic_actions.append(actns.ProjectEnvironment(proj)) + atomic_actions.extend(self.actions_for_project(proj)) + return [tp.cast(Step, actns.RequireAll(actions=atomic_actions))] + + for prj_cls in self.projects: + prj_actions: Actions = [] + + for revision in self.sample(prj_cls): + version_str = str(revision) + + p = prj_cls(revision) + + for flags in self.get_flags(): + p_clone = p.clone() + + p_clone.cflags = flags.cflags + p_clone.ldflags = flags.ldflags + result_folder = flags.result_folder_name or str(p.run_uuid) + p_clone.builddir = build_dir(self, p_clone) / result_folder + + prj_actions = new_actions(self, p_clone, flags) + actions.extend(prj_actions) + + if actions: + actions.append(actns.CleanExtra()) + + return actions + + +class VaryingStartingBudgetExperiment(MultiCompileExperiment, shorthand=""): + NAME = "VaryingStartingBudgetExperiment" + + REPORT_SPEC = ReportSpecification() + + @abstractmethod + def actions_for_project(self, + project: VProject) -> tp.MutableSequence[Step]: + """Get the actions a project wants to run.""" + + def get_flags(self) -> tp.List[Flags]: + flags = [] + for budget in range(20, 110, 10): + f = Flags( + cflags=["-mllvm", f"-vara-optimizer-starting-budget={budget}"], + result_folder_name=f"starting_budget_{budget}" + ) + flags.append(f) + return flags diff --git a/varats/varats/plot/plot.py b/varats/varats/plot/plot.py index bc77f1d34..240960bfe 100644 --- a/varats/varats/plot/plot.py +++ b/varats/varats/plot/plot.py @@ -149,8 +149,10 @@ def plot_file_name(self, filetype: str) -> str: """ plot_ident = '' if 'case_study' in self.plot_kwargs: - case_study: 'CaseStudy' = self.plot_kwargs['case_study'] - plot_ident = f"{case_study.project_name}_{case_study.version}_" + case_study: tp.Union[tp.List['CaseStudy'], + 'CaseStudy'] = self.plot_kwargs['case_study'] + if not isinstance(case_study, list): + plot_ident = f"{case_study.project_name}_{case_study.version}_" elif 'project' in self.plot_kwargs: plot_ident = f"{self.plot_kwargs['project']}_" diff --git a/varats/varats/plots/compare_traced.py b/varats/varats/plots/compare_traced.py index 921a25779..d57d5bd5d 100644 --- a/varats/varats/plots/compare_traced.py +++ b/varats/varats/plots/compare_traced.py @@ -31,8 +31,10 @@ def plot(self, view_mode: bool) -> None: for case_study in case_studies: project_name = case_study.project_name + print(project_name) for experiment in self.plot_kwargs["experiment_type"]: + print(experiment.NAME) report_files = get_processed_revisions_files( project_name, experiment, @@ -48,6 +50,7 @@ def plot(self, view_mode: bool) -> None: report_file = agg_time_report.filename for workload_name in agg_time_report.workload_names(): + print(workload_name) for wall_clock_time in \ agg_time_report.measurements_wall_clock_time( workload_name @@ -61,7 +64,9 @@ def plot(self, view_mode: bool) -> None: wall_clock_time * 1000, } - df = df.append(new_row, ignore_index=True) + df = pd.concat([df, pd.DataFrame([new_row])], + ignore_index=True) + # df = df.append(new_row, ignore_index=True) fig, ax = plt.subplots() fig.set_size_inches(11.7, 8.27) diff --git a/varats/varats/plots/compare_traced_budget.py b/varats/varats/plots/compare_traced_budget.py new file mode 100644 index 000000000..180cefb6e --- /dev/null +++ b/varats/varats/plots/compare_traced_budget.py @@ -0,0 +1,90 @@ +"""Example table that uses different workloads and visualizes the time it took +to run them.""" +import typing as tp +import re + +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sns +import numpy as np + +from varats.paper_mgmt.case_study import get_case_study_file_name_filter +from varats.plot.plot import Plot +from varats.plot.plots import PlotGenerator +from varats.report.gnu_time_report import WLTimeReportAggregate +from varats.revision.revisions import get_processed_revisions_files +from varats.ts_utils.click_param_types import REQUIRE_MULTI_CASE_STUDY, REQUIRE_EXPERIMENT_TYPE +from varats.utils.git_util import FullCommitHash + +starting_budget_command_regex = re.compile("starting_budget_([0-9]*)") + + +class CompareRuntimesBudgetPlot(Plot, plot_name="compare_runtimes_budget"): + + def plot(self, view_mode: bool) -> None: + df = pd.DataFrame() + + for case_study in self.plot_kwargs["case_study"]: + project_name = case_study.project_name + + experiment = self.plot_kwargs["experiment_type"] + report_files = get_processed_revisions_files( + project_name, + experiment, + WLTimeReportAggregate, + get_case_study_file_name_filter(case_study), + only_newest=False + ) + + for report_filepath in report_files: + agg_time_report = WLTimeReportAggregate( + report_filepath.full_path() + ) + + for workload_name in agg_time_report.workload_names(): + for report in agg_time_report.reports(workload_name): + m = re.search( + starting_budget_command_regex, report.command_name + ) + if m is None: + budget = 20 + else: + budget = int(m.group(1)) + + new_row = { + "Workload": + workload_name, + "Budget": + budget, + "Mean wall time (msecs)": + report.wall_clock_time.total_seconds() + } + + df = pd.concat([df, pd.DataFrame([new_row])], + ignore_index=True) + + fig, ax = plt.subplots() + fig.set_size_inches(11.7, 8.27) + sns.barplot( + x="Budget", + y="Mean wall time (msecs)", + estimator=np.mean, + data=df, + ax=ax, + ) + sns.despine() + + def calc_missing_revisions( + self, boundary_gradient: float + ) -> tp.Set[FullCommitHash]: + raise NotImplementedError + + +class CompareRuntimesBudgetPlotCSGenerator( + PlotGenerator, + generator_name="compare-runtimes-budget", + options=[REQUIRE_EXPERIMENT_TYPE, REQUIRE_MULTI_CASE_STUDY] +): + + def generate(self) -> tp.List[Plot]: + return [CompareRuntimesBudgetPlot(self.plot_config, **self.plot_kwargs)] diff --git a/varats/varats/plots/compare_traced_cs.py b/varats/varats/plots/compare_traced_cs.py new file mode 100644 index 000000000..f64753f4c --- /dev/null +++ b/varats/varats/plots/compare_traced_cs.py @@ -0,0 +1,92 @@ +"""Example table that uses different workloads and visualizes the time it took +to run them.""" +import typing as tp + +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sns +import numpy as np + +from varats.paper.paper_config import get_loaded_paper_config +from varats.paper_mgmt.case_study import get_case_study_file_name_filter +from varats.plot.plot import Plot +from varats.plot.plots import PlotGenerator +from varats.report.gnu_time_report import WLTimeReportAggregate +from varats.revision.revisions import get_processed_revisions_files +from varats.ts_utils.click_param_types import REQUIRE_MULTI_CASE_STUDY, REQUIRE_MULTI_EXPERIMENT_TYPE +from varats.utils.git_util import FullCommitHash + +# TODO: Is there a better way to include revisions of all workloads than to use +# only_newest=False ? +# Maybe the result files are not defined correctly. We should be able to find +# the revision files for all workloads with only_newest=True... + + +class CompareRuntimesCSPlot(Plot, plot_name="compare_runtimes_cs"): + + def plot(self, view_mode: bool) -> None: + df = pd.DataFrame() + + print(self.plot_kwargs["case_study"]) + for case_study in self.plot_kwargs["case_study"]: + project_name = case_study.project_name + + for experiment in self.plot_kwargs["experiment_type"]: + report_files = get_processed_revisions_files( + project_name, + experiment, + WLTimeReportAggregate, + get_case_study_file_name_filter(case_study), + only_newest=False + ) + + for report_filepath in report_files: + agg_time_report = WLTimeReportAggregate( + report_filepath.full_path() + ) + report_file = agg_time_report.filename + + for workload_name in agg_time_report.workload_names(): + for wall_clock_time in \ + agg_time_report.measurements_wall_clock_time( + workload_name + ): + new_row = { + "Workload": + workload_name, + "Experiment": + experiment.NAME, + "Mean wall time (msecs)": + wall_clock_time * 1000, + } + + df = pd.concat([df, pd.DataFrame([new_row])], + ignore_index=True) + + print(df) + fig, ax = plt.subplots() + fig.set_size_inches(11.7, 8.27) + sns.barplot( + x="Workload", + y="Mean wall time (msecs)", + hue="Experiment", + estimator=np.mean, + data=df, + ax=ax, + ) + sns.despine() + + def calc_missing_revisions( + self, boundary_gradient: float + ) -> tp.Set[FullCommitHash]: + raise NotImplementedError + + +class CompareRuntimesPlotCSGenerator( + PlotGenerator, + generator_name="compare-runtimes-cs", + options=[REQUIRE_MULTI_EXPERIMENT_TYPE, REQUIRE_MULTI_CASE_STUDY] +): + + def generate(self) -> tp.List[Plot]: + return [CompareRuntimesCSPlot(self.plot_config, **self.plot_kwargs)] diff --git a/varats/varats/plots/dynamic_overhead_plot.py b/varats/varats/plots/dynamic_overhead_plot.py index 1fa9fa251..54caea5e6 100644 --- a/varats/varats/plots/dynamic_overhead_plot.py +++ b/varats/varats/plots/dynamic_overhead_plot.py @@ -25,6 +25,7 @@ def plot(self, view_mode: bool) -> None: project_name = case_study.project_name for experiment in self.plot_kwargs["experiment_type"]: + report_files = get_processed_revisions_files( project_name, experiment, @@ -33,6 +34,7 @@ def plot(self, view_mode: bool) -> None: only_newest=False ) + for report_filepath in report_files: report = DynamicOverheadReport(report_filepath.full_path()) @@ -41,7 +43,8 @@ def plot(self, view_mode: bool) -> None: "Visited regions": report.regions_visited(), } - df = df.append(new_row, ignore_index=True) + df = pd.concat([df, pd.DataFrame([new_row])], + ignore_index=True) fig, ax = plt.subplots() fig.set_size_inches(11.7, 8.27) diff --git a/varats/varats/plots/instrumentation_verifier_compare_experiments_overview.py b/varats/varats/plots/instrumentation_verifier_compare_experiments_overview.py new file mode 100644 index 000000000..4e7a29337 --- /dev/null +++ b/varats/varats/plots/instrumentation_verifier_compare_experiments_overview.py @@ -0,0 +1,164 @@ +"""Generate graphs that show an overview of the instrumentation verifier +experiment state for all case studies in the paper config.""" + +import typing as tp + +import matplotlib.pyplot as plt +from matplotlib import ticker +import numpy as np + +import varats.paper.paper_config as PC +from varats.data.reports.instrumentation_verifier_report import ( + InstrVerifierReport, +) +from varats.plot.plot import Plot, PlotDataEmpty +from varats.plot.plots import PlotGenerator +from varats.report.report import ReportFilepath +from varats.revision.revisions import get_all_revisions_files +from varats.ts_utils.click_param_types import REQUIRE_MULTI_EXPERIMENT_TYPE +from varats.utils.exceptions import UnsupportedOperation +from varats.utils.git_util import FullCommitHash +import itertools + + +class InstrumentationOverviewCompareExperimentsPlot( + Plot, plot_name="instrumentation_overview_compare_experiments_plot" +): + """ + Plot configuration for the instrumentation verifier experiment. + + This plot shows an overview of the instrumentation verifier state for all + case studies in the paper config. + """ + + def plot(self, view_mode: bool) -> None: + self._generate_plot(**self.plot_kwargs) + + def calc_missing_revisions( + self, boundary_gradient: float + ) -> tp.Set[FullCommitHash]: + raise UnsupportedOperation + + @staticmethod + def _generate_plot(**kwargs: tp.Any) -> None: + case_study = kwargs['case_study'] + + width = 0.25 + multiplicator = 0 + + minor_labels = [] + minor_ticks = [] + + _, ax = plt.subplots() + + for experiment in kwargs["experiment_type"]: + + revisions_files: tp.List[ReportFilepath] = get_all_revisions_files( + case_study.project_name, experiment, only_newest=False + ) + + labels: tp.List[str] = [] + + reports: tp.List[InstrVerifierReport] = [ + InstrVerifierReport(rev_file.full_path()) + for rev_file in revisions_files + ] + + if len(reports) == 0: + raise PlotDataEmpty() + + num_enters: tp.List[int] = [] + num_leaves: tp.List[int] = [] + num_unclosed_enters: tp.List[int] = [] + num_unentered_leaves: tp.List[int] = [] + + for report in reports: + for binary in report.binaries(): + labels.append(f"{binary}") + num_enters.append(report.num_enters(binary),) + num_leaves.append(report.num_leaves(binary),) + num_unclosed_enters.append( + report.num_unclosed_enters(binary), + ) + num_unentered_leaves.append( + report.num_unentered_leaves(binary), + ) + + minor_labels.extend([ + x + "-" + y + for x, y in zip(labels, itertools.repeat(experiment.NAME)) + ]) + ind = np.arange(len(num_enters)) + offset = width * multiplicator + + ax.bar( + ind + offset, + num_enters, + width, + color="tab:blue", + edgecolor="black" + ) + ax.bar( + ind + offset, + num_leaves, + width, + color="tab:orange", + bottom=num_enters, + edgecolor="black" + ) + ax.bar( + ind + offset, + num_unclosed_enters, + width, + color="tab:cyan", + edgecolor="black", + bottom=[a + b for a, b in zip(num_enters, num_leaves)] + ) + ax.bar( + ind + offset, + num_unentered_leaves, + width, + color="tab:olive", + edgecolor="black", + bottom=[ + a + b + c for a, b, c in + zip(num_enters, num_leaves, num_unclosed_enters) + ] + ) + + minor_ticks.extend(ind + offset) + multiplicator += 1 + + ax.set_ylabel("Number of events") + ax.set_title( + f"Instrumentation Verifier " + f"Overview for {case_study.project_name}" + ) + ax.legend() + # ax.set_xticks(ind + width, labels=labels, rotation=30, ha="right") + + print(minor_labels) + + ax.set_xticks( + minor_ticks, + labels=minor_labels, + rotation=30, + ha="right", + ) + + plt.subplots_adjust(bottom=0.25) + + +class VerifierExperimentCompareOverviewGenerator( + PlotGenerator, + generator_name="iv-ce-overview-plot", + options=[REQUIRE_MULTI_EXPERIMENT_TYPE] +): + """Generates a single pc-overview plot for the current paper config.""" + + def generate(self) -> tp.List[Plot]: + return [ + InstrumentationOverviewCompareExperimentsPlot( + self.plot_config, case_study=cs, **self.plot_kwargs + ) for cs in PC.get_paper_config().get_all_case_studies() + ] diff --git a/varats/varats/projects/c_projects/gzip.py b/varats/varats/projects/c_projects/gzip.py index f3b2a106d..e2d9f03e3 100644 --- a/varats/varats/projects/c_projects/gzip.py +++ b/varats/varats/projects/c_projects/gzip.py @@ -106,8 +106,10 @@ class Gzip(VProject, ReleaseProviderHook): "--best", "--force", # needed because BB creates symlinks for the inputs "geo-maps/countries-land-1m.geo.json", - label="geo-maps/countries-land-1m", - creates=["geo-maps/countries-land-1m.geo.json.gz"] + label="countries-land-1m", + creates=[ + "geo-maps/countries-land-1m.geo.json.gz" + ] ) ], } diff --git a/varats/varats/projects/c_projects/picosat.py b/varats/varats/projects/c_projects/picosat.py index 103f9b814..d971a554e 100644 --- a/varats/varats/projects/c_projects/picosat.py +++ b/varats/varats/projects/c_projects/picosat.py @@ -5,6 +5,7 @@ import benchbuild as bb from benchbuild.command import WorkloadSet, Command, SourceRoot from benchbuild.source import HTTP +from benchbuild.source.http import HTTPUntar from benchbuild.utils.cmd import make from benchbuild.utils.settings import get_number_of_jobs from plumbum import local @@ -58,9 +59,42 @@ class PicoSAT(VProject, ReleaseProviderHook): "https://github.com/se-sic/picoSAT-mirror/releases/" "download/picoSAT-965/example.cnf" } - ) + ), + HTTPUntar( + local="abw-N-bcsstk07.mtx-w44.cnf", + remote={ + "1.0": + "https://github.com/se-sic/picoSAT-mirror/releases/" + "download/picoSAT-965/abw-N-bcsstk07.mtx-w44.cnf.tar.gz" + } + ), + HTTPUntar( + local="traffic_kkb_unknown.cnf", + remote={ + "1.0": + "https://github.com/se-sic/picoSAT-mirror/releases/" + "download/picoSAT-965/traffic_kkb_unknown.cnf.tar.gz" + } + ), + HTTPUntar( + local="UNSAT_H_instances_childsnack_p11.hddl_1.cnf", + remote={ + "1.0": + "https://github.com/se-sic/picoSAT-mirror/releases/" + "download/picoSAT-965/UNSAT_H_instances_childsnack_p11.hddl_1.cnf.tar.gz" + } + ), + HTTPUntar( + local="UNSAT_H_instances_childsnack_p12.hddl_1.cnf", + remote={ + "1.0": + "https://github.com/se-sic/picoSAT-mirror/releases/" + "download/picoSAT-965/UNSAT_H_instances_childsnack_p12.hddl_1.cnf.tar.gz" + } + ), ] + # TODO: Not sure about the categories here WORKLOADS = { WorkloadSet(WorkloadCategory.EXAMPLE): [ Command( @@ -69,6 +103,37 @@ class PicoSAT(VProject, ReleaseProviderHook): label="example.cnf", ) ], + WorkloadSet(WorkloadCategory.SMALL): [ + Command( + SourceRoot("picosat") / RSBinary("picosat"), + "aim-100-1_6-no-1.cnf", + label="aim-100-1-6-no-1.cnf", + ) + ], + WorkloadSet(WorkloadCategory.MEDIUM): [ + Command( + SourceRoot("picosat") / RSBinary("picosat"), + "traffic_kkb_unknown.cnf/traffic_kkb_unknown.cnf", + label="traffic-kkb-unknow.cnf", + ), + Command( + SourceRoot("picosat") / RSBinary("picosat"), + "abw-N-bcsstk07.mtx-w44.cnf/abw-N-bcsstk07.mtx-w44.cnf", + label="abw-N-bcsstk07.mtx-w44.cnf", + ), + ], + WorkloadSet(WorkloadCategory.LARGE): [ + Command( + SourceRoot("picosat") / RSBinary("picosat"), + "UNSAT_H_instances_childsnack_p11.hddl_1.cnf/UNSAT_H_instances_childsnack_p11.hddl_1.cnf", + label="UNSAT-H-instances-childsnack-p11.hddl-1.cnf", + ), + Command( + SourceRoot("picosat") / RSBinary("picosat"), + "UNSAT_H_instances_childsnack_p12.hddl_1.cnf/UNSAT_H_instances_childsnack_p12.hddl_1.cnf", + label="UNSAT-H-instances-childsnack-p12.hddl-1.cnf", + ) + ], } @staticmethod diff --git a/varats/varats/tables/instrumentation_verifier_table.py b/varats/varats/tables/instrumentation_verifier_table.py index fe7f058d5..cdfacac0d 100644 --- a/varats/varats/tables/instrumentation_verifier_table.py +++ b/varats/varats/tables/instrumentation_verifier_table.py @@ -15,6 +15,7 @@ from varats.table.table import Table, TableDataEmpty from varats.table.table_utils import dataframe_to_table from varats.table.tables import TableFormat, TableGenerator +from varats.ts_utils.click_param_types import REQUIRE_MULTI_EXPERIMENT_TYPE class InstrumentationVerifierTable( @@ -31,37 +32,38 @@ class InstrumentationVerifierTable( def tabulate(self, table_format: TableFormat, wrap_table: bool) -> str: variables = [ - "Workload name", "State", "ConfigID", "#Enters", "#Leaves", - "#Unclosed Enters", "#Unentered Leaves" + "Experiment type", "Workload name", "State", "ConfigID", "#Enters", + "#Leaves", "#Unclosed Enters", "#Unentered Leaves" ] - experiment_type = RunInstrVerifier + data = [] project_name: str = self.table_kwargs['case_study'].project_name - data = [] + for experiment in self.table_kwargs["experiment_type"]: - revision_files = get_all_revisions_files( - project_name, experiment_type, only_newest=False - ) + revision_files = get_all_revisions_files( + project_name, experiment, only_newest=False + ) - reports = [ - InstrVerifierReport(rev_file.full_path()) - for rev_file in revision_files - ] + reports = [ + InstrVerifierReport(rev_file.full_path()) + for rev_file in revision_files + ] - for report in reports: - for binary in report.binaries(): - data.append([ - f"{report.filename.commit_hash} - {binary}", - report.state(binary), report.filename.config_id, - report.num_enters(binary), - report.num_leaves(binary), - report.num_unclosed_enters(binary), - report.num_unentered_leaves(binary) - ]) + for report in reports: + for binary in report.binaries(): + data.append([ + experiment.NAME, + f"{report.filename.commit_hash} - {binary}", + report.state(binary), report.filename.config_id, + report.num_enters(binary), + report.num_leaves(binary), + report.num_unclosed_enters(binary), + report.num_unentered_leaves(binary) + ]) - if len(data) == 0: - raise TableDataEmpty() + if len(data) == 0: + raise TableDataEmpty() pd_data = pd.DataFrame(columns=variables, data=np.array(data)) @@ -71,7 +73,9 @@ def tabulate(self, table_format: TableFormat, wrap_table: bool) -> str: class InstrVerifierTableGenerator( - TableGenerator, generator_name="instrumentation-verifier-table", options=[] + TableGenerator, + generator_name="instrumentation-verifier-table", + options=[REQUIRE_MULTI_EXPERIMENT_TYPE] ): """Generates an overview table for the instrumentation verifier experiment.""" diff --git a/varats/varats/tools/driver_run.py b/varats/varats/tools/driver_run.py index fb82df69c..37f8a8a1a 100644 --- a/varats/varats/tools/driver_run.py +++ b/varats/varats/tools/driver_run.py @@ -24,7 +24,7 @@ from varats.paper.paper_config import get_paper_config from varats.projects.discover_projects import initialize_projects from varats.ts_utils.cli_util import initialize_cli_tool, tee -from varats.ts_utils.click_param_types import create_experiment_type_choice +from varats.ts_utils.click_param_types import create_multi_experiment_type_choice from varats.utils.exceptions import ConfigurationLookupError from varats.utils.git_util import ShortCommitHash from varats.utils.settings import bb_cfg, vara_cfg @@ -91,7 +91,7 @@ def __validate_project_parameters( @click.option( "-E", "--experiment", - type=create_experiment_type_choice(), + type=create_multi_experiment_type_choice(), required=True, help="The experiment to run." ) @@ -102,7 +102,7 @@ def main( slurm: bool, submit: bool, container: bool, - experiment: tp.Type['VersionExperiment'], + experiment: tp.List[tp.Type['VersionExperiment']], projects: tp.List[str], pretend: bool, ) -> None: @@ -157,7 +157,8 @@ def main( bb_args = list( itertools.chain( - bb_command_args, ["-E", experiment.NAME], projects, bb_extra_args + bb_command_args, *[["-E", e.NAME] for e in experiment], projects, + bb_extra_args ) ) From 99829279382e93439907f81fa57b27c72f535487 Mon Sep 17 00:00:00 2001 From: Daniel Gusenburger Date: Tue, 16 May 2023 10:46:01 +0200 Subject: [PATCH 25/31] added multiple budget experiment to instrumentation verifier --- .../vara/instrumentation_verifier.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/varats/varats/experiments/vara/instrumentation_verifier.py b/varats/varats/experiments/vara/instrumentation_verifier.py index f278dc4fb..02529433f 100644 --- a/varats/varats/experiments/vara/instrumentation_verifier.py +++ b/varats/varats/experiments/vara/instrumentation_verifier.py @@ -20,6 +20,7 @@ ) from varats.project.varats_project import VProject from varats.report.report import ReportSpecification +from varats.experiments.vara.multi_compile_experiment import VaryingStartingBudgetExperiment class RunInstrVerifier(FeatureExperiment, shorthand="RIV"): @@ -76,3 +77,58 @@ def actions_for_project( analysis_actions.append(actions.Clean(project)) return analysis_actions + +class RunInstrVerifierBudget(VaryingStartingBudgetExperiment, shorthand="RIVB"): + """Test runner for feature performance.""" + + NAME = "RunInstrVerifierBudget" + + REPORT_SPEC = ReportSpecification(InstrVerifierReport) + + def actions_for_project( + self, project: VProject + ) -> tp.MutableSequence[actions.Step]: + """ + Returns the specified steps to run the project(s) specified in the call + in a fixed order. + + Args: + project: to analyze + """ + project.cflags += self.get_vara_feature_cflags(project) + + project.cflags += self.get_vara_tracing_cflags( + FeatureInstrType.VERIFY, True + ) + + # Ensure that we detect all regions, when verifying + project.cflags += ["-fvara-instruction-threshold=0"] + + # Add debug information, so traces can be better interpreted + project.cflags += ["-g"] + + project.ldflags += self.get_vara_tracing_ldflags() + + # Add the required runtime extensions to the project(s). + project.runtime_extension = run.RuntimeExtension(project, self) + + # Add the required compiler extensions to the project(s). + project.compiler_extension = compiler.RunCompiler(project, self) \ + << WithUnlimitedStackSize() + + # Add own error handler to compile step. + project.compile = get_default_compile_error_wrapped( + self.get_handle(), project, self.REPORT_SPEC.main_report + ) + + analysis_actions = [] + + analysis_actions.append(actions.Compile(project)) + analysis_actions.append( + RunVaRATracedWorkloads( + project, self.get_handle(), report_file_ending="ivr", workload_categories=[WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL] + ) + ) + analysis_actions.append(actions.Clean(project)) + + return analysis_actions From ff9a904cd71546f62579a5af9e324659b1f52f87 Mon Sep 17 00:00:00 2001 From: Daniel Gusenburger Date: Wed, 17 May 2023 10:55:46 +0200 Subject: [PATCH 26/31] reverse compability to python 3.9.x --- .../varats/experiments/vara/multi_compile_experiment.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/varats/varats/experiments/vara/multi_compile_experiment.py b/varats/varats/experiments/vara/multi_compile_experiment.py index 111fff951..4fe0e7fba 100644 --- a/varats/varats/experiments/vara/multi_compile_experiment.py +++ b/varats/varats/experiments/vara/multi_compile_experiment.py @@ -19,9 +19,9 @@ class Flags: def __init__( self, - cflags: tp.List[str] | None = None, - ldflags: tp.List[str] | None = None, - result_folder_name: str | None = None + cflags: tp.Optional[tp.List[str]] = None, + ldflags: tp.Optional[tp.List[str]] = None, + result_folder_name: tp.Optional[str] = None ): self.__cflags = cflags or [] self.__ldflags = ldflags or [] @@ -36,7 +36,7 @@ def ldflags(self) -> tp.List[str]: return self.__ldflags @property - def result_folder_name(self) -> str | None: + def result_folder_name(self) -> tp.Optional[str]: return self.__result_folder_name def __str__(self): From 11cb37ce34ba15e98dded8123437e1e13dc9a6dc Mon Sep 17 00:00:00 2001 From: Daniel Gusenburger Date: Thu, 18 May 2023 16:15:31 +0200 Subject: [PATCH 27/31] separated implementations for RunTraced for VaryingBudget and regular Feature expperiments --- .../vara/compare_traced_untraced.py | 73 ++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/varats/varats/experiments/vara/compare_traced_untraced.py b/varats/varats/experiments/vara/compare_traced_untraced.py index 3aeee9b46..1a644bd12 100644 --- a/varats/varats/experiments/vara/compare_traced_untraced.py +++ b/varats/varats/experiments/vara/compare_traced_untraced.py @@ -59,7 +59,8 @@ def actions_for_project( ) -class RunTraced(VaryingStartingBudgetExperiment, shorthand="RT"): + +class RunTraced(FeatureExperiment, shorthand="RT"): """Build and run the traced version of the binary""" NAME = "RunTraced" @@ -125,3 +126,73 @@ class RunTracedAlternating(RunTraced, shorthand=RunTraced.SHORTHAND + "A"): @abstractmethod def optimizer_policy(self) -> OptimizerPolicyType: return OptimizerPolicyType.ALTERNATING + + +class RunTracedBudget(VaryingStartingBudgetExperiment, shorthand="RTB"): + """Build and run the traced version of the binary""" + + NAME = "RunTracedBudget" + REPORT_SPEC = ReportSpecification(WLTimeReportAggregate) + + @property + @abstractmethod + def optimizer_policy(self) -> OptimizerPolicyType: + return OptimizerPolicyType.NONE + + def actions_for_project( + self, project: VProject + ) -> tp.MutableSequence[actions.Step]: + + project.cflags += [ + "-mllvm", + f"-vara-optimizer-policy={self.optimizer_policy.value}", + ] + + actions = [] + for binary in project.binaries: + result_filepath = create_new_success_result_filepath( + self.get_handle(), + self.get_handle().report_spec().main_report, project, binary + ) + actions.append( + ZippedExperimentSteps( + result_filepath, [ + TimeProjectWorkloads( + project, + num, + binary, + categories=[ + WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL + ] + ) for num in range(MEASUREMENT_REPS) + ] + ) + ) + + return self.get_common_tracing_actions( + project, FeatureInstrType.TEF, actions, save_temps=True + ) + + + + +class RunTracedNaiveBudget(RunTracedBudget, shorthand=RunTracedBudget.SHORTHAND + "N"): + """Build and run the traced version of the binary""" + + NAME = "RunTracedNaive" + + @property + @abstractmethod + def optimizer_policy(self) -> OptimizerPolicyType: + return OptimizerPolicyType.NAIVE + + +class RunTracedAlternatingBudget(RunTracedBudget, shorthand=RunTracedBudget.SHORTHAND + "A"): + """Build and run the traced version of the binary""" + + NAME = "RunTracedAlternating" + + @property + @abstractmethod + def optimizer_policy(self) -> OptimizerPolicyType: + return OptimizerPolicyType.ALTERNATING From a729e2b8c61a8a1ef0dfc57a8cf8580c8a69cf16 Mon Sep 17 00:00:00 2001 From: Daniel Gusenburger Date: Fri, 9 Jun 2023 10:45:52 +0200 Subject: [PATCH 28/31] InstrVerifierReport should not produce useful results for experiments with varying budgt --- varats-core/varats/report/report.py | 1 - .../instrumentation_verifier_report.py | 12 ++++- .../vara/compare_traced_untraced.py | 12 +---- .../vara/dynamic_overhead_analysis.py | 25 +++++++++- .../experiments/vara/feature_experiment.py | 48 ++++++++++++++++++- .../vara/instrumentation_verifier.py | 8 ++-- .../vara/multi_compile_experiment.py | 41 +++------------- ...n_verifier_compare_experiments_overview.py | 3 +- 8 files changed, 95 insertions(+), 55 deletions(-) diff --git a/varats-core/varats/report/report.py b/varats-core/varats/report/report.py index d454b1ee6..02d11d9d6 100644 --- a/varats-core/varats/report/report.py +++ b/varats-core/varats/report/report.py @@ -655,7 +655,6 @@ def is_correct_report_type(cls, file_name: str) -> bool: except ValueError: return False - class ReportSpecification(): """Groups together multiple report types into a specification that can be used, e.g., by experiments, to request multiple reports.""" diff --git a/varats/varats/data/reports/instrumentation_verifier_report.py b/varats/varats/data/reports/instrumentation_verifier_report.py index 2bb27e98a..1add4f39d 100644 --- a/varats/varats/data/reports/instrumentation_verifier_report.py +++ b/varats/varats/data/reports/instrumentation_verifier_report.py @@ -3,6 +3,7 @@ import typing as tp from pathlib import Path from zipfile import ZipFile +import yaml from varats.report.report import BaseReport @@ -15,10 +16,15 @@ def __init__(self, report_path: Path) -> None: super().__init__(report_path) self.__report_data = {} + self.__metadata = {} with ZipFile(report_path, "r") as archive: for file in archive.namelist(): + if file == "metadata.yml": + with archive.open(file, "r") as f: + self.__metadata = yaml.load(f, yaml.Loader) + if not file.endswith(".ivr"): continue @@ -137,9 +143,13 @@ def num_unentered_leaves(self, binary: str) -> int: def states(self) -> tp.Dict[str, str]: return { - binary: data['state'] # type: ignore + binary: + data['state'] # type: ignore for binary, data in self.__report_data.items() } def state(self, binary: str) -> str: return self.__report_data[binary]['state'] # type: ignore + + def metadata(self) -> tp.Dict[tp.Any, tp.Any]: + return self.__metadata diff --git a/varats/varats/experiments/vara/compare_traced_untraced.py b/varats/varats/experiments/vara/compare_traced_untraced.py index 1a644bd12..962351a92 100644 --- a/varats/varats/experiments/vara/compare_traced_untraced.py +++ b/varats/varats/experiments/vara/compare_traced_untraced.py @@ -19,8 +19,7 @@ from varats.experiments.vara.feature_experiment import FeatureExperiment, FeatureInstrType from varats.experiments.vara.multi_compile_experiment import VaryingStartingBudgetExperiment -MEASUREMENT_REPS = 1 - +MEASUREMENT_REPS = 10 class RunUntraced(FeatureExperiment, shorthand="RU"): """Build and run the untraced version of the binary""" @@ -58,8 +57,6 @@ def actions_for_project( project, FeatureInstrType.NONE, actions, save_temps=True ) - - class RunTraced(FeatureExperiment, shorthand="RT"): """Build and run the traced version of the binary""" @@ -105,7 +102,6 @@ def actions_for_project( project, FeatureInstrType.TEF, actions, save_temps=True ) - class RunTracedNaive(RunTraced, shorthand=RunTraced.SHORTHAND + "N"): """Build and run the traced version of the binary""" @@ -116,7 +112,6 @@ class RunTracedNaive(RunTraced, shorthand=RunTraced.SHORTHAND + "N"): def optimizer_policy(self) -> OptimizerPolicyType: return OptimizerPolicyType.NAIVE - class RunTracedAlternating(RunTraced, shorthand=RunTraced.SHORTHAND + "A"): """Build and run the traced version of the binary""" @@ -127,7 +122,6 @@ class RunTracedAlternating(RunTraced, shorthand=RunTraced.SHORTHAND + "A"): def optimizer_policy(self) -> OptimizerPolicyType: return OptimizerPolicyType.ALTERNATING - class RunTracedBudget(VaryingStartingBudgetExperiment, shorthand="RTB"): """Build and run the traced version of the binary""" @@ -173,9 +167,6 @@ def actions_for_project( project, FeatureInstrType.TEF, actions, save_temps=True ) - - - class RunTracedNaiveBudget(RunTracedBudget, shorthand=RunTracedBudget.SHORTHAND + "N"): """Build and run the traced version of the binary""" @@ -186,7 +177,6 @@ class RunTracedNaiveBudget(RunTracedBudget, shorthand=RunTracedBudget.SHORTHAND def optimizer_policy(self) -> OptimizerPolicyType: return OptimizerPolicyType.NAIVE - class RunTracedAlternatingBudget(RunTracedBudget, shorthand=RunTracedBudget.SHORTHAND + "A"): """Build and run the traced version of the binary""" diff --git a/varats/varats/experiments/vara/dynamic_overhead_analysis.py b/varats/varats/experiments/vara/dynamic_overhead_analysis.py index 277a1beeb..8d8842c6b 100644 --- a/varats/varats/experiments/vara/dynamic_overhead_analysis.py +++ b/varats/varats/experiments/vara/dynamic_overhead_analysis.py @@ -1,10 +1,11 @@ from enum import Enum from typing import MutableSequence from varats.experiment.experiment_util import Step +from varats.experiments.vara.feature_experiment import Flags from varats.project.varats_project import VProject -from varats.experiments.vara.instrumentation_verifier import RunInstrVerifier +from varats.experiments.vara.instrumentation_verifier import RunInstrVerifier, RunInstrVerifierBudget class OptimizerPolicyType(Enum): @@ -31,3 +32,25 @@ class RunInstrVerifierAlternating( def actions_for_project(self, project: VProject) -> MutableSequence[Step]: project.cflags += ["-mllvm", f"-vara-optimizer-policy=alternating"] return super().actions_for_project(project) + + +class RunInstrVerifierNaiveBudget( + RunInstrVerifierBudget, shorthand=RunInstrVerifier.SHORTHAND + "NB" +): + NAME = "RunInstrVerifierNaiveBudget" + + def actions_for_project(self, project: VProject, + flags: Flags) -> MutableSequence[Step]: + project.cflags += ["-mllvm", f"-vara-optimizer-policy=naive"] + return super().actions_for_project(project, flags) + + +class RunInstrVerifierAlternatingBudget( + RunInstrVerifierBudget, shorthand=RunInstrVerifier.SHORTHAND + "AB" +): + NAME = "RunInstrVerifierAlternatingBudget" + + def actions_for_project(self, project: VProject, + flags: Flags) -> MutableSequence[Step]: + project.cflags += ["-mllvm", f"-vara-optimizer-policy=alternating"] + return super().actions_for_project(project, flags) diff --git a/varats/varats/experiments/vara/feature_experiment.py b/varats/varats/experiments/vara/feature_experiment.py index d6bca45c6..9df444d08 100644 --- a/varats/varats/experiments/vara/feature_experiment.py +++ b/varats/varats/experiments/vara/feature_experiment.py @@ -39,6 +39,44 @@ FeatureModelProvider, ) from varats.report.report import ReportSpecification +import yaml + +# TODO: Figure out a better location for this, this is just here to fix the circular import from multi_compile_experiment +class Flags: + + def __init__( + self, + cflags: tp.Optional[tp.List[str]] = None, + ldflags: tp.Optional[tp.List[str]] = None, + result_folder_name: tp.Optional[str] = None + ): + self.__cflags = cflags or [] + self.__ldflags = ldflags or [] + self.__result_folder_name = result_folder_name + + @property + def cflags(self) -> tp.List[str]: + return self.__cflags + + @property + def ldflags(self) -> tp.List[str]: + return self.__ldflags + + @property + def result_folder_name(self) -> tp.Optional[str]: + return self.__result_folder_name + + def __str__(self): + return f"Flags(cflags={self.cflags}, ldflags={self.ldflags}, result_folder_name={self.result_folder_name})" + + def dump_yaml(self): + return yaml.dump({ + "cflags": self.cflags, + "ldflags": self.ldflags, + }) + + __repr__ = __str__ + class FeatureInstrType(Enum): @@ -226,12 +264,14 @@ def __init__( report_file_ending: str = "json", workload_categories: tp.List[WorkloadCategory] = [ WorkloadCategory.EXAMPLE - ] + ], + additional_flags: tp.Optional[Flags] = None ): super().__init__(project=project) self.__experiment_handle = experiment_handle self.__report_file_ending = report_file_ending self.__workload_categories = workload_categories + self.__additional_flags = additional_flags def __call__(self) -> StepResult: return self.run_traced_code() @@ -259,6 +299,12 @@ def run_traced_code(self) -> StepResult: for prj_command in workload_commands( self.project, binary, self.__workload_categories ): + if self.__additional_flags is not None: + local_metadata_path = Path(tmp_dir) / "metadata.yml" + + with open(local_metadata_path, "w") as f: + f.write(self.__additional_flags.dump_yaml()) + local_tracefile_path = Path( tmp_dir ) / f"trace_{prj_command.command.label}" \ diff --git a/varats/varats/experiments/vara/instrumentation_verifier.py b/varats/varats/experiments/vara/instrumentation_verifier.py index 02529433f..c0d60ec04 100644 --- a/varats/varats/experiments/vara/instrumentation_verifier.py +++ b/varats/varats/experiments/vara/instrumentation_verifier.py @@ -20,7 +20,7 @@ ) from varats.project.varats_project import VProject from varats.report.report import ReportSpecification -from varats.experiments.vara.multi_compile_experiment import VaryingStartingBudgetExperiment +from varats.experiments.vara.multi_compile_experiment import Flags, VaryingStartingBudgetExperiment class RunInstrVerifier(FeatureExperiment, shorthand="RIV"): @@ -86,7 +86,7 @@ class RunInstrVerifierBudget(VaryingStartingBudgetExperiment, shorthand="RIVB"): REPORT_SPEC = ReportSpecification(InstrVerifierReport) def actions_for_project( - self, project: VProject + self, project: VProject, flags: Flags ) -> tp.MutableSequence[actions.Step]: """ Returns the specified steps to run the project(s) specified in the call @@ -95,6 +95,7 @@ def actions_for_project( Args: project: to analyze """ + print(flags) project.cflags += self.get_vara_feature_cflags(project) project.cflags += self.get_vara_tracing_cflags( @@ -126,7 +127,8 @@ def actions_for_project( analysis_actions.append(actions.Compile(project)) analysis_actions.append( RunVaRATracedWorkloads( - project, self.get_handle(), report_file_ending="ivr", workload_categories=[WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL] + project, self.get_handle(), report_file_ending="ivr", workload_categories=[WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL], + additional_flags=flags ) ) analysis_actions.append(actions.Clean(project)) diff --git a/varats/varats/experiments/vara/multi_compile_experiment.py b/varats/varats/experiments/vara/multi_compile_experiment.py index 4fe0e7fba..de4bab4d8 100644 --- a/varats/varats/experiments/vara/multi_compile_experiment.py +++ b/varats/varats/experiments/vara/multi_compile_experiment.py @@ -11,40 +11,10 @@ from varats.experiment.experiment_util import Project from varats.project.varats_project import VProject from varats.report.report import ReportSpecification -from varats.experiments.vara.feature_experiment import FeatureExperiment +from varats.experiments.vara.feature_experiment import FeatureExperiment, Flags from benchbuild.experiment import Actions -class Flags: - - def __init__( - self, - cflags: tp.Optional[tp.List[str]] = None, - ldflags: tp.Optional[tp.List[str]] = None, - result_folder_name: tp.Optional[str] = None - ): - self.__cflags = cflags or [] - self.__ldflags = ldflags or [] - self.__result_folder_name = result_folder_name - - @property - def cflags(self) -> tp.List[str]: - return self.__cflags - - @property - def ldflags(self) -> tp.List[str]: - return self.__ldflags - - @property - def result_folder_name(self) -> tp.Optional[str]: - return self.__result_folder_name - - def __str__(self): - return f"Flags(cflags={self.cflags}, ldflags={self.ldflags}, result_folder_name={self.result_folder_name})" - - __repr__ = __str__ - - class MultiCompileExperiment(FeatureExperiment, shorthand=""): """Base class experiment for feature specific experiments.""" @@ -54,7 +24,8 @@ class MultiCompileExperiment(FeatureExperiment, shorthand=""): @abstractmethod def actions_for_project(self, - project: VProject) -> tp.MutableSequence[Step]: + project: VProject, + flags: Flags) -> tp.MutableSequence[Step]: """Get the actions a project wants to run.""" def get_flags(self) -> tp.List[Flags]: @@ -64,7 +35,6 @@ def get_flags(self) -> tp.List[Flags]: def actions(self) -> Actions: actions: Actions = [] - # TODO: Add additional echo to provide more information about the current run def new_actions(self, proj: Project, flags: Flags) -> Actions: atomic_actions: Actions = [ tp.cast(Step, actns.Clean(proj)), @@ -91,7 +61,7 @@ def new_actions(self, proj: Project, flags: Flags) -> Actions: ) ) atomic_actions.append(actns.ProjectEnvironment(proj)) - atomic_actions.extend(self.actions_for_project(proj)) + atomic_actions.extend(self.actions_for_project(proj, flags)) return [tp.cast(Step, actns.RequireAll(actions=atomic_actions))] for prj_cls in self.projects: @@ -126,7 +96,8 @@ class VaryingStartingBudgetExperiment(MultiCompileExperiment, shorthand=""): @abstractmethod def actions_for_project(self, - project: VProject) -> tp.MutableSequence[Step]: + project: VProject, + flags: Flags) -> tp.MutableSequence[Step]: """Get the actions a project wants to run.""" def get_flags(self) -> tp.List[Flags]: diff --git a/varats/varats/plots/instrumentation_verifier_compare_experiments_overview.py b/varats/varats/plots/instrumentation_verifier_compare_experiments_overview.py index 4e7a29337..1909c525b 100644 --- a/varats/varats/plots/instrumentation_verifier_compare_experiments_overview.py +++ b/varats/varats/plots/instrumentation_verifier_compare_experiments_overview.py @@ -74,6 +74,7 @@ def _generate_plot(**kwargs: tp.Any) -> None: for report in reports: for binary in report.binaries(): + print(binary) labels.append(f"{binary}") num_enters.append(report.num_enters(binary),) num_leaves.append(report.num_leaves(binary),) @@ -137,8 +138,6 @@ def _generate_plot(**kwargs: tp.Any) -> None: ax.legend() # ax.set_xticks(ind + width, labels=labels, rotation=30, ha="right") - print(minor_labels) - ax.set_xticks( minor_ticks, labels=minor_labels, From 68aa7d7cefd7d2a14463af73db7c22ab7da7fd87 Mon Sep 17 00:00:00 2001 From: Daniel Gusenburger Date: Thu, 5 Oct 2023 19:27:21 +0200 Subject: [PATCH 29/31] updating --- .../instrumentation_verifier_report.py | 4 +- .../vara/compare_traced_untraced.py | 25 +++- .../vara/dynamic_overhead_analysis.py | 44 ++++-- .../experiments/vara/feature_experiment.py | 71 +++------- .../vara/instrumentation_verifier.py | 93 ++++-------- .../vara/multi_compile_experiment.py | 47 +++++- ...n_verifier_compare_experiments_overview.py | 121 +++++++--------- ...nstrumentation_verifier_overview_budget.py | 134 ++++++++++++++++++ .../perf_tests/feature_perf_cs_collection.py | 51 ++++--- 9 files changed, 351 insertions(+), 239 deletions(-) create mode 100644 varats/varats/plots/instrumentation_verifier_overview_budget.py diff --git a/varats/varats/data/reports/instrumentation_verifier_report.py b/varats/varats/data/reports/instrumentation_verifier_report.py index 1add4f39d..47b6c1b18 100644 --- a/varats/varats/data/reports/instrumentation_verifier_report.py +++ b/varats/varats/data/reports/instrumentation_verifier_report.py @@ -64,7 +64,9 @@ def __init__(self, report_path: Path) -> None: unclosed_enter_begin = content.index( 'Unclosed Region-ID(s):' ) + 1 - wrong_leaves = content[wrong_leaves_begin:-1] + failure_begin = content.index("Finalization: Failure") + wrong_leaves = content[ + wrong_leaves_begin:failure_begin - 1] unclosed_regions = content[ unclosed_enter_begin:wrong_leaves_begin - 1] diff --git a/varats/varats/experiments/vara/compare_traced_untraced.py b/varats/varats/experiments/vara/compare_traced_untraced.py index 962351a92..62dd71840 100644 --- a/varats/varats/experiments/vara/compare_traced_untraced.py +++ b/varats/varats/experiments/vara/compare_traced_untraced.py @@ -19,7 +19,8 @@ from varats.experiments.vara.feature_experiment import FeatureExperiment, FeatureInstrType from varats.experiments.vara.multi_compile_experiment import VaryingStartingBudgetExperiment -MEASUREMENT_REPS = 10 +MEASUREMENT_REPS = 20 + class RunUntraced(FeatureExperiment, shorthand="RU"): """Build and run the untraced version of the binary""" @@ -57,6 +58,7 @@ def actions_for_project( project, FeatureInstrType.NONE, actions, save_temps=True ) + class RunTraced(FeatureExperiment, shorthand="RT"): """Build and run the traced version of the binary""" @@ -73,8 +75,8 @@ def actions_for_project( ) -> tp.MutableSequence[actions.Step]: project.cflags += [ - "-mllvm", - f"-vara-optimizer-policy={self.optimizer_policy.value}", + "-mllvm", f"-vara-optimizer-policy={self.optimizer_policy.value}", + "-mllvm", "-debug-only=OPT,InstrMark,IRT" ] actions = [] @@ -102,6 +104,7 @@ def actions_for_project( project, FeatureInstrType.TEF, actions, save_temps=True ) + class RunTracedNaive(RunTraced, shorthand=RunTraced.SHORTHAND + "N"): """Build and run the traced version of the binary""" @@ -112,6 +115,7 @@ class RunTracedNaive(RunTraced, shorthand=RunTraced.SHORTHAND + "N"): def optimizer_policy(self) -> OptimizerPolicyType: return OptimizerPolicyType.NAIVE + class RunTracedAlternating(RunTraced, shorthand=RunTraced.SHORTHAND + "A"): """Build and run the traced version of the binary""" @@ -122,6 +126,7 @@ class RunTracedAlternating(RunTraced, shorthand=RunTraced.SHORTHAND + "A"): def optimizer_policy(self) -> OptimizerPolicyType: return OptimizerPolicyType.ALTERNATING + class RunTracedBudget(VaryingStartingBudgetExperiment, shorthand="RTB"): """Build and run the traced version of the binary""" @@ -167,20 +172,26 @@ def actions_for_project( project, FeatureInstrType.TEF, actions, save_temps=True ) -class RunTracedNaiveBudget(RunTracedBudget, shorthand=RunTracedBudget.SHORTHAND + "N"): + +class RunTracedNaiveBudget( + RunTracedBudget, shorthand=RunTracedBudget.SHORTHAND + "N" +): """Build and run the traced version of the binary""" - NAME = "RunTracedNaive" + NAME = "RunTracedNaiveBudget" @property @abstractmethod def optimizer_policy(self) -> OptimizerPolicyType: return OptimizerPolicyType.NAIVE -class RunTracedAlternatingBudget(RunTracedBudget, shorthand=RunTracedBudget.SHORTHAND + "A"): + +class RunTracedAlternatingBudget( + RunTracedBudget, shorthand=RunTracedBudget.SHORTHAND + "A" +): """Build and run the traced version of the binary""" - NAME = "RunTracedAlternating" + NAME = "RunTracedAlternatingBudget" @property @abstractmethod diff --git a/varats/varats/experiments/vara/dynamic_overhead_analysis.py b/varats/varats/experiments/vara/dynamic_overhead_analysis.py index 8d8842c6b..b6ec9d012 100644 --- a/varats/varats/experiments/vara/dynamic_overhead_analysis.py +++ b/varats/varats/experiments/vara/dynamic_overhead_analysis.py @@ -1,7 +1,6 @@ from enum import Enum from typing import MutableSequence from varats.experiment.experiment_util import Step -from varats.experiments.vara.feature_experiment import Flags from varats.project.varats_project import VProject @@ -20,7 +19,10 @@ class RunInstrVerifierNaive( NAME = "RunInstrVerifierNaive" def actions_for_project(self, project: VProject) -> MutableSequence[Step]: - project.cflags += ["-mllvm", f"-vara-optimizer-policy=naive"] + project.cflags += [ + "-mllvm", "-vara-optimizer-policy=naive", "-vara-optimizer-starting-budget=0", "-mllvm", + "-debug-only=OPT,IRT,InstrMark" + ] return super().actions_for_project(project) @@ -30,27 +32,43 @@ class RunInstrVerifierAlternating( NAME = "RunInstrVerifierAlternating" def actions_for_project(self, project: VProject) -> MutableSequence[Step]: - project.cflags += ["-mllvm", f"-vara-optimizer-policy=alternating"] + project.cflags += [ + "-mllvm", "-vara-optimizer-policy=alternating", "-mllvm", + "-debug-only=OPT,IRT,InstrMark" + ] return super().actions_for_project(project) +class RunInstrVerifierLoopExtract( + RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "L" +): + NAME = "RunInstrVerifierLoopExtract" + + def actions_for_project(self, project: VProject) -> MutableSequence[Step]: + project.cflags += [ + "-mllvm", "-vara-optimizer-policy=loop_extract", "-mllvm", + "-debug-only=OPT,IRT,InstrMark" + ] + return super().actions_for_project(project) class RunInstrVerifierNaiveBudget( - RunInstrVerifierBudget, shorthand=RunInstrVerifier.SHORTHAND + "NB" + RunInstrVerifierBudget, shorthand=RunInstrVerifierBudget.SHORTHAND + "N" ): NAME = "RunInstrVerifierNaiveBudget" - def actions_for_project(self, project: VProject, - flags: Flags) -> MutableSequence[Step]: - project.cflags += ["-mllvm", f"-vara-optimizer-policy=naive"] - return super().actions_for_project(project, flags) + def actions_for_project(self, project: VProject) -> MutableSequence[Step]: + project.cflags += [ + "-mllvm", "-vara-optimizer-policy=naive", "-mllvm", + "-debug-only=OPT,IRT,InstrMark" + ] + return super().actions_for_project(project) class RunInstrVerifierAlternatingBudget( - RunInstrVerifierBudget, shorthand=RunInstrVerifier.SHORTHAND + "AB" + RunInstrVerifierBudget, shorthand=RunInstrVerifierBudget.SHORTHAND + "A" ): NAME = "RunInstrVerifierAlternatingBudget" - def actions_for_project(self, project: VProject, - flags: Flags) -> MutableSequence[Step]: - project.cflags += ["-mllvm", f"-vara-optimizer-policy=alternating"] - return super().actions_for_project(project, flags) + def actions_for_project(self, project: VProject) -> MutableSequence[Step]: + project.cflags += ["-mllvm", "-vara-optimizer-policy=alternating"] + return super().actions_for_project(project) + diff --git a/varats/varats/experiments/vara/feature_experiment.py b/varats/varats/experiments/vara/feature_experiment.py index 9df444d08..e4b5ddf84 100644 --- a/varats/varats/experiments/vara/feature_experiment.py +++ b/varats/varats/experiments/vara/feature_experiment.py @@ -41,43 +41,6 @@ from varats.report.report import ReportSpecification import yaml -# TODO: Figure out a better location for this, this is just here to fix the circular import from multi_compile_experiment -class Flags: - - def __init__( - self, - cflags: tp.Optional[tp.List[str]] = None, - ldflags: tp.Optional[tp.List[str]] = None, - result_folder_name: tp.Optional[str] = None - ): - self.__cflags = cflags or [] - self.__ldflags = ldflags or [] - self.__result_folder_name = result_folder_name - - @property - def cflags(self) -> tp.List[str]: - return self.__cflags - - @property - def ldflags(self) -> tp.List[str]: - return self.__ldflags - - @property - def result_folder_name(self) -> tp.Optional[str]: - return self.__result_folder_name - - def __str__(self): - return f"Flags(cflags={self.cflags}, ldflags={self.ldflags}, result_folder_name={self.result_folder_name})" - - def dump_yaml(self): - return yaml.dump({ - "cflags": self.cflags, - "ldflags": self.ldflags, - }) - - __repr__ = __str__ - - class FeatureInstrType(Enum): """Type of instrumentation to be used in feature tracing .""" @@ -123,6 +86,7 @@ def get_common_tracing_actions( analysis_actions: tp.List[Step], save_temps: bool = False, instruction_threshold: tp.Optional[int] = None, + lto: bool = False, ) -> tp.MutableSequence[Step]: """ Set common options and return a list of common actions for feature @@ -141,9 +105,9 @@ def get_common_tracing_actions( """ project.cflags += self.get_vara_feature_cflags(project) project.cflags += self.get_vara_tracing_cflags( - instr_type, save_temps, instruction_threshold=instruction_threshold + instr_type, save_temps, instruction_threshold=instruction_threshold, lto=lto ) - project.ldflags += self.get_vara_tracing_ldflags() + project.ldflags += self.get_vara_tracing_ldflags(lto=lto) # runtime and compiler extensions project.runtime_extension = run.RuntimeExtension(project, self) \ @@ -204,7 +168,8 @@ def get_vara_tracing_cflags( instr_type: FeatureInstrType, save_temps: bool = False, project: tp.Optional[VProject] = None, - instruction_threshold: tp.Optional[int] = None + instruction_threshold: tp.Optional[int] = None, + lto: bool = True ) -> tp.List[str]: """ Returns the cflags needed to trace projects with VaRA, using the @@ -221,8 +186,11 @@ def get_vara_tracing_cflags( c_flags = [] if instr_type != FeatureInstrType.NONE: c_flags += ["-fsanitize=vara", f"-fvara-instr={instr_type.value}"] + + if lto: + c_flags += ["-flto"] + c_flags += [ - "-flto", "-fuse-ld=lld", "-flegacy-pass-manager", "-fno-omit-frame-pointer", @@ -240,13 +208,13 @@ def get_vara_tracing_cflags( return c_flags @staticmethod - def get_vara_tracing_ldflags() -> tp.List[str]: + def get_vara_tracing_ldflags(lto: bool = True) -> tp.List[str]: """ Returns the ldflags needed to instrument projects with VaRA during LTO. Returns: ldflags for VaRA LTO support """ - return ["-flto"] + return ["-flto"] if lto else [] class RunVaRATracedWorkloads(ProjectStep): # type: ignore @@ -264,14 +232,12 @@ def __init__( report_file_ending: str = "json", workload_categories: tp.List[WorkloadCategory] = [ WorkloadCategory.EXAMPLE - ], - additional_flags: tp.Optional[Flags] = None + ] ): super().__init__(project=project) self.__experiment_handle = experiment_handle self.__report_file_ending = report_file_ending self.__workload_categories = workload_categories - self.__additional_flags = additional_flags def __call__(self) -> StepResult: return self.run_traced_code() @@ -299,11 +265,14 @@ def run_traced_code(self) -> StepResult: for prj_command in workload_commands( self.project, binary, self.__workload_categories ): - if self.__additional_flags is not None: - local_metadata_path = Path(tmp_dir) / "metadata.yml" - - with open(local_metadata_path, "w") as f: - f.write(self.__additional_flags.dump_yaml()) + metadata_obj = { + "cflags": self.project.cflags, + "ldflags": self.project.ldflags + } + + local_metadata_path = Path(tmp_dir) / "metadata.yml" + with open(local_metadata_path, "w") as f: + f.write(yaml.dump(metadata_obj)) local_tracefile_path = Path( tmp_dir diff --git a/varats/varats/experiments/vara/instrumentation_verifier.py b/varats/varats/experiments/vara/instrumentation_verifier.py index c0d60ec04..31acb7d68 100644 --- a/varats/varats/experiments/vara/instrumentation_verifier.py +++ b/varats/varats/experiments/vara/instrumentation_verifier.py @@ -2,17 +2,12 @@ is used during execution to check if regions are correctly opend/closed.""" import typing as tp -from benchbuild.extensions import compiler, run from benchbuild.utils import actions from varats.experiment.workload_util import WorkloadCategory from varats.data.reports.instrumentation_verifier_report import ( InstrVerifierReport, ) -from varats.experiment.experiment_util import ( - get_default_compile_error_wrapped, - WithUnlimitedStackSize, -) from varats.experiments.vara.feature_experiment import ( FeatureExperiment, RunVaRATracedWorkloads, @@ -20,7 +15,7 @@ ) from varats.project.varats_project import VProject from varats.report.report import ReportSpecification -from varats.experiments.vara.multi_compile_experiment import Flags, VaryingStartingBudgetExperiment +from varats.experiments.vara.multi_compile_experiment import VaryingStartingBudgetExperiment class RunInstrVerifier(FeatureExperiment, shorthand="RIV"): @@ -40,43 +35,29 @@ def actions_for_project( Args: project: to analyze """ - project.cflags += self.get_vara_feature_cflags(project) - - project.cflags += self.get_vara_tracing_cflags( - FeatureInstrType.VERIFY, True - ) - - # Ensure that we detect all regions, when verifying - project.cflags += ["-fvara-instruction-threshold=0"] - - # Add debug information, so traces can be better interpreted - project.cflags += ["-g"] - - project.ldflags += self.get_vara_tracing_ldflags() - - # Add the required runtime extensions to the project(s). - project.runtime_extension = run.RuntimeExtension(project, self) - - # Add the required compiler extensions to the project(s). - project.compiler_extension = compiler.RunCompiler(project, self) \ - << WithUnlimitedStackSize() - - # Add own error handler to compile step. - project.compile = get_default_compile_error_wrapped( - self.get_handle(), project, self.REPORT_SPEC.main_report - ) - analysis_actions = [] analysis_actions.append(actions.Compile(project)) analysis_actions.append( RunVaRATracedWorkloads( - project, self.get_handle(), report_file_ending="ivr", workload_categories=[WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL] + project, + self.get_handle(), + report_file_ending="ivr", + workload_categories=[ + WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL + ] ) ) analysis_actions.append(actions.Clean(project)) - return analysis_actions + return self.get_common_tracing_actions( + project, + FeatureInstrType.VERIFY, + analysis_actions, + save_temps=True, + instruction_threshold=0 + ) + class RunInstrVerifierBudget(VaryingStartingBudgetExperiment, shorthand="RIVB"): """Test runner for feature performance.""" @@ -86,7 +67,7 @@ class RunInstrVerifierBudget(VaryingStartingBudgetExperiment, shorthand="RIVB"): REPORT_SPEC = ReportSpecification(InstrVerifierReport) def actions_for_project( - self, project: VProject, flags: Flags + self, project: VProject ) -> tp.MutableSequence[actions.Step]: """ Returns the specified steps to run the project(s) specified in the call @@ -95,42 +76,26 @@ def actions_for_project( Args: project: to analyze """ - print(flags) - project.cflags += self.get_vara_feature_cflags(project) - - project.cflags += self.get_vara_tracing_cflags( - FeatureInstrType.VERIFY, True - ) - - # Ensure that we detect all regions, when verifying - project.cflags += ["-fvara-instruction-threshold=0"] - - # Add debug information, so traces can be better interpreted - project.cflags += ["-g"] - - project.ldflags += self.get_vara_tracing_ldflags() - - # Add the required runtime extensions to the project(s). - project.runtime_extension = run.RuntimeExtension(project, self) - - # Add the required compiler extensions to the project(s). - project.compiler_extension = compiler.RunCompiler(project, self) \ - << WithUnlimitedStackSize() - - # Add own error handler to compile step. - project.compile = get_default_compile_error_wrapped( - self.get_handle(), project, self.REPORT_SPEC.main_report - ) analysis_actions = [] analysis_actions.append(actions.Compile(project)) analysis_actions.append( RunVaRATracedWorkloads( - project, self.get_handle(), report_file_ending="ivr", workload_categories=[WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL], - additional_flags=flags + project, + self.get_handle(), + report_file_ending="ivr", + workload_categories=[ + WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL + ], ) ) analysis_actions.append(actions.Clean(project)) - return analysis_actions + return self.get_common_tracing_actions( + project, + FeatureInstrType.VERIFY, + analysis_actions, + save_temps=True, + instruction_threshold=0 + ) diff --git a/varats/varats/experiments/vara/multi_compile_experiment.py b/varats/varats/experiments/vara/multi_compile_experiment.py index de4bab4d8..156ca372e 100644 --- a/varats/varats/experiments/vara/multi_compile_experiment.py +++ b/varats/varats/experiments/vara/multi_compile_experiment.py @@ -11,10 +11,40 @@ from varats.experiment.experiment_util import Project from varats.project.varats_project import VProject from varats.report.report import ReportSpecification -from varats.experiments.vara.feature_experiment import FeatureExperiment, Flags +from varats.experiments.vara.feature_experiment import FeatureExperiment from benchbuild.experiment import Actions +class Flags: + + def __init__( + self, + cflags: tp.Optional[tp.List[str]] = None, + ldflags: tp.Optional[tp.List[str]] = None, + result_folder_name: tp.Optional[str] = None + ): + self.__cflags = cflags or [] + self.__ldflags = ldflags or [] + self.__result_folder_name = result_folder_name + + @property + def cflags(self) -> tp.List[str]: + return self.__cflags + + @property + def ldflags(self) -> tp.List[str]: + return self.__ldflags + + @property + def result_folder_name(self) -> tp.Optional[str]: + return self.__result_folder_name + + def __str__(self): + return f"Flags(cflags={self.cflags}, ldflags={self.ldflags}, result_folder_name={self.result_folder_name})" + + __repr__ = __str__ + + class MultiCompileExperiment(FeatureExperiment, shorthand=""): """Base class experiment for feature specific experiments.""" @@ -23,8 +53,7 @@ class MultiCompileExperiment(FeatureExperiment, shorthand=""): REPORT_SPEC = ReportSpecification() @abstractmethod - def actions_for_project(self, - project: VProject, + def actions_for_project(self, project: VProject, flags: Flags) -> tp.MutableSequence[Step]: """Get the actions a project wants to run.""" @@ -61,7 +90,7 @@ def new_actions(self, proj: Project, flags: Flags) -> Actions: ) ) atomic_actions.append(actns.ProjectEnvironment(proj)) - atomic_actions.extend(self.actions_for_project(proj, flags)) + atomic_actions.extend(self.actions_for_project(proj)) return [tp.cast(Step, actns.RequireAll(actions=atomic_actions))] for prj_cls in self.projects: @@ -89,20 +118,24 @@ def new_actions(self, proj: Project, flags: Flags) -> Actions: return actions +STARTING_BUDGET = 0 +END_BUDGET = 100 +BUDGET_STEP = 20 + + class VaryingStartingBudgetExperiment(MultiCompileExperiment, shorthand=""): NAME = "VaryingStartingBudgetExperiment" REPORT_SPEC = ReportSpecification() @abstractmethod - def actions_for_project(self, - project: VProject, + def actions_for_project(self, project: VProject, flags: Flags) -> tp.MutableSequence[Step]: """Get the actions a project wants to run.""" def get_flags(self) -> tp.List[Flags]: flags = [] - for budget in range(20, 110, 10): + for budget in range(STARTING_BUDGET, END_BUDGET, BUDGET_STEP): f = Flags( cflags=["-mllvm", f"-vara-optimizer-starting-budget={budget}"], result_folder_name=f"starting_budget_{budget}" diff --git a/varats/varats/plots/instrumentation_verifier_compare_experiments_overview.py b/varats/varats/plots/instrumentation_verifier_compare_experiments_overview.py index 1909c525b..ab0bb67c5 100644 --- a/varats/varats/plots/instrumentation_verifier_compare_experiments_overview.py +++ b/varats/varats/plots/instrumentation_verifier_compare_experiments_overview.py @@ -4,8 +4,8 @@ import typing as tp import matplotlib.pyplot as plt -from matplotlib import ticker import numpy as np +import pandas as pd import varats.paper.paper_config as PC from varats.data.reports.instrumentation_verifier_report import ( @@ -18,7 +18,6 @@ from varats.ts_utils.click_param_types import REQUIRE_MULTI_EXPERIMENT_TYPE from varats.utils.exceptions import UnsupportedOperation from varats.utils.git_util import FullCommitHash -import itertools class InstrumentationOverviewCompareExperimentsPlot( @@ -43,13 +42,7 @@ def calc_missing_revisions( def _generate_plot(**kwargs: tp.Any) -> None: case_study = kwargs['case_study'] - width = 0.25 - multiplicator = 0 - - minor_labels = [] - minor_ticks = [] - - _, ax = plt.subplots() + rows = [] for experiment in kwargs["experiment_type"]: @@ -57,8 +50,6 @@ def _generate_plot(**kwargs: tp.Any) -> None: case_study.project_name, experiment, only_newest=False ) - labels: tp.List[str] = [] - reports: tp.List[InstrVerifierReport] = [ InstrVerifierReport(rev_file.full_path()) for rev_file in revisions_files @@ -67,84 +58,76 @@ def _generate_plot(**kwargs: tp.Any) -> None: if len(reports) == 0: raise PlotDataEmpty() - num_enters: tp.List[int] = [] - num_leaves: tp.List[int] = [] - num_unclosed_enters: tp.List[int] = [] - num_unentered_leaves: tp.List[int] = [] - for report in reports: for binary in report.binaries(): - print(binary) - labels.append(f"{binary}") - num_enters.append(report.num_enters(binary),) - num_leaves.append(report.num_leaves(binary),) - num_unclosed_enters.append( - report.num_unclosed_enters(binary), - ) - num_unentered_leaves.append( - report.num_unentered_leaves(binary), - ) - - minor_labels.extend([ - x + "-" + y - for x, y in zip(labels, itertools.repeat(experiment.NAME)) - ]) - ind = np.arange(len(num_enters)) - offset = width * multiplicator + rows.append({ + "experiment": experiment.NAME, + "binary": binary, + "enters": report.num_enters(binary), + "leaves": report.num_leaves(binary), + "unclosed_enters": report.num_unclosed_enters(binary), + "unentered_leaves": report.num_unentered_leaves(binary) + }) + + df = pd.DataFrame(rows) + binaries = df["binary"].unique() + experiments = df["experiment"].unique() + fig, axs = plt.subplots((1 + len(binaries)) // 2, 2 - len(binaries) % 2) + + for i, binary in enumerate(binaries): + if len(binaries) == 1: + ax = axs + elif len(binaries) == 2: + ax = axs[i % 2] + else: + ax = axs[*divmod(i, 2)] + + d = df[df["binary"] == binary] + + num_enters = np.array(d["enters"]) + num_leaves = np.array(d["leaves"]) + num_unclosed_enters = np.array(d["unclosed_enters"]) + num_unentered_leaves = np.array(d["unentered_leaves"]) + + # num_enters = num_enters - num_unclosed_enters + # num_leaves = num_leaves - num_unentered_leaves ax.bar( - ind + offset, - num_enters, - width, - color="tab:blue", - edgecolor="black" + experiments, + num_enters ) ax.bar( - ind + offset, + experiments, num_leaves, - width, - color="tab:orange", bottom=num_enters, - edgecolor="black" ) ax.bar( - ind + offset, + experiments, num_unclosed_enters, - width, - color="tab:cyan", - edgecolor="black", - bottom=[a + b for a, b in zip(num_enters, num_leaves)] + bottom=num_enters + num_leaves ) ax.bar( - ind + offset, + experiments, num_unentered_leaves, - width, - color="tab:olive", - edgecolor="black", - bottom=[ - a + b + c for a, b, c in - zip(num_enters, num_leaves, num_unclosed_enters) - ] + bottom=num_enters + num_leaves + num_unclosed_enters ) - minor_ticks.extend(ind + offset) - multiplicator += 1 - ax.set_ylabel("Number of events") - ax.set_title( + ax.set_ylabel("Number of events") + ax.set_xticks(ax.get_xticks(), ax.get_xticklabels(), rotation=45, ha='right') + ax.set_title(binary) + # ax.set_xticks( + # minor_ticks, + # labels=minor_labels, + # rotation=30, + # ha="right", + # ) + + fig.suptitle( f"Instrumentation Verifier " f"Overview for {case_study.project_name}" ) - ax.legend() - # ax.set_xticks(ind + width, labels=labels, rotation=30, ha="right") - - ax.set_xticks( - minor_ticks, - labels=minor_labels, - rotation=30, - ha="right", - ) - + fig.legend(labels=["Enters", "Leaves", "Unclosed enters", "Unentered leaves"]) plt.subplots_adjust(bottom=0.25) diff --git a/varats/varats/plots/instrumentation_verifier_overview_budget.py b/varats/varats/plots/instrumentation_verifier_overview_budget.py new file mode 100644 index 000000000..ad0feb76b --- /dev/null +++ b/varats/varats/plots/instrumentation_verifier_overview_budget.py @@ -0,0 +1,134 @@ +"""Example table that uses different workloads and visualizes the time it took +to run them.""" +import typing as tp +import re + +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sns +import numpy as np + +import varats.paper.paper_config as PC +from varats.plot.plot import Plot, PlotDataEmpty +from varats.plot.plots import PlotGenerator +from varats.data.reports.instrumentation_verifier_report import ( + InstrVerifierReport, +) +from varats.report.report import ReportFilepath +from varats.revision.revisions import get_all_revisions_files +from varats.ts_utils.click_param_types import REQUIRE_EXPERIMENT_TYPE +from varats.utils.git_util import FullCommitHash + + +class InstrumentationVerifierOverviewBudgetPlot( + Plot, plot_name="instrumentation_verifier_overview_budget" +): + + def plot(self, view_mode: bool) -> None: + self._generate_plot(**self.plot_kwargs) + + @staticmethod + def _generate_plot(**kwargs: tp.Any) -> None: + case_study = kwargs["case_study"] + experiment = kwargs["experiment_type"] + + revisions_files: tp.List[ReportFilepath] = get_all_revisions_files( + case_study.project_name, experiment, only_newest=False + ) + + reports: tp.List[InstrVerifierReport] = [ + InstrVerifierReport(rev_file.full_path()) + for rev_file in revisions_files + ] + + if len(reports) == 0: + raise PlotDataEmpty() + + rows = [] + + for report in reports: + budget = 0 + for cf in report.metadata()["cflags"]: + if "budget" not in cf: + continue + + budget = int(cf.split("=")[1]) + + for binary in report.binaries(): + rows.append({ + "binary": binary, + "budget": budget, + "enters": report.num_enters(binary), + "leaves": report.num_leaves(binary), + "unclosed_enters": report.num_unclosed_enters(binary), + "unentered_leaves": report.num_unentered_leaves(binary) + }) + + df = pd.DataFrame(rows) + + binaries = df["binary"].unique() + fig, axs = plt.subplots((1 + len(binaries)) // 2, 2 - len(binaries) % 2) + fig.suptitle(f"Results of {experiment.NAME} by budget for case study {case_study.project_name}") + # fig.set_size_inches(11.7, 8.27) + + for i, binary in enumerate(binaries): + if len(binaries) == 1: + ax = axs + elif len(binaries) == 2: + ax = axs[i % 2] + else: + ax = axs[*divmod(i, 2)] + + d = df[df["binary"] == binary].sort_values("budget") + + num_enters_arr = np.array(d["enters"]) + num_leaves_arr = np.array(d["leaves"]) + num_unclosed_enters_arr = np.array(d["unclosed_enters"]) + num_unentered_leaves_arr = np.array(d["unentered_leaves"]) + + num_enters_arr = num_enters_arr - num_unclosed_enters_arr + num_leaves_arr = num_leaves_arr - num_unentered_leaves_arr + + X = np.arange(len(d["budget"])) + + ax.bar(X, num_enters_arr, label="#Enters") + ax.bar(X, num_leaves_arr, label="#Leaves", bottom=num_enters_arr) + ax.bar( + X, + num_unclosed_enters_arr, + label="#Unclosed Enters", + bottom=num_enters_arr + num_leaves_arr + ) + ax.bar( + X, + num_unentered_leaves_arr, + label="#Unentered Leaves", + bottom=num_enters_arr + num_leaves_arr + num_unclosed_enters_arr + ) + + ax.set_ylabel("# Events") + ax.set_xlabel("Budget") + ax.set_xticks(X, labels=d["budget"], rotation=45) + ax.set_title(binary) + + fig.legend(labels=["Closed enters", "Entered leaves", "Unclosed enters", "Unentered leaves"]) + sns.despine() + + def calc_missing_revisions( + self, boundary_gradient: float + ) -> tp.Set[FullCommitHash]: + raise NotImplementedError + + +class CompareRuntimesBudgetPlotCSGenerator( + PlotGenerator, + generator_name="iv-ce-overview-budget-plot", + options=[REQUIRE_EXPERIMENT_TYPE] +): + + def generate(self) -> tp.List[Plot]: + return [ + InstrumentationVerifierOverviewBudgetPlot( + self.plot_config, case_study=cs, **self.plot_kwargs + ) for cs in PC.get_paper_config().get_all_case_studies() + ] diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py index 6674e3e2c..3dae8c8a4 100644 --- a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -44,6 +44,19 @@ class FeaturePerfCSCollection(VProject): WORKLOADS = { WorkloadSet(WorkloadCategory.EXAMPLE): [ + Command( + SourceRoot("FeaturePerfCSCollection") / + RSBinary("SimpleBusyLoop"), + "--iterations", + str(10**4), + "--count_to", + str(5 * 10**3), + label="SBL-iterations-10K-count-to-5K" + ), + # Command( + # SourceRoot("FeaturePerfCSCollection") / RSBinary("LoopBubble"), + # label="LB-no-input" + # ), Command( SourceRoot("FeaturePerfCSCollection") / RSBinary("SingleLocalSimple"), @@ -54,15 +67,6 @@ class FeaturePerfCSCollection(VProject): RSBinary("MultiSharedMultipleRegions"), label="MSMR-no-input" ), - # Command( - # SourceRoot("FeaturePerfCSCollection") / - # RSBinary("SimpleSleepLoop"), - # "--iterations", - # "10000", - # "--sleepns", - # "10000", - # label="SSL-10K-10K" - # ), Command( SourceRoot("FeaturePerfCSCollection") / RSBinary("SimpleFeatureInteraction"), @@ -71,17 +75,17 @@ class FeaturePerfCSCollection(VProject): label="SFI-enc-compress" ) ], - WorkloadSet(WorkloadCategory.MEDIUM): [ - Command( - SourceRoot("FeaturePerfCSCollection") / - RSBinary("SimpleBusyLoop"), - "--iterations", - str(10**7), - "--count_to", - str(5 * 10**3), - label="SBL-iterations-10M-count-to-5K" - ) - ] + # WorkloadSet(WorkloadCategory.SMALL): [ + # Command( + # SourceRoot("FeaturePerfCSCollection") / + # RSBinary("SimpleBusyLoop"), + # "--iterations", + # str(10**7), + # "--count_to", + # str(5 * 10**3), + # label="SBL-iterations-10M-count-to-5K" + # ) + # ] } @staticmethod @@ -100,13 +104,6 @@ def binaries_for_revision( BinaryType.EXECUTABLE, only_valid_in=RevisionRange("162db88346", "master") ) - # binary_map.specify_binary( - # "build/bin/SimpleSleepLoop", - # BinaryType.EXECUTABLE, - # only_valid_in=RevisionRange( - # "c77bca4c6888970fb721069c82455137943ccf49", "master" - # ) - # ) binary_map.specify_binary( "build/bin/SimpleBusyLoop", BinaryType.EXECUTABLE, From b0113aa850d15d4c283e8f0c87b269df841eb874 Mon Sep 17 00:00:00 2001 From: Daniel Gusenburger Date: Mon, 9 Oct 2023 20:28:59 +0200 Subject: [PATCH 30/31] updates --- .../varats/experiment/experiment_util.py | 1 + varats-core/varats/report/gnu_time_report.py | 2 - .../varats/experiments/base/time_workloads.py | 8 +- .../vara/compare_traced_untraced.py | 179 ++++++++++++------ .../vara/dynamic_overhead_analysis.py | 50 ++++- .../vara/instrumentation_verifier.py | 1 + varats/varats/plots/compare_traced_budget.py | 77 ++++---- varats/varats/projects/c_projects/bzip2.py | 18 ++ varats/varats/projects/c_projects/picosat.py | 16 +- varats/varats/projects/c_projects/xz.py | 20 +- varats/varats/ts_utils/__init__.py | 0 11 files changed, 241 insertions(+), 131 deletions(-) create mode 100644 varats/varats/ts_utils/__init__.py diff --git a/varats-core/varats/experiment/experiment_util.py b/varats-core/varats/experiment/experiment_util.py index bad60ba6f..a30f42c8b 100644 --- a/varats-core/varats/experiment/experiment_util.py +++ b/varats-core/varats/experiment/experiment_util.py @@ -10,6 +10,7 @@ from collections import defaultdict from pathlib import Path from types import TracebackType +import yaml from benchbuild import source from benchbuild.experiment import Experiment diff --git a/varats-core/varats/report/gnu_time_report.py b/varats-core/varats/report/gnu_time_report.py index 124929602..1991701e7 100644 --- a/varats-core/varats/report/gnu_time_report.py +++ b/varats-core/varats/report/gnu_time_report.py @@ -74,8 +74,6 @@ def __init__(self, path: Path) -> None: TimeReport._parse_involuntary_ctx_switches(line) continue - # print("Not matched: ", line) - @property def command_name(self) -> str: """Name of the command that was executed.""" diff --git a/varats/varats/experiments/base/time_workloads.py b/varats/varats/experiments/base/time_workloads.py index 98618acd3..1d8d8c44c 100644 --- a/varats/varats/experiments/base/time_workloads.py +++ b/varats/varats/experiments/base/time_workloads.py @@ -56,6 +56,7 @@ def analyze(self, tmp_dir: Path) -> actions.StepResult: """Only create a report file.""" with local.cwd(self.project.builddir): + print(f"Step {self.__num}") for prj_command in workload_commands( self.project, self.__binary, self.__workload_categories ): @@ -73,13 +74,14 @@ def analyze(self, tmp_dir: Path) -> actions.StepResult: with cleanup(prj_command): run_cmd(retcode=self.__binary.valid_exit_codes) + print("Done") + return actions.StepResult.OK def __str__(self, indent: int = 0) -> str: return textwrap.indent( - f"* Run workloads of categories {', '.join(str(x) for x in self.__workload_categories)} " - f"for binary {self.__binary.name} ({self.__num})", - indent * " " + f"* Run workloads of categories {', '.join(str(x) for x in self.__workload_categories)} " + f"for binary {self.__binary.name} ({self.__num})", indent * " " ) diff --git a/varats/varats/experiments/vara/compare_traced_untraced.py b/varats/varats/experiments/vara/compare_traced_untraced.py index 62dd71840..6548a0faf 100644 --- a/varats/varats/experiments/vara/compare_traced_untraced.py +++ b/varats/varats/experiments/vara/compare_traced_untraced.py @@ -19,7 +19,7 @@ from varats.experiments.vara.feature_experiment import FeatureExperiment, FeatureInstrType from varats.experiments.vara.multi_compile_experiment import VaryingStartingBudgetExperiment -MEASUREMENT_REPS = 20 +MEASUREMENT_REPS = 1 class RunUntraced(FeatureExperiment, shorthand="RU"): @@ -70,15 +70,25 @@ class RunTraced(FeatureExperiment, shorthand="RT"): def optimizer_policy(self) -> OptimizerPolicyType: return OptimizerPolicyType.NONE + @property + @abstractmethod + def budget(self) -> tp.Optional[int]: + return None + def actions_for_project( self, project: VProject ) -> tp.MutableSequence[actions.Step]: project.cflags += [ - "-mllvm", f"-vara-optimizer-policy={self.optimizer_policy.value}", - "-mllvm", "-debug-only=OPT,InstrMark,IRT" + "-mllvm", f"-vara-optimizer-policy={self.optimizer_policy.value}" ] + if self.budget is not None: + project.cflags += [ + "-mllvm", + f"-vara-optimizer-starting-budget={self.budget}", + ] + actions = [] for binary in project.binaries: result_filepath = create_new_success_result_filepath( @@ -116,84 +126,127 @@ def optimizer_policy(self) -> OptimizerPolicyType: return OptimizerPolicyType.NAIVE -class RunTracedAlternating(RunTraced, shorthand=RunTraced.SHORTHAND + "A"): - """Build and run the traced version of the binary""" - - NAME = "RunTracedAlternating" +class RunTracedNaive20(RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "20"): + NAME = "RunTracedNaive20" @property @abstractmethod - def optimizer_policy(self) -> OptimizerPolicyType: - return OptimizerPolicyType.ALTERNATING + def budget(self) -> int: + return 20 -class RunTracedBudget(VaryingStartingBudgetExperiment, shorthand="RTB"): - """Build and run the traced version of the binary""" - - NAME = "RunTracedBudget" - REPORT_SPEC = ReportSpecification(WLTimeReportAggregate) +class RunTracedNaive40(RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "40"): + NAME = "RunTracedNaive40" @property @abstractmethod - def optimizer_policy(self) -> OptimizerPolicyType: - return OptimizerPolicyType.NONE + def budget(self) -> int: + return 40 - def actions_for_project( - self, project: VProject - ) -> tp.MutableSequence[actions.Step]: - - project.cflags += [ - "-mllvm", - f"-vara-optimizer-policy={self.optimizer_policy.value}", - ] - - actions = [] - for binary in project.binaries: - result_filepath = create_new_success_result_filepath( - self.get_handle(), - self.get_handle().report_spec().main_report, project, binary - ) - actions.append( - ZippedExperimentSteps( - result_filepath, [ - TimeProjectWorkloads( - project, - num, - binary, - categories=[ - WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL - ] - ) for num in range(MEASUREMENT_REPS) - ] - ) - ) - return self.get_common_tracing_actions( - project, FeatureInstrType.TEF, actions, save_temps=True - ) +class RunTracedNaive60(RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "60"): + NAME = "RunTracedNaive60" + @property + @abstractmethod + def budget(self) -> int: + return 60 -class RunTracedNaiveBudget( - RunTracedBudget, shorthand=RunTracedBudget.SHORTHAND + "N" -): - """Build and run the traced version of the binary""" - NAME = "RunTracedNaiveBudget" +class RunTracedNaive80(RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "80"): + NAME = "RunTracedNaive80" @property @abstractmethod - def optimizer_policy(self) -> OptimizerPolicyType: - return OptimizerPolicyType.NAIVE + def budget(self) -> int: + return 80 -class RunTracedAlternatingBudget( - RunTracedBudget, shorthand=RunTracedBudget.SHORTHAND + "A" -): - """Build and run the traced version of the binary""" - - NAME = "RunTracedAlternatingBudget" +class RunTracedNaive100(RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "100"): + NAME = "RunTracedNaive100" @property @abstractmethod - def optimizer_policy(self) -> OptimizerPolicyType: - return OptimizerPolicyType.ALTERNATING + def budget(self) -> int: + return 100 + + +# class RunTracedAlternating(RunTraced, shorthand=RunTraced.SHORTHAND + "A"): +# """Build and run the traced version of the binary""" + +# NAME = "RunTracedAlternating" + +# @property +# @abstractmethod +# def optimizer_policy(self) -> OptimizerPolicyType: +# return OptimizerPolicyType.ALTERNATING + + +# class RunTracedBudget(VaryingStartingBudgetExperiment, shorthand="RTB"): +# """Build and run the traced version of the binary""" + +# NAME = "RunTracedBudget" +# REPORT_SPEC = ReportSpecification(WLTimeReportAggregate) + +# @property +# @abstractmethod +# def optimizer_policy(self) -> OptimizerPolicyType: +# return OptimizerPolicyType.NONE + +# def actions_for_project( +# self, project: VProject +# ) -> tp.MutableSequence[actions.Step]: + +# project.cflags += [ +# "-mllvm", +# f"-vara-optimizer-policy={self.optimizer_policy.value}", +# ] + +# actions = [] +# for binary in project.binaries: +# result_filepath = create_new_success_result_filepath( +# self.get_handle(), +# self.get_handle().report_spec().main_report, project, binary +# ) +# actions.append( +# ZippedExperimentSteps( +# result_filepath, [ +# TimeProjectWorkloads( +# project, +# num, +# binary, +# categories=[ +# WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL +# ] +# ) for num in range(MEASUREMENT_REPS) +# ] +# ) +# ) + +# return self.get_common_tracing_actions( +# project, FeatureInstrType.TEF, actions, save_temps=True +# ) + +# class RunTracedNaiveBudget( +# RunTracedBudget, shorthand=RunTracedBudget.SHORTHAND + "N" +# ): +# """Build and run the traced version of the binary""" + +# NAME = "RunTracedNaiveBudget" + +# @property +# @abstractmethod +# def optimizer_policy(self) -> OptimizerPolicyType: +# return OptimizerPolicyType.NAIVE + +# class RunTracedAlternatingBudget( +# RunTracedBudget, shorthand=RunTracedBudget.SHORTHAND + "A" +# ): +# """Build and run the traced version of the binary""" + +# NAME = "RunTracedAlternatingBudget" + +# @property +# @abstractmethod +# def optimizer_policy(self) -> OptimizerPolicyType: +# return OptimizerPolicyType.ALTERNATING diff --git a/varats/varats/experiments/vara/dynamic_overhead_analysis.py b/varats/varats/experiments/vara/dynamic_overhead_analysis.py index b6ec9d012..f0949767f 100644 --- a/varats/varats/experiments/vara/dynamic_overhead_analysis.py +++ b/varats/varats/experiments/vara/dynamic_overhead_analysis.py @@ -20,36 +20,67 @@ class RunInstrVerifierNaive( def actions_for_project(self, project: VProject) -> MutableSequence[Step]: project.cflags += [ - "-mllvm", "-vara-optimizer-policy=naive", "-vara-optimizer-starting-budget=0", "-mllvm", + "-mllvm", "-vara-optimizer-policy=naive", "-mllvm", "-debug-only=OPT,IRT,InstrMark" ] return super().actions_for_project(project) -class RunInstrVerifierAlternating( - RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "A" +class RunInstrVerifierNaive40( + RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "N40" ): - NAME = "RunInstrVerifierAlternating" + NAME = "RunInstrVerifierNaive40" def actions_for_project(self, project: VProject) -> MutableSequence[Step]: project.cflags += [ - "-mllvm", "-vara-optimizer-policy=alternating", "-mllvm", + "-mllvm", "-vara-optimizer-policy=naive", "-mllvm", + "-vara-optimizer-starting-budget=40", "-mllvm", "-debug-only=OPT,IRT,InstrMark" ] return super().actions_for_project(project) -class RunInstrVerifierLoopExtract( - RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "L" + +class RunInstrVerifierNaive60( + RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "N60" ): - NAME = "RunInstrVerifierLoopExtract" + NAME = "RunInstrVerifierNaive60" def actions_for_project(self, project: VProject) -> MutableSequence[Step]: project.cflags += [ - "-mllvm", "-vara-optimizer-policy=loop_extract", "-mllvm", + "-mllvm", "-vara-optimizer-policy=naive", "-mllvm", + "-vara-optimizer-starting-budget=60", "-mllvm", "-debug-only=OPT,IRT,InstrMark" ] return super().actions_for_project(project) + +class RunInstrVerifierNaive80( + RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "N60" +): + NAME = "RunInstrVerifierNaive80" + + def actions_for_project(self, project: VProject) -> MutableSequence[Step]: + project.cflags += [ + "-mllvm", "-vara-optimizer-policy=naive", "-mllvm", + "-vara-optimizer-starting-budget=80", "-mllvm", + "-debug-only=OPT,IRT,InstrMark" + ] + return super().actions_for_project(project) + + +class RunInstrVerifierAlternating( + RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "A" +): + NAME = "RunInstrVerifierAlternating" + + def actions_for_project(self, project: VProject) -> MutableSequence[Step]: + project.cflags += [ + "-mllvm", "-vara-optimizer-policy=alternating", "-mllvm", + "-debug-only=OPT,IRT,InstrMark" + ] + return super().actions_for_project(project) + + class RunInstrVerifierNaiveBudget( RunInstrVerifierBudget, shorthand=RunInstrVerifierBudget.SHORTHAND + "N" ): @@ -71,4 +102,3 @@ class RunInstrVerifierAlternatingBudget( def actions_for_project(self, project: VProject) -> MutableSequence[Step]: project.cflags += ["-mllvm", "-vara-optimizer-policy=alternating"] return super().actions_for_project(project) - diff --git a/varats/varats/experiments/vara/instrumentation_verifier.py b/varats/varats/experiments/vara/instrumentation_verifier.py index 31acb7d68..0927ed842 100644 --- a/varats/varats/experiments/vara/instrumentation_verifier.py +++ b/varats/varats/experiments/vara/instrumentation_verifier.py @@ -44,6 +44,7 @@ def actions_for_project( self.get_handle(), report_file_ending="ivr", workload_categories=[ + # WorkloadCategory.SMALL WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL ] ) diff --git a/varats/varats/plots/compare_traced_budget.py b/varats/varats/plots/compare_traced_budget.py index 180cefb6e..f96e9db1a 100644 --- a/varats/varats/plots/compare_traced_budget.py +++ b/varats/varats/plots/compare_traced_budget.py @@ -13,10 +13,10 @@ from varats.plot.plots import PlotGenerator from varats.report.gnu_time_report import WLTimeReportAggregate from varats.revision.revisions import get_processed_revisions_files -from varats.ts_utils.click_param_types import REQUIRE_MULTI_CASE_STUDY, REQUIRE_EXPERIMENT_TYPE +from varats.ts_utils.click_param_types import REQUIRE_MULTI_CASE_STUDY, REQUIRE_MULTI_EXPERIMENT_TYPE from varats.utils.git_util import FullCommitHash -starting_budget_command_regex = re.compile("starting_budget_([0-9]*)") +starting_budget_command_regex = re.compile("RunTracedNaive([0-9]+)") class CompareRuntimesBudgetPlot(Plot, plot_name="compare_runtimes_budget"): @@ -27,41 +27,46 @@ def plot(self, view_mode: bool) -> None: for case_study in self.plot_kwargs["case_study"]: project_name = case_study.project_name - experiment = self.plot_kwargs["experiment_type"] - report_files = get_processed_revisions_files( - project_name, - experiment, - WLTimeReportAggregate, - get_case_study_file_name_filter(case_study), - only_newest=False - ) - - for report_filepath in report_files: - agg_time_report = WLTimeReportAggregate( - report_filepath.full_path() + experiments = self.plot_kwargs["experiment_type"] + + for experiment in experiments: + report_files = get_processed_revisions_files( + project_name, + experiment, + WLTimeReportAggregate, + get_case_study_file_name_filter(case_study), + only_newest=False ) - for workload_name in agg_time_report.workload_names(): - for report in agg_time_report.reports(workload_name): - m = re.search( - starting_budget_command_regex, report.command_name - ) - if m is None: - budget = 20 - else: - budget = int(m.group(1)) - - new_row = { - "Workload": - workload_name, - "Budget": - budget, - "Mean wall time (msecs)": - report.wall_clock_time.total_seconds() - } - - df = pd.concat([df, pd.DataFrame([new_row])], - ignore_index=True) + if ( + m := + re.search(starting_budget_command_regex, experiment.NAME) + ) is not None: + budget = int(m.group(1)) + elif experiment.NAME.startswith("RunTracedNaive"): + budget = 20 + else: + budget = 0 + + for report_filepath in report_files: + agg_time_report = WLTimeReportAggregate( + report_filepath.full_path() + ) + + for workload_name in agg_time_report.workload_names(): + for report in agg_time_report.reports(workload_name): + + new_row = { + "Workload": + workload_name, + "Budget": + budget, + "Mean wall time (msecs)": + report.wall_clock_time.total_seconds() + } + + df = pd.concat([df, pd.DataFrame([new_row])], + ignore_index=True) fig, ax = plt.subplots() fig.set_size_inches(11.7, 8.27) @@ -83,7 +88,7 @@ def calc_missing_revisions( class CompareRuntimesBudgetPlotCSGenerator( PlotGenerator, generator_name="compare-runtimes-budget", - options=[REQUIRE_EXPERIMENT_TYPE, REQUIRE_MULTI_CASE_STUDY] + options=[REQUIRE_MULTI_EXPERIMENT_TYPE, REQUIRE_MULTI_CASE_STUDY] ): def generate(self) -> tp.List[Plot]: diff --git a/varats/varats/projects/c_projects/bzip2.py b/varats/varats/projects/c_projects/bzip2.py index 52b407769..27efc1111 100644 --- a/varats/varats/projects/c_projects/bzip2.py +++ b/varats/varats/projects/c_projects/bzip2.py @@ -82,6 +82,23 @@ class Bzip2(VProject): ] WORKLOADS = { + WorkloadSet(WorkloadCategory.SMALL): [ + Command( + SourceRoot("bzip2") / RSBinary("bzip2"), + "--compress", + "--best", + "-vvv", + "--keep", + # bzip2 compresses very fast even on the best setting, so we + # need the three input files to get approximately 30 seconds + # total execution time + "geo-maps/countries-land-1m.geo.json", + label="countries-land-1m", + creates=[ + "geo-maps/countries-land-1m.geo.json.bz2", + ] + ) + ], WorkloadSet(WorkloadCategory.MEDIUM): [ Command( SourceRoot("bzip2") / RSBinary("bzip2"), @@ -95,6 +112,7 @@ class Bzip2(VProject): "geo-maps/countries-land-1m.geo.json", "geo-maps/countries-land-10m.geo.json", "geo-maps/countries-land-100m.geo.json", + label="countries-land-1m-10m-100m", creates=[ "geo-maps/countries-land-1m.geo.json.bz2", "geo-maps/countries-land-10m.geo.json.bz2", diff --git a/varats/varats/projects/c_projects/picosat.py b/varats/varats/projects/c_projects/picosat.py index dad18bf46..5cd27d536 100644 --- a/varats/varats/projects/c_projects/picosat.py +++ b/varats/varats/projects/c_projects/picosat.py @@ -105,11 +105,21 @@ class PicoSAT(VProject, ReleaseProviderHook): ) ], WorkloadSet(WorkloadCategory.SMALL): [ + # Command( + # SourceRoot("picosat") / RSBinary("picosat"), + # "aim-100-1_6-no-1.cnf", + # label="aim-100-1-6-no-1.cnf", + # ) Command( SourceRoot("picosat") / RSBinary("picosat"), - "aim-100-1_6-no-1.cnf", - label="aim-100-1-6-no-1.cnf", - ) + "traffic_kkb_unknown.cnf/traffic_kkb_unknown.cnf", + label="traffic-kkb-unknow.cnf", + ), + # Command( + # SourceRoot("picosat") / RSBinary("picosat"), + # "abw-N-bcsstk07.mtx-w44.cnf/abw-N-bcsstk07.mtx-w44.cnf", + # label="abw-N-bcsstk07.mtx-w44.cnf", + # ), ], WorkloadSet(WorkloadCategory.MEDIUM): [ Command( diff --git a/varats/varats/projects/c_projects/xz.py b/varats/varats/projects/c_projects/xz.py index 3d1a580ed..1fac7c349 100644 --- a/varats/varats/projects/c_projects/xz.py +++ b/varats/varats/projects/c_projects/xz.py @@ -2,7 +2,7 @@ import typing as tp import benchbuild as bb -from benchbuild.command import SourceRoot, WorkloadSet +from benchbuild.command import Command, SourceRoot, WorkloadSet from benchbuild.source import HTTPMultiple from benchbuild.utils.cmd import autoreconf, make from benchbuild.utils.revision_ranges import ( @@ -24,7 +24,6 @@ verify_binaries, ) from varats.project.sources import FeatureSource -from varats.project.varats_command import VCommand from varats.project.varats_project import VProject from varats.utils.git_util import ( ShortCommitHash, @@ -85,19 +84,16 @@ class Xz(VProject): WORKLOADS = { WorkloadSet(WorkloadCategory.EXAMPLE): [ - VCommand( + Command( SourceRoot("xz") / RSBinary("xz"), "-k", - # Use output_param to ensure input file - # gets appended after all arguments. - output_param=["{output}"], - output=SourceRoot("geo-maps/countries-land-250m.geo.json"), + "geo-maps/countries-land-1km.geo.json", label="countries-land-1km", creates=["geo-maps/countries-land-1km.geo.json.xz"] ) ], WorkloadSet(WorkloadCategory.MEDIUM): [ - VCommand( + Command( SourceRoot("xz") / RSBinary("xz"), "-k", "-9e", @@ -105,13 +101,9 @@ class Xz(VProject): "--threads=1", "--format=xz", "-vv", - # Use output_param to ensure input file - # gets appended after all arguments. - output_param=["{output}"], - output=SourceRoot("geo-maps/countries-land-250m.geo.json"), + "geo-maps/countries-land-250m.geo.json", label="countries-land-250m", - creates=["geo-maps/countries-land-250m.geo.json.xz"], - requires_all_args={"--compress"}, + creates=["geo-maps/countries-land-250m.geo.json.xz"] ) ], } diff --git a/varats/varats/ts_utils/__init__.py b/varats/varats/ts_utils/__init__.py new file mode 100644 index 000000000..e69de29bb From 5f0772064254756c425d40c8c89ad230a0554977 Mon Sep 17 00:00:00 2001 From: Daniel Gusenburger Date: Mon, 16 Oct 2023 09:22:09 +0200 Subject: [PATCH 31/31] final commit --- .../vara/compare_traced_untraced.py | 157 +++++++----------- .../vara/dynamic_overhead_analysis.py | 74 ++++++--- .../experiments/vara/feature_experiment.py | 3 - .../vara/instrumentation_verifier.py | 17 +- .../experiments/vara/output_verifier.py | 0 varats/varats/plots/compare_traced_budget.py | 126 ++++++++------ varats/varats/plots/compare_traced_cs.py | 2 - .../plots/compare_traced_cs_budget_labels.py | 90 ++++++++++ ...n_verifier_compare_experiments_overview.py | 41 ++--- ...pare_experiments_overview_budget_labels.py | 155 +++++++++++++++++ .../instrumentation_verifier_overview.py | 2 +- ...nstrumentation_verifier_overview_budget.py | 19 ++- varats/varats/projects/c_projects/picosat.py | 23 ++- .../perf_tests/feature_perf_cs_collection.py | 26 ++- varats/varats/tables/time_workloads.py | 5 +- varats/varats/tables/time_workloads2.py | 109 ++++++++++++ 16 files changed, 598 insertions(+), 251 deletions(-) create mode 100644 varats/varats/experiments/vara/output_verifier.py create mode 100644 varats/varats/plots/compare_traced_cs_budget_labels.py create mode 100644 varats/varats/plots/instrumentation_verifier_compare_experiments_overview_budget_labels.py create mode 100644 varats/varats/tables/time_workloads2.py diff --git a/varats/varats/experiments/vara/compare_traced_untraced.py b/varats/varats/experiments/vara/compare_traced_untraced.py index 6548a0faf..4875af442 100644 --- a/varats/varats/experiments/vara/compare_traced_untraced.py +++ b/varats/varats/experiments/vara/compare_traced_untraced.py @@ -17,9 +17,8 @@ from varats.experiments.vara.dynamic_overhead_analysis import OptimizerPolicyType from varats.experiments.vara.feature_experiment import FeatureExperiment, FeatureInstrType -from varats.experiments.vara.multi_compile_experiment import VaryingStartingBudgetExperiment -MEASUREMENT_REPS = 1 +MEASUREMENT_REPS = 10 class RunUntraced(FeatureExperiment, shorthand="RU"): @@ -73,22 +72,19 @@ def optimizer_policy(self) -> OptimizerPolicyType: @property @abstractmethod def budget(self) -> tp.Optional[int]: - return None + return 0 def actions_for_project( self, project: VProject ) -> tp.MutableSequence[actions.Step]: project.cflags += [ - "-mllvm", f"-vara-optimizer-policy={self.optimizer_policy.value}" + "-mllvm", + f"-vara-optimizer-policy={self.optimizer_policy.value}", + "-mllvm", + f"-vara-optimizer-starting-budget={self.budget}", ] - if self.budget is not None: - project.cflags += [ - "-mllvm", - f"-vara-optimizer-starting-budget={self.budget}", - ] - actions = [] for binary in project.binaries: result_filepath = create_new_success_result_filepath( @@ -126,7 +122,9 @@ def optimizer_policy(self) -> OptimizerPolicyType: return OptimizerPolicyType.NAIVE -class RunTracedNaive20(RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "20"): +class RunTracedNaive20( + RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "20" +): NAME = "RunTracedNaive20" @property @@ -135,7 +133,9 @@ def budget(self) -> int: return 20 -class RunTracedNaive40(RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "40"): +class RunTracedNaive40( + RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "40" +): NAME = "RunTracedNaive40" @property @@ -144,7 +144,9 @@ def budget(self) -> int: return 40 -class RunTracedNaive60(RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "60"): +class RunTracedNaive60( + RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "60" +): NAME = "RunTracedNaive60" @property @@ -153,7 +155,9 @@ def budget(self) -> int: return 60 -class RunTracedNaive80(RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "80"): +class RunTracedNaive80( + RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "80" +): NAME = "RunTracedNaive80" @property @@ -162,7 +166,9 @@ def budget(self) -> int: return 80 -class RunTracedNaive100(RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "100"): +class RunTracedNaive100( + RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "100" +): NAME = "RunTracedNaive100" @property @@ -171,82 +177,45 @@ def budget(self) -> int: return 100 -# class RunTracedAlternating(RunTraced, shorthand=RunTraced.SHORTHAND + "A"): -# """Build and run the traced version of the binary""" - -# NAME = "RunTracedAlternating" - -# @property -# @abstractmethod -# def optimizer_policy(self) -> OptimizerPolicyType: -# return OptimizerPolicyType.ALTERNATING - - -# class RunTracedBudget(VaryingStartingBudgetExperiment, shorthand="RTB"): -# """Build and run the traced version of the binary""" - -# NAME = "RunTracedBudget" -# REPORT_SPEC = ReportSpecification(WLTimeReportAggregate) - -# @property -# @abstractmethod -# def optimizer_policy(self) -> OptimizerPolicyType: -# return OptimizerPolicyType.NONE - -# def actions_for_project( -# self, project: VProject -# ) -> tp.MutableSequence[actions.Step]: - -# project.cflags += [ -# "-mllvm", -# f"-vara-optimizer-policy={self.optimizer_policy.value}", -# ] - -# actions = [] -# for binary in project.binaries: -# result_filepath = create_new_success_result_filepath( -# self.get_handle(), -# self.get_handle().report_spec().main_report, project, binary -# ) -# actions.append( -# ZippedExperimentSteps( -# result_filepath, [ -# TimeProjectWorkloads( -# project, -# num, -# binary, -# categories=[ -# WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL -# ] -# ) for num in range(MEASUREMENT_REPS) -# ] -# ) -# ) - -# return self.get_common_tracing_actions( -# project, FeatureInstrType.TEF, actions, save_temps=True -# ) - -# class RunTracedNaiveBudget( -# RunTracedBudget, shorthand=RunTracedBudget.SHORTHAND + "N" -# ): -# """Build and run the traced version of the binary""" - -# NAME = "RunTracedNaiveBudget" - -# @property -# @abstractmethod -# def optimizer_policy(self) -> OptimizerPolicyType: -# return OptimizerPolicyType.NAIVE - -# class RunTracedAlternatingBudget( -# RunTracedBudget, shorthand=RunTracedBudget.SHORTHAND + "A" -# ): -# """Build and run the traced version of the binary""" - -# NAME = "RunTracedAlternatingBudget" - -# @property -# @abstractmethod -# def optimizer_policy(self) -> OptimizerPolicyType: -# return OptimizerPolicyType.ALTERNATING +class RunTracedNaive200( + RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "200" +): + NAME = "RunTracedNaive200" + + @property + @abstractmethod + def budget(self) -> int: + return 200 + + +class RunTracedNaive500( + RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "500" +): + NAME = "RunTracedNaive500" + + @property + @abstractmethod + def budget(self) -> int: + return 500 + + +class RunTracedNaive1000( + RunTracedNaive, shorthand=RunTracedNaive.SHORTHAND + "1000" +): + NAME = "RunTracedNaive1000" + + @property + @abstractmethod + def budget(self) -> int: + return 1000 + + +class RunTracedAlternating(RunTraced, shorthand=RunTraced.SHORTHAND + "A"): + """Build and run the traced version of the binary""" + + NAME = "RunTracedAlternating" + + @property + @abstractmethod + def optimizer_policy(self) -> OptimizerPolicyType: + return OptimizerPolicyType.ALTERNATING diff --git a/varats/varats/experiments/vara/dynamic_overhead_analysis.py b/varats/varats/experiments/vara/dynamic_overhead_analysis.py index f0949767f..933ab3528 100644 --- a/varats/varats/experiments/vara/dynamic_overhead_analysis.py +++ b/varats/varats/experiments/vara/dynamic_overhead_analysis.py @@ -13,15 +13,15 @@ class OptimizerPolicyType(Enum): ALTERNATING = "alternating" -class RunInstrVerifierNaive( - RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "N" +class RunInstrVerifierNaive20( + RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "N20" ): - NAME = "RunInstrVerifierNaive" + NAME = "RunInstrVerifierNaive20" def actions_for_project(self, project: VProject) -> MutableSequence[Step]: project.cflags += [ "-mllvm", "-vara-optimizer-policy=naive", "-mllvm", - "-debug-only=OPT,IRT,InstrMark" + "-vara-optimizer-starting-budget=20" ] return super().actions_for_project(project) @@ -34,8 +34,7 @@ class RunInstrVerifierNaive40( def actions_for_project(self, project: VProject) -> MutableSequence[Step]: project.cflags += [ "-mllvm", "-vara-optimizer-policy=naive", "-mllvm", - "-vara-optimizer-starting-budget=40", "-mllvm", - "-debug-only=OPT,IRT,InstrMark" + "-vara-optimizer-starting-budget=40" ] return super().actions_for_project(project) @@ -48,57 +47,84 @@ class RunInstrVerifierNaive60( def actions_for_project(self, project: VProject) -> MutableSequence[Step]: project.cflags += [ "-mllvm", "-vara-optimizer-policy=naive", "-mllvm", - "-vara-optimizer-starting-budget=60", "-mllvm", - "-debug-only=OPT,IRT,InstrMark" + "-vara-optimizer-starting-budget=60" ] return super().actions_for_project(project) class RunInstrVerifierNaive80( - RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "N60" + RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "N80" ): NAME = "RunInstrVerifierNaive80" def actions_for_project(self, project: VProject) -> MutableSequence[Step]: project.cflags += [ "-mllvm", "-vara-optimizer-policy=naive", "-mllvm", - "-vara-optimizer-starting-budget=80", "-mllvm", - "-debug-only=OPT,IRT,InstrMark" + "-vara-optimizer-starting-budget=80" ] return super().actions_for_project(project) -class RunInstrVerifierAlternating( - RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "A" +class RunInstrVerifierNaive100( + RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "N100" ): - NAME = "RunInstrVerifierAlternating" + NAME = "RunInstrVerifierNaive100" def actions_for_project(self, project: VProject) -> MutableSequence[Step]: project.cflags += [ - "-mllvm", "-vara-optimizer-policy=alternating", "-mllvm", - "-debug-only=OPT,IRT,InstrMark" + "-mllvm", "-vara-optimizer-policy=naive", "-mllvm", + "-vara-optimizer-starting-budget=100" ] return super().actions_for_project(project) -class RunInstrVerifierNaiveBudget( - RunInstrVerifierBudget, shorthand=RunInstrVerifierBudget.SHORTHAND + "N" +class RunInstrVerifierNaive200( + RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "N200" ): - NAME = "RunInstrVerifierNaiveBudget" + NAME = "RunInstrVerifierNaive200" def actions_for_project(self, project: VProject) -> MutableSequence[Step]: project.cflags += [ "-mllvm", "-vara-optimizer-policy=naive", "-mllvm", - "-debug-only=OPT,IRT,InstrMark" + "-vara-optimizer-starting-budget=200" + ] + return super().actions_for_project(project) + + +class RunInstrVerifierNaive500( + RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "N500" +): + NAME = "RunInstrVerifierNaive500" + + def actions_for_project(self, project: VProject) -> MutableSequence[Step]: + project.cflags += [ + "-mllvm", "-vara-optimizer-policy=naive", "-mllvm", + "-vara-optimizer-starting-budget=500" + ] + return super().actions_for_project(project) + + +class RunInstrVerifierNaive1000( + RunInstrVerifier, shorthand=RunInstrVerifier.SHORTHAND + "N1000" +): + NAME = "RunInstrVerifierNaive1000" + + def actions_for_project(self, project: VProject) -> MutableSequence[Step]: + project.cflags += [ + "-mllvm", "-vara-optimizer-policy=naive", "-mllvm", + "-vara-optimizer-starting-budget=1000" ] return super().actions_for_project(project) -class RunInstrVerifierAlternatingBudget( - RunInstrVerifierBudget, shorthand=RunInstrVerifierBudget.SHORTHAND + "A" +class RunInstrVerifierNaiveBudget( + RunInstrVerifierBudget, shorthand=RunInstrVerifierBudget.SHORTHAND + "N" ): - NAME = "RunInstrVerifierAlternatingBudget" + NAME = "RunInstrVerifierNaiveBudget" def actions_for_project(self, project: VProject) -> MutableSequence[Step]: - project.cflags += ["-mllvm", "-vara-optimizer-policy=alternating"] + project.cflags += [ + "-mllvm", "-vara-optimizer-policy=naive", "-mllvm", + "-debug-only=OPT,IRT,InstrMark" + ] return super().actions_for_project(project) diff --git a/varats/varats/experiments/vara/feature_experiment.py b/varats/varats/experiments/vara/feature_experiment.py index e4b5ddf84..0316c9190 100644 --- a/varats/varats/experiments/vara/feature_experiment.py +++ b/varats/varats/experiments/vara/feature_experiment.py @@ -282,9 +282,6 @@ def run_traced_code(self) -> StepResult: pb_cmd = prj_command.command.as_plumbum( project=self.project ) - print( - f"Running example {prj_command.command.label}" - ) extra_options = get_extra_config_options( self.project diff --git a/varats/varats/experiments/vara/instrumentation_verifier.py b/varats/varats/experiments/vara/instrumentation_verifier.py index 0927ed842..c653f8b4a 100644 --- a/varats/varats/experiments/vara/instrumentation_verifier.py +++ b/varats/varats/experiments/vara/instrumentation_verifier.py @@ -35,21 +35,16 @@ def actions_for_project( Args: project: to analyze """ - analysis_actions = [] - - analysis_actions.append(actions.Compile(project)) - analysis_actions.append( + analysis_actions = [ RunVaRATracedWorkloads( project, self.get_handle(), report_file_ending="ivr", workload_categories=[ - # WorkloadCategory.SMALL WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL ] ) - ) - analysis_actions.append(actions.Clean(project)) + ] return self.get_common_tracing_actions( project, @@ -78,10 +73,7 @@ def actions_for_project( project: to analyze """ - analysis_actions = [] - - analysis_actions.append(actions.Compile(project)) - analysis_actions.append( + analysis_actions = [ RunVaRATracedWorkloads( project, self.get_handle(), @@ -90,8 +82,7 @@ def actions_for_project( WorkloadCategory.EXAMPLE, WorkloadCategory.SMALL ], ) - ) - analysis_actions.append(actions.Clean(project)) + ] return self.get_common_tracing_actions( project, diff --git a/varats/varats/experiments/vara/output_verifier.py b/varats/varats/experiments/vara/output_verifier.py new file mode 100644 index 000000000..e69de29bb diff --git a/varats/varats/plots/compare_traced_budget.py b/varats/varats/plots/compare_traced_budget.py index f96e9db1a..8887b4a72 100644 --- a/varats/varats/plots/compare_traced_budget.py +++ b/varats/varats/plots/compare_traced_budget.py @@ -13,7 +13,7 @@ from varats.plot.plots import PlotGenerator from varats.report.gnu_time_report import WLTimeReportAggregate from varats.revision.revisions import get_processed_revisions_files -from varats.ts_utils.click_param_types import REQUIRE_MULTI_CASE_STUDY, REQUIRE_MULTI_EXPERIMENT_TYPE +from varats.ts_utils.click_param_types import REQUIRE_CASE_STUDY, REQUIRE_MULTI_EXPERIMENT_TYPE from varats.utils.git_util import FullCommitHash starting_budget_command_regex = re.compile("RunTracedNaive([0-9]+)") @@ -24,60 +24,78 @@ class CompareRuntimesBudgetPlot(Plot, plot_name="compare_runtimes_budget"): def plot(self, view_mode: bool) -> None: df = pd.DataFrame() - for case_study in self.plot_kwargs["case_study"]: - project_name = case_study.project_name - - experiments = self.plot_kwargs["experiment_type"] - - for experiment in experiments: - report_files = get_processed_revisions_files( - project_name, - experiment, - WLTimeReportAggregate, - get_case_study_file_name_filter(case_study), - only_newest=False + case_study = self.plot_kwargs["case_study"] + project_name = case_study.project_name + + experiments = self.plot_kwargs["experiment_type"] + + for experiment in experiments: + report_files = get_processed_revisions_files( + project_name, + experiment, + WLTimeReportAggregate, + get_case_study_file_name_filter(case_study), + only_newest=False + ) + + budget = "0" + if ( + m := re.search(starting_budget_command_regex, experiment.NAME) + ) is not None: + budget = m.group(1) + elif experiment.NAME == "RunUntraced": + budget = "Untraced" + + for report_filepath in report_files: + agg_time_report = WLTimeReportAggregate( + report_filepath.full_path() ) - if ( - m := - re.search(starting_budget_command_regex, experiment.NAME) - ) is not None: - budget = int(m.group(1)) - elif experiment.NAME.startswith("RunTracedNaive"): - budget = 20 - else: - budget = 0 - - for report_filepath in report_files: - agg_time_report = WLTimeReportAggregate( - report_filepath.full_path() - ) - - for workload_name in agg_time_report.workload_names(): - for report in agg_time_report.reports(workload_name): - - new_row = { - "Workload": - workload_name, - "Budget": - budget, - "Mean wall time (msecs)": - report.wall_clock_time.total_seconds() - } - - df = pd.concat([df, pd.DataFrame([new_row])], - ignore_index=True) - - fig, ax = plt.subplots() - fig.set_size_inches(11.7, 8.27) - sns.barplot( - x="Budget", - y="Mean wall time (msecs)", - estimator=np.mean, - data=df, - ax=ax, - ) - sns.despine() + for workload_name in agg_time_report.workload_names(): + for report in agg_time_report.reports(workload_name): + new_row = { + "Workload": + workload_name, + "Budget": + budget, + "Mean wall time (secs)": + report.wall_clock_time.total_seconds() + } + + df = pd.concat([df, pd.DataFrame([new_row])], + ignore_index=True) + + df = df.drop(df[df["Workload"] == "example.cnf"].index) + workloads = df["Workload"].unique() + + fig, axs = plt.subplots((1 + len(workloads)) // 2, + 2 - len(workloads) % 2, + constrained_layout=True) + + for i, workload in enumerate(workloads): + if len(workloads) == 1: + ax = axs + elif len(workloads) == 2: + ax = axs[i % 2] + else: + x, y = divmod(i, 2) + ax = axs[(x, y)] + + d = df[df["Workload"] == workload] + + sns.barplot( + x="Budget", + y="Mean wall time (secs)", + estimator=np.mean, + data=d, + ax=ax, + ) + ax.set_xticks( + ax.get_xticks(), ax.get_xticklabels(), rotation=45, ha='right' + ) + ax.set_title(workload) + + fig.suptitle(f"Runtimes by budget for {case_study.project_name}") def calc_missing_revisions( self, boundary_gradient: float @@ -88,7 +106,7 @@ def calc_missing_revisions( class CompareRuntimesBudgetPlotCSGenerator( PlotGenerator, generator_name="compare-runtimes-budget", - options=[REQUIRE_MULTI_EXPERIMENT_TYPE, REQUIRE_MULTI_CASE_STUDY] + options=[REQUIRE_MULTI_EXPERIMENT_TYPE, REQUIRE_CASE_STUDY] ): def generate(self) -> tp.List[Plot]: diff --git a/varats/varats/plots/compare_traced_cs.py b/varats/varats/plots/compare_traced_cs.py index f64753f4c..76bdacb6b 100644 --- a/varats/varats/plots/compare_traced_cs.py +++ b/varats/varats/plots/compare_traced_cs.py @@ -44,7 +44,6 @@ def plot(self, view_mode: bool) -> None: agg_time_report = WLTimeReportAggregate( report_filepath.full_path() ) - report_file = agg_time_report.filename for workload_name in agg_time_report.workload_names(): for wall_clock_time in \ @@ -63,7 +62,6 @@ def plot(self, view_mode: bool) -> None: df = pd.concat([df, pd.DataFrame([new_row])], ignore_index=True) - print(df) fig, ax = plt.subplots() fig.set_size_inches(11.7, 8.27) sns.barplot( diff --git a/varats/varats/plots/compare_traced_cs_budget_labels.py b/varats/varats/plots/compare_traced_cs_budget_labels.py new file mode 100644 index 000000000..a38ae64b2 --- /dev/null +++ b/varats/varats/plots/compare_traced_cs_budget_labels.py @@ -0,0 +1,90 @@ +"""Example table that uses different workloads and visualizes the time it took +to run them.""" +import typing as tp + +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sns +import numpy as np + +from varats.paper.paper_config import get_loaded_paper_config +from varats.paper_mgmt.case_study import get_case_study_file_name_filter +from varats.plot.plot import Plot +from varats.plot.plots import PlotGenerator +from varats.report.gnu_time_report import WLTimeReportAggregate +from varats.revision.revisions import get_processed_revisions_files +from varats.ts_utils.click_param_types import REQUIRE_MULTI_CASE_STUDY, REQUIRE_MULTI_EXPERIMENT_TYPE +from varats.utils.git_util import FullCommitHash + +# TODO: Is there a better way to include revisions of all workloads than to use +# only_newest=False ? +# Maybe the result files are not defined correctly. We should be able to find +# the revision files for all workloads with only_newest=True... + + +class CompareRuntimesCSBPlot(Plot, plot_name="compare_runtimes_csb"): + + def plot(self, view_mode: bool) -> None: + df = pd.DataFrame() + + print(self.plot_kwargs["case_study"]) + for case_study in self.plot_kwargs["case_study"]: + project_name = case_study.project_name + + for experiment in self.plot_kwargs["experiment_type"]: + report_files = get_processed_revisions_files( + project_name, + experiment, + WLTimeReportAggregate, + get_case_study_file_name_filter(case_study), + only_newest=True + ) + + for report_filepath in report_files: + agg_time_report = WLTimeReportAggregate( + report_filepath.full_path() + ) + + for workload_name in agg_time_report.workload_names(): + for wall_clock_time in \ + agg_time_report.measurements_wall_clock_time( + workload_name + ): + new_row = { + "Workload": + workload_name, + "Experiment": + experiment.NAME, + "Mean wall time (msecs)": + wall_clock_time * 1000, + } + + df = pd.concat([df, pd.DataFrame([new_row])], + ignore_index=True) + + fig, ax = plt.subplots() + fig.set_size_inches(11.7, 8.27) + sns.barplot( + x="Workload", + y="Mean wall time (msecs)", + hue="Experiment", + estimator=np.mean, + data=df, + ax=ax, + ) + sns.despine() + + def calc_missing_revisions( + self, boundary_gradient: float + ) -> tp.Set[FullCommitHash]: + raise NotImplementedError + + +class CompareRuntimesPlotCSBGenerator( + PlotGenerator, + generator_name="compare-runtimes-csb", + options=[REQUIRE_MULTI_EXPERIMENT_TYPE, REQUIRE_MULTI_CASE_STUDY] +): + + def generate(self) -> tp.List[Plot]: + return [CompareRuntimesCSPlot(self.plot_config, **self.plot_kwargs)] diff --git a/varats/varats/plots/instrumentation_verifier_compare_experiments_overview.py b/varats/varats/plots/instrumentation_verifier_compare_experiments_overview.py index ab0bb67c5..c1c55174f 100644 --- a/varats/varats/plots/instrumentation_verifier_compare_experiments_overview.py +++ b/varats/varats/plots/instrumentation_verifier_compare_experiments_overview.py @@ -15,7 +15,7 @@ from varats.plot.plots import PlotGenerator from varats.report.report import ReportFilepath from varats.revision.revisions import get_all_revisions_files -from varats.ts_utils.click_param_types import REQUIRE_MULTI_EXPERIMENT_TYPE +from varats.ts_utils.click_param_types import REQUIRE_MULTI_EXPERIMENT_TYPE, REQUIRE_MULTI_CASE_STUDY from varats.utils.exceptions import UnsupportedOperation from varats.utils.git_util import FullCommitHash @@ -45,7 +45,6 @@ def _generate_plot(**kwargs: tp.Any) -> None: rows = [] for experiment in kwargs["experiment_type"]: - revisions_files: tp.List[ReportFilepath] = get_all_revisions_files( case_study.project_name, experiment, only_newest=False ) @@ -72,7 +71,9 @@ def _generate_plot(**kwargs: tp.Any) -> None: df = pd.DataFrame(rows) binaries = df["binary"].unique() experiments = df["experiment"].unique() - fig, axs = plt.subplots((1 + len(binaries)) // 2, 2 - len(binaries) % 2) + fig, axs = plt.subplots((1 + len(binaries)) // 2, + 2 - len(binaries) % 2, + constrained_layout=True) for i, binary in enumerate(binaries): if len(binaries) == 1: @@ -80,7 +81,8 @@ def _generate_plot(**kwargs: tp.Any) -> None: elif len(binaries) == 2: ax = axs[i % 2] else: - ax = axs[*divmod(i, 2)] + x, y = divmod(i, 2) + ax = axs[(x, y)] d = df[df["binary"] == binary] @@ -89,13 +91,7 @@ def _generate_plot(**kwargs: tp.Any) -> None: num_unclosed_enters = np.array(d["unclosed_enters"]) num_unentered_leaves = np.array(d["unentered_leaves"]) - # num_enters = num_enters - num_unclosed_enters - # num_leaves = num_leaves - num_unentered_leaves - - ax.bar( - experiments, - num_enters - ) + ax.bar(experiments, num_enters) ax.bar( experiments, num_leaves, @@ -112,35 +108,32 @@ def _generate_plot(**kwargs: tp.Any) -> None: bottom=num_enters + num_leaves + num_unclosed_enters ) - ax.set_ylabel("Number of events") - ax.set_xticks(ax.get_xticks(), ax.get_xticklabels(), rotation=45, ha='right') + ax.set_xticks( + ax.get_xticks(), ax.get_xticklabels() + ) ax.set_title(binary) - # ax.set_xticks( - # minor_ticks, - # labels=minor_labels, - # rotation=30, - # ha="right", - # ) fig.suptitle( f"Instrumentation Verifier " f"Overview for {case_study.project_name}" ) - fig.legend(labels=["Enters", "Leaves", "Unclosed enters", "Unentered leaves"]) - plt.subplots_adjust(bottom=0.25) + fig.legend( + labels=["Enters", "Leaves", "Unclosed enters", "Unentered leaves"] + ) class VerifierExperimentCompareOverviewGenerator( PlotGenerator, generator_name="iv-ce-overview-plot", - options=[REQUIRE_MULTI_EXPERIMENT_TYPE] + options=[REQUIRE_MULTI_EXPERIMENT_TYPE, REQUIRE_MULTI_CASE_STUDY] ): """Generates a single pc-overview plot for the current paper config.""" def generate(self) -> tp.List[Plot]: + case_studies = self.plot_kwargs.pop("case_study") return [ InstrumentationOverviewCompareExperimentsPlot( - self.plot_config, case_study=cs, **self.plot_kwargs - ) for cs in PC.get_paper_config().get_all_case_studies() + self.plot_config, cse_study=cs, **self.plot_kwargs + ) for cs in case_studies ] diff --git a/varats/varats/plots/instrumentation_verifier_compare_experiments_overview_budget_labels.py b/varats/varats/plots/instrumentation_verifier_compare_experiments_overview_budget_labels.py new file mode 100644 index 000000000..6535dadec --- /dev/null +++ b/varats/varats/plots/instrumentation_verifier_compare_experiments_overview_budget_labels.py @@ -0,0 +1,155 @@ +"""Generate graphs that show an overview of the instrumentation verifier +experiment state for all case studies in the paper config.""" + +import typing as tp + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd + +import varats.paper.paper_config as PC +from varats.data.reports.instrumentation_verifier_report import ( + InstrVerifierReport, +) +from varats.plot.plot import Plot, PlotDataEmpty +from varats.plot.plots import PlotGenerator +from varats.report.report import ReportFilepath +from varats.revision.revisions import get_all_revisions_files +from varats.ts_utils.click_param_types import REQUIRE_MULTI_EXPERIMENT_TYPE, REQUIRE_MULTI_CASE_STUDY +from varats.utils.exceptions import UnsupportedOperation +from varats.utils.git_util import FullCommitHash +import re +from varats.paper_mgmt.case_study import get_case_study_file_name_filter + +starting_budget_command_regex = re.compile("RunInstrVerifierNaive([0-9]+)") + + +class InstrumentationOverviewCompareExperimentsBudgetLabelsPlot( + Plot, + plot_name="instrumentation_overview_compare_experiments_budget_labels_plot" +): + """ + Plot configuration for the instrumentation verifier experiment. + + This plot shows an overview of the instrumentation verifier state for all + case studies in the paper config. + """ + + def plot(self, view_mode: bool) -> None: + self._generate_plot(**self.plot_kwargs) + + def calc_missing_revisions( + self, boundary_gradient: float + ) -> tp.Set[FullCommitHash]: + raise UnsupportedOperation + + @staticmethod + def _generate_plot(**kwargs: tp.Any) -> None: + case_study = kwargs['case_study'] + + rows = [] + + for experiment in kwargs["experiment_type"]: + revisions_files: tp.List[ReportFilepath] = get_all_revisions_files( + case_study.project_name, + experiment, + InstrVerifierReport, + get_case_study_file_name_filter(case_study), + only_newest=False + ) + + reports: tp.List[InstrVerifierReport] = [ + InstrVerifierReport(rev_file.full_path()) + for rev_file in revisions_files + ] + + if len(reports) == 0: + raise PlotDataEmpty() + + budget = 0 + if ( + m := re.search(starting_budget_command_regex, experiment.NAME) + ) is not None: + budget = int(m.group(1)) + + for report in reports: + for binary in report.binaries(): + rows.append({ + "experiment": str(budget), + "binary": binary, + "enters": report.num_enters(binary), + "leaves": report.num_leaves(binary), + "unclosed_enters": report.num_unclosed_enters(binary), + "unentered_leaves": report.num_unentered_leaves(binary) + }) + + df = pd.DataFrame(rows) + df = df.drop(df[df["binary"] == "example"].index) + binaries = df["binary"].unique() + + experiments = df["experiment"].unique() + + fig, axs = plt.subplots((1 + len(binaries)) // 2, + 2 - len(binaries) % 2, + constrained_layout=True) + + for i, binary in enumerate(binaries): + if len(binaries) == 1: + ax = axs + elif len(binaries) == 2: + ax = axs[i % 2] + else: + x, y = divmod(i, 2) + ax = axs[(x, y)] + + d = df[df["binary"] == binary] + + num_enters = np.array(d["enters"]) + num_leaves = np.array(d["leaves"]) + num_unclosed_enters = np.array(d["unclosed_enters"]) + num_unentered_leaves = np.array(d["unentered_leaves"]) + + ax.bar(experiments, num_enters) + ax.bar( + experiments, + num_leaves, + bottom=num_enters, + ) + ax.bar( + experiments, + num_unclosed_enters, + bottom=num_enters + num_leaves + ) + ax.bar( + experiments, + num_unentered_leaves, + bottom=num_enters + num_leaves + num_unclosed_enters + ) + + ax.set_ylabel("Number of events") + ax.set_xticks(ax.get_xticks(), ax.get_xticklabels(), rotation=45) + ax.set_title(binary) + + fig.suptitle( + f"Instrumentation Verifier " + f"Overview for {case_study.project_name}" + ) + fig.legend( + labels=["Enters", "Leaves", "Unclosed enters", "Unentered leaves"] + ) + + +class VerifierExperimentCompareBudgetLabelsOverviewGenerator( + PlotGenerator, + generator_name="iv-ceb-overview-plot", + options=[REQUIRE_MULTI_EXPERIMENT_TYPE, REQUIRE_MULTI_CASE_STUDY] +): + """Generates a single pc-overview plot for the current paper config.""" + + def generate(self) -> tp.List[Plot]: + case_studies = self.plot_kwargs.pop("case_study") + return [ + InstrumentationOverviewCompareExperimentsBudgetLabelsPlot( + self.plot_config, case_study=cs, **self.plot_kwargs + ) for cs in case_studies + ] diff --git a/varats/varats/plots/instrumentation_verifier_overview.py b/varats/varats/plots/instrumentation_verifier_overview.py index 092fc48fb..470a7d227 100644 --- a/varats/varats/plots/instrumentation_verifier_overview.py +++ b/varats/varats/plots/instrumentation_verifier_overview.py @@ -102,7 +102,7 @@ def _generate_plot(**kwargs: tp.Any) -> None: f"Overview for {case_study.project_name}" ) ax.legend() - plt.xticks(rotation=90, ha='right') + plt.xticks(ha='right') plt.subplots_adjust(bottom=0.25) diff --git a/varats/varats/plots/instrumentation_verifier_overview_budget.py b/varats/varats/plots/instrumentation_verifier_overview_budget.py index ad0feb76b..24d5ae0ad 100644 --- a/varats/varats/plots/instrumentation_verifier_overview_budget.py +++ b/varats/varats/plots/instrumentation_verifier_overview_budget.py @@ -67,9 +67,10 @@ def _generate_plot(**kwargs: tp.Any) -> None: df = pd.DataFrame(rows) binaries = df["binary"].unique() - fig, axs = plt.subplots((1 + len(binaries)) // 2, 2 - len(binaries) % 2) - fig.suptitle(f"Results of {experiment.NAME} by budget for case study {case_study.project_name}") - # fig.set_size_inches(11.7, 8.27) + fig, axs = plt.subplots((1 + len(binaries)) // 2, 2 - len(binaries) % 2, constrained_layout=True) + fig.suptitle( + f"Results of {experiment.NAME} by budget for case study {case_study.project_name}" + ) for i, binary in enumerate(binaries): if len(binaries) == 1: @@ -77,7 +78,8 @@ def _generate_plot(**kwargs: tp.Any) -> None: elif len(binaries) == 2: ax = axs[i % 2] else: - ax = axs[*divmod(i, 2)] + x, y = divmod(i, 2) + ax = axs[x, y] d = df[df["binary"] == binary].sort_values("budget") @@ -108,10 +110,15 @@ def _generate_plot(**kwargs: tp.Any) -> None: ax.set_ylabel("# Events") ax.set_xlabel("Budget") - ax.set_xticks(X, labels=d["budget"], rotation=45) + ax.set_xticks(X, labels=d["budget"]) ax.set_title(binary) - fig.legend(labels=["Closed enters", "Entered leaves", "Unclosed enters", "Unentered leaves"]) + fig.legend( + labels=[ + "Closed enters", "Entered leaves", "Unclosed enters", + "Unentered leaves" + ] + ) sns.despine() def calc_missing_revisions( diff --git a/varats/varats/projects/c_projects/picosat.py b/varats/varats/projects/c_projects/picosat.py index 5cd27d536..c42087a63 100644 --- a/varats/varats/projects/c_projects/picosat.py +++ b/varats/varats/projects/c_projects/picosat.py @@ -76,6 +76,14 @@ class PicoSAT(VProject, ReleaseProviderHook): "download/picoSAT-965/traffic_kkb_unknown.cnf.tar.gz" } ), + HTTPUntar( + local="SAT_H_instances_childsnack_p08.hddl_2.cnf", + remote={ + "1.0": + "https://github.com/se-sic/picoSAT-mirror/releases/" + "download/picoSAT-965/SAT_H_instances_childsnack_p08.hddl_2.cnf.tar.gz" + } + ), HTTPUntar( local="UNSAT_H_instances_childsnack_p11.hddl_1.cnf", remote={ @@ -95,7 +103,6 @@ class PicoSAT(VProject, ReleaseProviderHook): ), ] - # TODO: Not sure about the categories here WORKLOADS = { WorkloadSet(WorkloadCategory.EXAMPLE): [ Command( @@ -105,21 +112,11 @@ class PicoSAT(VProject, ReleaseProviderHook): ) ], WorkloadSet(WorkloadCategory.SMALL): [ - # Command( - # SourceRoot("picosat") / RSBinary("picosat"), - # "aim-100-1_6-no-1.cnf", - # label="aim-100-1-6-no-1.cnf", - # ) Command( SourceRoot("picosat") / RSBinary("picosat"), - "traffic_kkb_unknown.cnf/traffic_kkb_unknown.cnf", - label="traffic-kkb-unknow.cnf", + "SAT_H_instances_childsnack_p08.hddl_2.cnf/SAT_H_instances_childsnack_p08.hddl_2.cnf", + label="SAT-H-instances-childsnack-p08.hddl-2.cnf", ), - # Command( - # SourceRoot("picosat") / RSBinary("picosat"), - # "abw-N-bcsstk07.mtx-w44.cnf/abw-N-bcsstk07.mtx-w44.cnf", - # label="abw-N-bcsstk07.mtx-w44.cnf", - # ), ], WorkloadSet(WorkloadCategory.MEDIUM): [ Command( diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py index 9cb3e31be..d1b711420 100644 --- a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -88,10 +88,6 @@ class FeaturePerfCSCollection(VProject): str(5 * 10**3), label="SBL-iterations-10K-count-to-5K" ), - # Command( - # SourceRoot("FeaturePerfCSCollection") / RSBinary("LoopBubble"), - # label="LB-no-input" - # ), Command( SourceRoot("FeaturePerfCSCollection") / RSBinary("SingleLocalSimple"), @@ -110,17 +106,17 @@ class FeaturePerfCSCollection(VProject): label="SFI-enc-compress" ) ], - # WorkloadSet(WorkloadCategory.SMALL): [ - # Command( - # SourceRoot("FeaturePerfCSCollection") / - # RSBinary("SimpleBusyLoop"), - # "--iterations", - # str(10**7), - # "--count_to", - # str(5 * 10**3), - # label="SBL-iterations-10M-count-to-5K" - # ) - # ] + WorkloadSet(WorkloadCategory.SMALL): [ + Command( + SourceRoot("FeaturePerfCSCollection") / + RSBinary("SimpleBusyLoop"), + "--iterations", + str(10**7), + "--count_to", + str(5 * 10**3), + label="SBL-iterations-10M-count-to-5K" + ) + ] } @staticmethod diff --git a/varats/varats/tables/time_workloads.py b/varats/varats/tables/time_workloads.py index 4120c5210..a7ba4e6bf 100644 --- a/varats/varats/tables/time_workloads.py +++ b/varats/varats/tables/time_workloads.py @@ -35,7 +35,8 @@ def tabulate(self, table_format: TableFormat, wrap_table: bool) -> str: report_files = get_processed_revisions_files( project_name, self.table_kwargs["experiment_type"][0], WLTimeReportAggregate, - get_case_study_file_name_filter(case_study) + get_case_study_file_name_filter(case_study), + only_newest=False ) def wall_clock_time_in_msecs( @@ -82,7 +83,7 @@ def wall_clock_time_in_msecs( len(agg_time_report.reports(workload_name)) } - df = df.append(new_row, ignore_index=True) + df = pd.concat([df, pd.DataFrame([new_row])], ignore_index=True) df.sort_values(["Project", "Binary"], inplace=True) df.set_index( diff --git a/varats/varats/tables/time_workloads2.py b/varats/varats/tables/time_workloads2.py new file mode 100644 index 000000000..dda5764ef --- /dev/null +++ b/varats/varats/tables/time_workloads2.py @@ -0,0 +1,109 @@ +"""Module for the TimedWorkloadsTable.""" +import typing as tp + +import numpy as np +import pandas as pd + +from varats.paper.paper_config import get_loaded_paper_config +from varats.paper_mgmt.case_study import get_case_study_file_name_filter +from varats.report.gnu_time_report import WLTimeReportAggregate +from varats.revision.revisions import get_processed_revisions_files +from varats.table.table import Table +from varats.table.table_utils import dataframe_to_table +from varats.table.tables import TableFormat, TableGenerator +from varats.ts_utils.click_param_types import REQUIRE_MULTI_EXPERIMENT_TYPE, REQUIRE_CASE_STUDY + + +import re + +budgetre = re.compile("RunTracedNaive([0-9]+)") + +def budget_from_experiment_name(name): + if (m := re.search(budgetre, name)) is not None: + return int(m.group(1)) + elif name == "RunTraced": + return 0 + elif name == "RunUntraced": + return -1 + + +class TimedWorkloadTable(Table, table_name="time_workloads_2"): + """Simple table to print the run-time and memory consumption of different + workloads.""" + + def tabulate(self, table_format: TableFormat, wrap_table: bool) -> str: + df = pd.DataFrame() + + case_study = self.table_kwargs["case_study"] + project_name = case_study.project_name + + experiments = self.table_kwargs["experiment_type"] + + for experiment in experiments: + + project_name = case_study.project_name + + report_files = get_processed_revisions_files( + project_name, experiment, + WLTimeReportAggregate, + get_case_study_file_name_filter(case_study), + only_newest=False + ) + + def wall_clock_time_in_msecs( + agg_time_report: WLTimeReportAggregate + ) -> tp.List[float]: + return list( + map( + lambda x: x * 1000, + agg_time_report. + measurements_wall_clock_time(workload_name) + ) + ) + + for report_filepath in report_files: + agg_time_report = WLTimeReportAggregate( + report_filepath.full_path() + ) + + for workload_name in agg_time_report.workload_names(): + new_row = { + "Experiment": + experiment.NAME, + "Budget": budget_from_experiment_name(experiment.NAME), + "Workload": + workload_name, + "Mean wall time (msecs)": + np.mean(wall_clock_time_in_msecs(agg_time_report)), + "StdDev": + round( + np.std( + wall_clock_time_in_msecs(agg_time_report) + ), 2 + ), + "Reps": + len(agg_time_report.reports(workload_name)) + } + + df = pd.concat([df, pd.DataFrame([new_row])], ignore_index=True) + + df.sort_values(["Workload", "Budget"], inplace=True) + + kwargs: tp.Dict[str, tp.Any] = {} + if table_format.is_latex(): + kwargs["column_format"] = "llr|rr|r|r" + + return dataframe_to_table( + df, table_format, wrap_table, wrap_landscape=True, **kwargs + ) + + +class TimedWorkloadTableGenerator( + TableGenerator, + generator_name="time-workloads-2", + options=[REQUIRE_MULTI_EXPERIMENT_TYPE, REQUIRE_CASE_STUDY] +): + """Generator for `TimeWorkloadsTable`.""" + + def generate(self) -> tp.List[Table]: + return [TimedWorkloadTable(self.table_config, **self.table_kwargs)]