From c80ce016bc4c4870f3af5e59a804356f46e72d16 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Mon, 22 May 2023 22:49:13 -0400 Subject: [PATCH 01/52] Allow local installation from source tree without docker --- Makefile | 11 ++ install.py | 236 +++++++++++++++++++++++++++++++++++++ ofrak_core/Makefile | 1 + ofrak_patch_maker/Makefile | 12 +- 4 files changed, 254 insertions(+), 6 deletions(-) create mode 100755 install.py diff --git a/Makefile b/Makefile index 33b7dd8a9..3caa5aa34 100644 --- a/Makefile +++ b/Makefile @@ -18,3 +18,14 @@ tutorial-image: tutorial-run: make -C ofrak_tutorial run + +.PHONY: install_tutorial install_code install_develop +install_tutorial: + python3 install.py --config ofrak-tutorial.yml --target install + +install_core: + python3 install.py --config ofrak-core-dev.yml --target install + +install_develop: + python3 install.py --config ofrak-dev.yml --target develop + diff --git a/install.py b/install.py new file mode 100755 index 000000000..36622b727 --- /dev/null +++ b/install.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +from dataclasses import dataclass +from enum import Enum +from typing import List, Optional + +import argparse +import os +import subprocess +import sys +import pkg_resources +import yaml +import shutil + +from build_image import InstallTarget + + +class DependencyMechanism(Enum): + NONE = "none" + SHOW = "show" + APT = "apt" + BREW = "brew" + + +@dataclass +class OfrakInstallConfig: + packages_paths: List[str] + python_command: str + install_target: InstallTarget + dependency_mechanism: DependencyMechanism + dep_install: List[str] + quiet: bool + + +def main(): + config = parse_args() + + for package_path in config.packages_paths: + check_package_contents(package_path) + + if not config.quiet: + print(f"*** Checking whether npm and rollup are installed") + check_executable(config, "npm") + check_executable(config, "rollup") + + if not config.quiet: + install_type = ( + "development environment " if config.install_target == InstallTarget.DEVELOP else "" + ) + print( + f"*** Installing OFRAK {install_type} for {config.python_command} from: " + f"{', '.join(config.packages_paths)}." + ) + for package_path in config.packages_paths: + if not config.quiet: + print(f"** Installing {package_path}") + install_package(config, package_path) + + if config.dependency_mechanism == DependencyMechanism.NONE: + pass + elif config.dependency_mechanism == DependencyMechanism.SHOW: + print("*** Checking for missing OFRAK dependencies") + show_dependencies(config) + show_install(config, "apt", "sudo apt install -y") + show_install(config, "brew", "brew install") + else: + install_deps(config) + if not config.quiet: + print("*** Checking OFRAK dependencies that may need to be installed manually") + show_dependencies(config) + + +def parse_args() -> OfrakInstallConfig: + parser = argparse.ArgumentParser() + parser.add_argument( + "--config", + help="Path to a .yml configuration file specifying which OFRAK packages to install", + required=True, + ) + parser.add_argument( + "--target", + choices=[InstallTarget.DEVELOP.value, InstallTarget.INSTALL.value], + default=InstallTarget.DEVELOP.value, + help='Installation type. Use "install" for regular installation, and "develop" to install in ' + 'editable mode (i.e. setuptools "develop mode") from the source tree paths, and include ' + 'development (test, build documentation, etc) dependencies. Defaults to "develop"', + ) + parser.add_argument( + "--python", + default=os.getenv("OFRAK_INSTALL_PYTHON", "python3"), + help="Path to, or name of the python executable to install OFRAK for. Defaults to the value of" + 'the "OFRAK_INSTALL_PYTHON" environment variable if set, and "python3", if not', + ) + parser.add_argument( + "--install_deps", + choices=[ + DependencyMechanism.NONE.value, + DependencyMechanism.SHOW.value, + DependencyMechanism.APT.value, + DependencyMechanism.BREW.value, + ], + default=os.getenv("OFRAK_INSTALL_DEPS", DependencyMechanism.SHOW.value), + help='Method for installing non-pip dependencies. One of: "none" - do not install, "show" - ' + 'show how to install them, but do not install automatically, "apt" - install using APT ' + '(requires sudo), "brew" - install using brew. Defaults to the value of the ' + '"OFRAK_INSTALL_DEPS" environment variable if set, and "show", if not', + ) + parser.add_argument("--quiet", "-q", action="store_true", help="Reduce verbosity") + args = parser.parse_args() + python = shutil.which(args.python) + if python is None: + if not args.quiet: + print( + "*** Specify correct name or path of python binary to use, using either the " + '"--python" command line argument, or the "OFRAK_INSTALL_PYTHON" environment' + "variable.", + file=sys.stderr, + ) + raise ValueError(f"{args.python} not found") + with open(args.config) as file_handle: + config_dict = yaml.safe_load(file_handle) + if args.install_deps == "apt": + install_command = ["sudo", "apt", "install", "-y"] + elif args.install_deps == "brew": + install_command = ["brew", "install"] + else: + install_command = [] + install_config = OfrakInstallConfig( + packages_paths=config_dict["packages_paths"], + python_command=python, + install_target=InstallTarget(args.target), + dependency_mechanism=DependencyMechanism(args.install_deps), + quiet=args.quiet, + dep_install=install_command, + ) + return install_config + + +def check_package_contents(package_path: str): + for content in [package_path, os.path.join(package_path, "Makefile")]: + if not os.path.exists(content): + raise ValueError(f"Required path {content} do not exist") + return + + +def check_executable(config: OfrakInstallConfig, executable: str) -> None: + if shutil.which(executable) is None: + if config.dependency_mechanism in [DependencyMechanism.APT, DependencyMechanism.BREW]: + if not config.quiet: + print("** {executable} not found, attempting to install") + run_command(config, config.dep_install + [executable]) + elif config.dependency_mechanism in [DependencyMechanism.NONE, DependencyMechanism.SHOW]: + if config.dependency_mechanism == DependencyMechanism.SHOW: + print("** {executable} not found, please install manually,") + print('** or use "--install_deps" / "OFRAK_INSTALL_DEPS"') + print("** to have it be installed automatically for you:") + print("** apt: sudo apt install -y {executable}") + print("** brew: brew install {executable}") + raise FileNotFoundError(2, "{executable} not found", executable) + + +def install_package(config: OfrakInstallConfig, package_path: str) -> None: + if not config.quiet: + print(f"** Installing from {package_path}") + etc_dir = os.path.join(os.getenv("HOME", "/"), "etc") + run_command( + config, + [ + "make", + f"PYTHON={config.python_command}", + f"PIP={config.python_command} -m pip", + f"ETC={etc_dir}", + "-C", + package_path, + config.install_target.value, + ], + ) + + +def show_dependencies(config: OfrakInstallConfig) -> None: + run_ofrak_command(config, ["deps", "--missing-only"]) + + +def show_install(config: OfrakInstallConfig, dep: str, prefix: str) -> None: + deps = run_ofrak_command( + config, + ["deps", "--missing-only", f"--packages-for={dep}"], + True, + ) + if deps: + print(f"** If your system has {dep}, you can install some of the dependencies using:") + print(f"** {prefix} {deps}") + + +def install_deps(config: OfrakInstallConfig) -> None: + if not config.quiet: + print( + "*** Installing those OFRAK dependencies that can be installed with " + "{config.dependency_mechanism.value}" + ) + deps = run_ofrak_command( + config, + ["deps", "--missing-only", f"--packages-for={config.dependency_mechanism.value}"], + True, + ) + assert deps is not None + run_command(config, config.dep_install + deps.split()) + + +def run_ofrak_command( + config: OfrakInstallConfig, args: List[str], capture_out=False +) -> Optional[str]: + return run_command(config, [config.python_command, "-m", "ofrak"] + args, capture_out) + + +def run_command(config: OfrakInstallConfig, args: List[str], capture_out=False) -> Optional[str]: + if not config.quiet: + print("% " + " ".join(args)) + result = subprocess.run(args=args, capture_output=capture_out, check=True) + return result.stdout.decode("ascii") if capture_out else None + + +if __name__ == "__main__": + try: + main() + except subprocess.CalledProcessError as error: + print(f"*** Error running shell command, exit status: {error.returncode}", file=sys.stderr) + sys.exit(error.returncode) + except ValueError as error: + print(f"*** Error: {error}", file=sys.stderr) + sys.exit(1) + except FileNotFoundError as error: + print(f"*** Error: No such file or directory: {error.filename}", file=sys.stderr) + sys.exit(1) + except Exception as error: + print(f"*** Unexpected exception: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ofrak_core/Makefile b/ofrak_core/Makefile index 8c1b8996c..ea2add664 100644 --- a/ofrak_core/Makefile +++ b/ofrak_core/Makefile @@ -23,6 +23,7 @@ ofrak/gui/public: cp -r /ofrak_gui ofrak/gui/public ; \ elif [ -d ../frontend ]; then \ cd ../frontend && \ + npm install && \ npm run build && \ cd ../ofrak_core && \ cp -r ../frontend/public ofrak/gui/public ; \ diff --git a/ofrak_patch_maker/Makefile b/ofrak_patch_maker/Makefile index 3b85f9017..93555bc31 100644 --- a/ofrak_patch_maker/Makefile +++ b/ofrak_patch_maker/Makefile @@ -1,20 +1,20 @@ PYTHON=python3 PIP=pip3 +ETC=/etc # toolchain.conf is a file mapping ID to the various binaries responsible for preprocessing, # assembling, compiling, linking, analyzing binaries for each currently supported toolchain. -.PHONY: toolchain_conf -toolchain_conf: - cp ofrak_patch_maker/toolchain.conf /etc/toolchain.conf - mv ofrak_patch_maker/toolchain.conf ofrak_patch_maker/toolchain.conf.bak +$(ETC)/toolchain.conf: ofrak_patch_maker/toolchain.conf + mkdir -p $(ETC) + cp -a $< $@ .PHONY: install -install: toolchain_conf +install: $(ETC)/toolchain.conf $(PIP) install . .PHONY: develop -develop: toolchain_conf +develop: $(ETC)/toolchain.conf $(PIP) install -e .[test] .PHONY: inspect From 8e2fd5492c8ad41f0977180d3a5a8474344b7189 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Mon, 22 May 2023 23:10:07 -0400 Subject: [PATCH 02/52] Fix unused import --- install.py | 1 - 1 file changed, 1 deletion(-) diff --git a/install.py b/install.py index 36622b727..4dc6ddd25 100755 --- a/install.py +++ b/install.py @@ -7,7 +7,6 @@ import os import subprocess import sys -import pkg_resources import yaml import shutil From 2bbd207442074768b3ae06886bd6b5c5bf28d645 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Mon, 22 May 2023 23:20:40 -0400 Subject: [PATCH 03/52] Handle missing binja dependency --- .../components/binary_ninja_analyzer.py | 28 +++++++++++++++++-- .../components/blocks/unpackers.py | 14 +++++++++- .../components/data/ref_analyzer.py | 8 ++++-- .../ofrak_binary_ninja/model.py | 5 +++- 4 files changed, 49 insertions(+), 6 deletions(-) diff --git a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/binary_ninja_analyzer.py b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/binary_ninja_analyzer.py index 0ca5a0234..a0c97d525 100644 --- a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/binary_ninja_analyzer.py +++ b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/binary_ninja_analyzer.py @@ -2,11 +2,17 @@ import tempfile from dataclasses import dataclass from typing import Optional, List +from ofrak.component.abstract import ComponentMissingDependencyError -from binaryninja import open_view, BinaryViewType +try: + from binaryninja import open_view, BinaryViewType + + BINJA_INSTALLED = True +except ImportError: + BINJA_INSTALLED = False from ofrak.component.analyzer import Analyzer -from ofrak.model.component_model import ComponentConfig +from ofrak.model.component_model import ComponentConfig, ComponentExternalTool from ofrak.model.resource_model import ResourceAttributeDependency from ofrak_binary_ninja.components.identifiers import BinaryNinjaAnalysisResource from ofrak_binary_ninja.model import BinaryNinjaAnalysis @@ -15,6 +21,21 @@ LOGGER = logging.getLogger(__file__) +class _BinjaExternalTool(ComponentExternalTool): + def __init__(self): + super().__init__( + "binary_ninja", + "https://ofrak.com/docs/user-guide/disassembler-backends/binary_ninja.html", + install_check_arg="", + ) + + async def is_tool_installed(self) -> bool: + return BINJA_INSTALLED + + +BINJA_TOOL = _BinjaExternalTool() + + @dataclass class BinaryNinjaAnalyzerConfig(ComponentConfig): bndb_file: str # Path to BinaryNinja DB pre-analyzed file @@ -24,10 +45,13 @@ class BinaryNinjaAnalyzer(Analyzer[Optional[BinaryNinjaAnalyzerConfig], BinaryNi id = b"BinaryNinjaAnalyzer" targets = (BinaryNinjaAnalysisResource,) outputs = (BinaryNinjaAnalysis,) + external_dependencies = (BINJA_TOOL,) async def analyze( self, resource: Resource, config: Optional[BinaryNinjaAnalyzerConfig] = None ) -> BinaryNinjaAnalysis: + if not BINJA_INSTALLED: + raise ComponentMissingDependencyError(self, BINJA_TOOL) if not config: resource_data = await resource.get_data() temp_file = tempfile.NamedTemporaryFile() diff --git a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/blocks/unpackers.py b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/blocks/unpackers.py index d10725e64..a1a63e5f0 100644 --- a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/blocks/unpackers.py +++ b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/blocks/unpackers.py @@ -3,8 +3,8 @@ from typing import Iterable, Tuple, List from typing import Optional from warnings import warn +from ofrak.component.abstract import ComponentMissingDependencyError -from binaryninja import BinaryView, Endianness, TypeClass from ofrak_type.architecture import InstructionSetMode from ofrak_type.range import Range @@ -17,9 +17,15 @@ from ofrak.model.component_model import ComponentConfig from ofrak.resource import Resource from ofrak.service.resource_service_i import ResourceFilter +from ofrak_binary_ninja.components.binary_ninja_analyzer import BINJA_INSTALLED, BINJA_TOOL from ofrak_binary_ninja.components.identifiers import BinaryNinjaAnalysisResource from ofrak_binary_ninja.model import BinaryNinjaAnalysis +if BINJA_INSTALLED: + from binaryninja import BinaryView, Endianness, TypeClass +else: + BinaryView=None + LOGGER = logging.getLogger(__name__) @@ -27,7 +33,11 @@ class BinaryNinjaCodeRegionUnpacker(CodeRegionUnpacker): + external_dependencies = (BINJA_TOOL,) + async def unpack(self, resource: Resource, config=None): + if not BINJA_INSTALLED: + raise ComponentMissingDependencyError(self, BINJA_TOOL) region_view = await resource.view_as(CodeRegion) program_r = await region_view.resource.get_only_ancestor_as_view( Program, ResourceFilter.with_tags(Program) @@ -148,6 +158,8 @@ def _binary_ninja_get_complex_blocks( class BinaryNinjaComplexBlockUnpacker(ComplexBlockUnpacker): + external_dependencies = (BINJA_TOOL,) + async def unpack(self, resource: Resource, config: Optional[ComponentConfig] = None): cb_view = await resource.view_as(ComplexBlock) program_r = await cb_view.resource.get_only_ancestor_as_view( diff --git a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/data/ref_analyzer.py b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/data/ref_analyzer.py index 9cc2e7f07..9bc9e201b 100644 --- a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/data/ref_analyzer.py +++ b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/data/ref_analyzer.py @@ -2,11 +2,14 @@ from collections import defaultdict from typing import DefaultDict, Iterable, List, Tuple -from binaryninja import BinaryView +try: + from binaryninja.binaryview import BinaryView +except ImportError: + BinaryView = None from ofrak.core.data import ReferencedDataAttributes from ofrak.core.program import ReferencedDataAnalyzer -from ofrak_binary_ninja.components.binary_ninja_analyzer import BinaryNinjaAnalyzer +from ofrak_binary_ninja.components.binary_ninja_analyzer import BINJA_TOOL, BinaryNinjaAnalyzer from ofrak_binary_ninja.model import BinaryNinjaAnalysis from ofrak.resource import Resource @@ -18,6 +21,7 @@ class BinaryNinjaReferencedDataAnalyzer(ReferencedDataAnalyzer): """ Analyzer to get all data references in the program """ + external_dependencies = (BINJA_TOOL,) async def analyze(self, resource: Resource, config=None) -> Tuple[ReferencedDataAttributes]: if not resource.has_attributes(BinaryNinjaAnalysis): diff --git a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/model.py b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/model.py index d58fe1da9..855b9f607 100644 --- a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/model.py +++ b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/model.py @@ -1,6 +1,9 @@ from dataclasses import dataclass -from binaryninja.binaryview import BinaryView +try: + from binaryninja.binaryview import BinaryView +except ImportError: + BinaryView = None from ofrak.model.resource_model import ResourceAttributes From f8d6fa5c3bd66c26c21651d0d97f240598612791 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Mon, 22 May 2023 20:34:28 -0700 Subject: [PATCH 04/52] black formatting --- .../ofrak_binary_ninja/components/blocks/unpackers.py | 2 +- .../ofrak_binary_ninja/components/data/ref_analyzer.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/blocks/unpackers.py b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/blocks/unpackers.py index a1a63e5f0..01165ecf7 100644 --- a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/blocks/unpackers.py +++ b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/blocks/unpackers.py @@ -24,7 +24,7 @@ if BINJA_INSTALLED: from binaryninja import BinaryView, Endianness, TypeClass else: - BinaryView=None + BinaryView = None LOGGER = logging.getLogger(__name__) diff --git a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/data/ref_analyzer.py b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/data/ref_analyzer.py index 9bc9e201b..a8a399215 100644 --- a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/data/ref_analyzer.py +++ b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/data/ref_analyzer.py @@ -21,6 +21,7 @@ class BinaryNinjaReferencedDataAnalyzer(ReferencedDataAnalyzer): """ Analyzer to get all data references in the program """ + external_dependencies = (BINJA_TOOL,) async def analyze(self, resource: Resource, config=None) -> Tuple[ReferencedDataAttributes]: From f0377117539b5a2a1bda36b08c29a1e2fb4e5690 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Mon, 22 May 2023 20:35:32 -0700 Subject: [PATCH 05/52] Use OFRAK_INSTALL_PYTHON to run install.py --- Makefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 3caa5aa34..ae7fb184c 100644 --- a/Makefile +++ b/Makefile @@ -19,13 +19,14 @@ tutorial-image: tutorial-run: make -C ofrak_tutorial run +OFRAK_INSTALL_PYTHON=python3 + .PHONY: install_tutorial install_code install_develop install_tutorial: - python3 install.py --config ofrak-tutorial.yml --target install + $(OFRAK_INSTALL_PYTHON) install.py --config ofrak-tutorial.yml --target install install_core: - python3 install.py --config ofrak-core-dev.yml --target install + $(OFRAK_INSTALL_PYTHON) install.py --config ofrak-core-dev.yml --target install install_develop: - python3 install.py --config ofrak-dev.yml --target develop - + $(OFRAK_INSTALL_PYTHON) install.py --config ofrak-dev.yml --target develop From c71df7f48e3eddd211147621735766b544b40f17 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Mon, 22 May 2023 20:36:01 -0700 Subject: [PATCH 06/52] Ignore more files in .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 8e7dfdfcd..26381c787 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ frontend/public/build ofrak_core/ofrak/core/entropy/entropy_c.cpython* ofrak_core/ofrak/gui/public ofrak_core/build +ofrak_io/build/ +ofrak_patch_maker/build/ +ofrak_type/build/ +disassemblers/ofrak_ghidra/ofrak_ghidra/config/ofrak_ghidra.conf.yml From b48474c48c37f4bf246558a6e9edb123f6745466 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Mon, 22 May 2023 20:44:25 -0700 Subject: [PATCH 07/52] Pre-install pyyaml --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index ae7fb184c..87848f4b8 100644 --- a/Makefile +++ b/Makefile @@ -23,10 +23,13 @@ OFRAK_INSTALL_PYTHON=python3 .PHONY: install_tutorial install_code install_develop install_tutorial: + $(OFRAK_INSTALL_PYTHON) -m pip install pyyaml $(OFRAK_INSTALL_PYTHON) install.py --config ofrak-tutorial.yml --target install install_core: + $(OFRAK_INSTALL_PYTHON) -m pip install pyyaml $(OFRAK_INSTALL_PYTHON) install.py --config ofrak-core-dev.yml --target install install_develop: + $(OFRAK_INSTALL_PYTHON) -m pip install pyyaml $(OFRAK_INSTALL_PYTHON) install.py --config ofrak-dev.yml --target develop From 4b1c79a15a60acb4204bcf6c22f1f6d46230be4f Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Tue, 23 May 2023 14:40:00 -0400 Subject: [PATCH 08/52] Fix some of the typos Jacob found --- Makefile | 2 +- install.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index ae7fb184c..3e7596af2 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ tutorial-run: OFRAK_INSTALL_PYTHON=python3 -.PHONY: install_tutorial install_code install_develop +.PHONY: install_tutorial install_core install_develop install_tutorial: $(OFRAK_INSTALL_PYTHON) install.py --config ofrak-tutorial.yml --target install diff --git a/install.py b/install.py index 4dc6ddd25..c0dabf535 100755 --- a/install.py +++ b/install.py @@ -145,16 +145,16 @@ def check_executable(config: OfrakInstallConfig, executable: str) -> None: if shutil.which(executable) is None: if config.dependency_mechanism in [DependencyMechanism.APT, DependencyMechanism.BREW]: if not config.quiet: - print("** {executable} not found, attempting to install") + print(f"** {executable} not found, attempting to install") run_command(config, config.dep_install + [executable]) elif config.dependency_mechanism in [DependencyMechanism.NONE, DependencyMechanism.SHOW]: if config.dependency_mechanism == DependencyMechanism.SHOW: - print("** {executable} not found, please install manually,") + print(f"** {executable} not found, please install manually,") print('** or use "--install_deps" / "OFRAK_INSTALL_DEPS"') print("** to have it be installed automatically for you:") - print("** apt: sudo apt install -y {executable}") - print("** brew: brew install {executable}") - raise FileNotFoundError(2, "{executable} not found", executable) + print(f"** apt: sudo apt install -y {executable}") + print(f"** brew: brew install {executable}") + raise FileNotFoundError(2, f"{executable} not found", executable) def install_package(config: OfrakInstallConfig, package_path: str) -> None: @@ -194,7 +194,7 @@ def install_deps(config: OfrakInstallConfig) -> None: if not config.quiet: print( "*** Installing those OFRAK dependencies that can be installed with " - "{config.dependency_mechanism.value}" + + config.dependency_mechanism.value ) deps = run_ofrak_command( config, From bf465a84b6a619894658b6d44f646d3c27c0d87b Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Tue, 23 May 2023 11:49:05 -0700 Subject: [PATCH 09/52] Fix a typo noticed by Jacob Co-authored-by: Jacob Strieb <99368685+rbs-jacob@users.noreply.github.com> --- install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.py b/install.py index c0dabf535..89008f809 100755 --- a/install.py +++ b/install.py @@ -46,7 +46,7 @@ def main(): "development environment " if config.install_target == InstallTarget.DEVELOP else "" ) print( - f"*** Installing OFRAK {install_type} for {config.python_command} from: " + f"*** Installing OFRAK {install_type}for {config.python_command} from: " f"{', '.join(config.packages_paths)}." ) for package_path in config.packages_paths: From ce9cc7ef7bc04b3bd088c068a27708fb05d511d8 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Tue, 23 May 2023 15:09:31 -0700 Subject: [PATCH 10/52] Use logging instead of raw priting, where appropriate --- install.py | 93 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 52 insertions(+), 41 deletions(-) diff --git a/install.py b/install.py index 89008f809..b86a80659 100755 --- a/install.py +++ b/install.py @@ -9,9 +9,13 @@ import sys import yaml import shutil +import logging from build_image import InstallTarget +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.INFO) + class DependencyMechanism(Enum): NONE = "none" @@ -31,40 +35,42 @@ class OfrakInstallConfig: def main(): + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(logging.INFO) + formatter = logging.Formatter('*** %(msg)s') + handler.setFormatter(formatter) + LOGGER.addHandler(handler) + config = parse_args() for package_path in config.packages_paths: check_package_contents(package_path) - if not config.quiet: - print(f"*** Checking whether npm and rollup are installed") + LOGGER.info(f"Checking whether npm and rollup are installed") check_executable(config, "npm") check_executable(config, "rollup") - if not config.quiet: - install_type = ( - "development environment " if config.install_target == InstallTarget.DEVELOP else "" - ) - print( - f"*** Installing OFRAK {install_type}for {config.python_command} from: " - f"{', '.join(config.packages_paths)}." - ) + install_type = ( + "development environment " if config.install_target == InstallTarget.DEVELOP else "" + ) + LOGGER.info( + f"Installing OFRAK {install_type}for {config.python_command} from: " + f"{', '.join(config.packages_paths)}." + ) for package_path in config.packages_paths: - if not config.quiet: - print(f"** Installing {package_path}") install_package(config, package_path) if config.dependency_mechanism == DependencyMechanism.NONE: pass elif config.dependency_mechanism == DependencyMechanism.SHOW: - print("*** Checking for missing OFRAK dependencies") + LOGGER.info("Checking for missing OFRAK dependencies") show_dependencies(config) show_install(config, "apt", "sudo apt install -y") show_install(config, "brew", "brew install") else: install_deps(config) + LOGGER.info("Checking OFRAK dependencies that may need to be installed manually") if not config.quiet: - print("*** Checking OFRAK dependencies that may need to be installed manually") show_dependencies(config) @@ -105,15 +111,14 @@ def parse_args() -> OfrakInstallConfig: ) parser.add_argument("--quiet", "-q", action="store_true", help="Reduce verbosity") args = parser.parse_args() + if args.quiet: + LOGGER.setLevel(logging.ERROR) python = shutil.which(args.python) if python is None: - if not args.quiet: - print( - "*** Specify correct name or path of python binary to use, using either the " - '"--python" command line argument, or the "OFRAK_INSTALL_PYTHON" environment' - "variable.", - file=sys.stderr, - ) + LOGGER.critical( + "Specify correct name or path of python binary to use, using either the " + '"--python" command line argument, or the "OFRAK_INSTALL_PYTHON" environment variable.', + ) raise ValueError(f"{args.python} not found") with open(args.config) as file_handle: config_dict = yaml.safe_load(file_handle) @@ -144,22 +149,21 @@ def check_package_contents(package_path: str): def check_executable(config: OfrakInstallConfig, executable: str) -> None: if shutil.which(executable) is None: if config.dependency_mechanism in [DependencyMechanism.APT, DependencyMechanism.BREW]: - if not config.quiet: - print(f"** {executable} not found, attempting to install") + LOGGER.warning(f"{executable} not found, attempting to install") run_command(config, config.dep_install + [executable]) elif config.dependency_mechanism in [DependencyMechanism.NONE, DependencyMechanism.SHOW]: if config.dependency_mechanism == DependencyMechanism.SHOW: - print(f"** {executable} not found, please install manually,") - print('** or use "--install_deps" / "OFRAK_INSTALL_DEPS"') - print("** to have it be installed automatically for you:") - print(f"** apt: sudo apt install -y {executable}") - print(f"** brew: brew install {executable}") + LOGGER.critical( + f"{executable} not found, please install manually, or use" + ' "--install_deps" / "OFRAK_INSTALL_DEPS" to have it be installed automatically' + f' for you: with apt: "sudo apt install -y {executable}";' + f' with brew: "brew install {executable}"' + ) raise FileNotFoundError(2, f"{executable} not found", executable) def install_package(config: OfrakInstallConfig, package_path: str) -> None: - if not config.quiet: - print(f"** Installing from {package_path}") + LOGGER.info(f"Installing from {package_path}") etc_dir = os.path.join(os.getenv("HOME", "/"), "etc") run_command( config, @@ -176,7 +180,13 @@ def install_package(config: OfrakInstallConfig, package_path: str) -> None: def show_dependencies(config: OfrakInstallConfig) -> None: - run_ofrak_command(config, ["deps", "--missing-only"]) + missing_deps = run_ofrak_command(config, ["deps", "--missing-only"], capture_out=True) + if missing_deps: + print("\n*** Some optional OFRAK dependencies are missing. ***") + print( + "To get full mileage out of OFRAK, you may want to also install some of the following:" + ) + print(missing_deps.rstrip()) def show_install(config: OfrakInstallConfig, dep: str, prefix: str) -> None: @@ -186,14 +196,16 @@ def show_install(config: OfrakInstallConfig, dep: str, prefix: str) -> None: True, ) if deps: - print(f"** If your system has {dep}, you can install some of the dependencies using:") + deps = deps.strip().replace('\n', ' ') + print( + f"** If your system has {dep}, you can install some of the above missing dependencies using:" + ) print(f"** {prefix} {deps}") def install_deps(config: OfrakInstallConfig) -> None: - if not config.quiet: - print( - "*** Installing those OFRAK dependencies that can be installed with " + LOGGER.info( + "Installing those OFRAK dependencies that can be installed with " + config.dependency_mechanism.value ) deps = run_ofrak_command( @@ -212,8 +224,7 @@ def run_ofrak_command( def run_command(config: OfrakInstallConfig, args: List[str], capture_out=False) -> Optional[str]: - if not config.quiet: - print("% " + " ".join(args)) + (LOGGER.debug if capture_out else LOGGER.info) ("% " + " ".join(args)) result = subprocess.run(args=args, capture_output=capture_out, check=True) return result.stdout.decode("ascii") if capture_out else None @@ -222,14 +233,14 @@ def run_command(config: OfrakInstallConfig, args: List[str], capture_out=False) try: main() except subprocess.CalledProcessError as error: - print(f"*** Error running shell command, exit status: {error.returncode}", file=sys.stderr) + LOGGER.critical(f"Error running shell command, exit status: {error.returncode}") sys.exit(error.returncode) except ValueError as error: - print(f"*** Error: {error}", file=sys.stderr) + LOGGER.critical(f"Error: {error}") sys.exit(1) except FileNotFoundError as error: - print(f"*** Error: No such file or directory: {error.filename}", file=sys.stderr) + LOGGER.critical(f"Error: No such file or directory: {error.filename}") sys.exit(1) except Exception as error: - print(f"*** Unexpected exception: {error}", file=sys.stderr) + LOGGER.critical(f"Unexpected exception: {error}") sys.exit(1) From 93afc76b2f3cb2a55cc8ad35048ece32304a2db3 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Tue, 23 May 2023 15:33:08 -0700 Subject: [PATCH 11/52] Correct the formatting --- install.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/install.py b/install.py index b86a80659..fd723907e 100755 --- a/install.py +++ b/install.py @@ -37,7 +37,7 @@ class OfrakInstallConfig: def main(): handler = logging.StreamHandler(sys.stdout) handler.setLevel(logging.INFO) - formatter = logging.Formatter('*** %(msg)s') + formatter = logging.Formatter("*** %(msg)s") handler.setFormatter(formatter) LOGGER.addHandler(handler) @@ -196,7 +196,7 @@ def show_install(config: OfrakInstallConfig, dep: str, prefix: str) -> None: True, ) if deps: - deps = deps.strip().replace('\n', ' ') + deps = deps.strip().replace("\n", " ") print( f"** If your system has {dep}, you can install some of the above missing dependencies using:" ) @@ -205,9 +205,9 @@ def show_install(config: OfrakInstallConfig, dep: str, prefix: str) -> None: def install_deps(config: OfrakInstallConfig) -> None: LOGGER.info( - "Installing those OFRAK dependencies that can be installed with " - + config.dependency_mechanism.value - ) + "Installing those OFRAK dependencies that can be installed with " + + config.dependency_mechanism.value + ) deps = run_ofrak_command( config, ["deps", "--missing-only", f"--packages-for={config.dependency_mechanism.value}"], @@ -224,7 +224,7 @@ def run_ofrak_command( def run_command(config: OfrakInstallConfig, args: List[str], capture_out=False) -> Optional[str]: - (LOGGER.debug if capture_out else LOGGER.info) ("% " + " ".join(args)) + (LOGGER.debug if capture_out else LOGGER.info)("% " + " ".join(args)) result = subprocess.run(args=args, capture_output=capture_out, check=True) return result.stdout.decode("ascii") if capture_out else None From 1ec4b3f6d65e7836137dcf918836f7403001d5df Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Wed, 26 Jul 2023 19:38:19 -0700 Subject: [PATCH 12/52] Use "npm install" to install rollup --- frontend/Makefile | 9 +++++++++ install.py | 5 +++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/frontend/Makefile b/frontend/Makefile index 2df14a12d..920b2a731 100644 --- a/frontend/Makefile +++ b/frontend/Makefile @@ -96,3 +96,12 @@ develop: .PHONY: test test: + +################################################################################ +# Run for local install from source +################################################################################ + +.PHONY: npm_install_build +npm_install_build: + npm install + npm run build diff --git a/install.py b/install.py index fd723907e..e805c7892 100755 --- a/install.py +++ b/install.py @@ -46,9 +46,8 @@ def main(): for package_path in config.packages_paths: check_package_contents(package_path) - LOGGER.info(f"Checking whether npm and rollup are installed") + LOGGER.info(f"Checking whether npm is installed") check_executable(config, "npm") - check_executable(config, "rollup") install_type = ( "development environment " if config.install_target == InstallTarget.DEVELOP else "" @@ -165,6 +164,8 @@ def check_executable(config: OfrakInstallConfig, executable: str) -> None: def install_package(config: OfrakInstallConfig, package_path: str) -> None: LOGGER.info(f"Installing from {package_path}") etc_dir = os.path.join(os.getenv("HOME", "/"), "etc") + if os.path.exists(os.path.join(package_path, "package.json")): + run_command(config, ["make", "-C", package_path, "npm_install_build"]) run_command( config, [ From 3d50cf32246533070f90ede7cab156b70a527310 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Wed, 26 Jul 2023 20:03:35 -0700 Subject: [PATCH 13/52] Make the black version requirement consistent --- ofrak_io/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_io/setup.py b/ofrak_io/setup.py index 9534ae35c..fa6e4f5cb 100644 --- a/ofrak_io/setup.py +++ b/ofrak_io/setup.py @@ -32,7 +32,7 @@ def run(self): ], extras_require={ "test": [ - "black==22.6.0", + "black==23.3.0", "fun-coverage==0.2.0", "hypothesis~=6.39.3", "mypy==0.942", From 17ed5e887839564ba0718f7b857e84d5fc4d0681 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Wed, 26 Jul 2023 20:07:52 -0700 Subject: [PATCH 14/52] Add an option to run tests after install --- install.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/install.py b/install.py index e805c7892..9e4b41aaf 100755 --- a/install.py +++ b/install.py @@ -32,6 +32,7 @@ class OfrakInstallConfig: dependency_mechanism: DependencyMechanism dep_install: List[str] quiet: bool + run_tests: bool def main(): @@ -109,6 +110,11 @@ def parse_args() -> OfrakInstallConfig: '"OFRAK_INSTALL_DEPS" environment variable if set, and "show", if not', ) parser.add_argument("--quiet", "-q", action="store_true", help="Reduce verbosity") + parser.add_argument( + "--test", + action="store_true", + help="Run OFRAK tests after install. Can also be enabled by setting the OFRAK_TEST_AFTER_INSTALL environment valiable to a non-empty value", + ) args = parser.parse_args() if args.quiet: LOGGER.setLevel(logging.ERROR) @@ -134,6 +140,7 @@ def parse_args() -> OfrakInstallConfig: dependency_mechanism=DependencyMechanism(args.install_deps), quiet=args.quiet, dep_install=install_command, + run_tests=args.test or bool(os.getenv("OFRAK_TEST_AFTER_INSTALL", "")), ) return install_config @@ -163,7 +170,6 @@ def check_executable(config: OfrakInstallConfig, executable: str) -> None: def install_package(config: OfrakInstallConfig, package_path: str) -> None: LOGGER.info(f"Installing from {package_path}") - etc_dir = os.path.join(os.getenv("HOME", "/"), "etc") if os.path.exists(os.path.join(package_path, "package.json")): run_command(config, ["make", "-C", package_path, "npm_install_build"]) run_command( @@ -172,12 +178,22 @@ def install_package(config: OfrakInstallConfig, package_path: str) -> None: "make", f"PYTHON={config.python_command}", f"PIP={config.python_command} -m pip", - f"ETC={etc_dir}", "-C", package_path, config.install_target.value, ], ) + if config.run_tests: + run_command( + config, + [ + "make", + f"PYTHON={config.python_command}", + "-C", + package_path, + "test", + ], + ) def show_dependencies(config: OfrakInstallConfig) -> None: From 0d7ff95fd7efb7490ae69d49ac682f51b488c30c Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Wed, 26 Jul 2023 20:19:12 -0700 Subject: [PATCH 15/52] Updated the instructions for installing from source --- docs/environment-setup.md | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/docs/environment-setup.md b/docs/environment-setup.md index 7a00c452f..2f0cb89a5 100644 --- a/docs/environment-setup.md +++ b/docs/environment-setup.md @@ -7,7 +7,7 @@ There are three main ways one can set up an environment to use OFRAK: 1. From [PyPI](https://pypi.org/project/ofrak/) via `pip`. **This is the simplest setup and generally recommended. Use this!** -2. From the [source code](https://github.com/redballoonsecurity/ofrak) via `pip` or the `setup.py`. +2. From the [source code](https://github.com/redballoonsecurity/ofrak) via a custom install script that relies on `pip`. This is a little more complicated, but allows one to keep with and contribute to OFRAK development. 3. By building the appropriate OFRAK [Docker](https://www.docker.com/get-started) image. This has the most overhead as it requires installing Docker, but provides the most consistent and comprehensive environment. @@ -49,23 +49,14 @@ Install Git LFS by following [the instructions here](https://git-lfs.github.com/ You can install Git LFS before or after you clone OFRAK, but if you clone the OFRAK repo first, you will need to `cd` into the repository and run `git lfs install && git lfs pull`. -Once cloned, go into each directory in the top level and run the installation command `make develop` -(if you do not have and do not wish to have `make` installed, try inspecting the `Makefile` in each directory to see what commands it tries to run, usually something like `pip install -e .`). -The best order to install each directory is as follows: - -1. `ofrak_type` -2. `ofrak_io` -3. `ofrak_patch_maker` -4. `ofrak_core` -5. Any/all others: `frontend`, `ofrak_tutorial`, `disassemblers/ofrak_angr`, `disassemblers/ofrak_binary_ninja`, `disassemblers/ofrak_capstone`, `disassemblers/ofrak_ghidra` - -You *can* skip the installation step for any of the packages above. -Any subsequent OFRAK packages which require a non-installed package should be able to simply install it from PyPI. -However, this will result in a somewhat confusing environment where some of the OFRAK code in your local repo is actively used by your system, and the rest is not. +Once cloned, make sure you have Python and NPM installed, then run `make install_develop` to create a development install if OFRAK linked to your cloned source tree. +You can also use `make install_tutorial` or `make install_core` to create a standalone install of just the portions of OFRAK needed for the tutorial, or just the core OFRAK respectively. +There are other installation options - run `./install.py --help` for more information. Installing OFRAK from source code will not install all of OFRAK's non-Python dependencies (for same reason as when installing OFRAK from PyPI - not all of its dependencies are pip-installable). These dependencies are, however, optional, and the OFRAK code that requires them can be disabled in order avoid runtime errors. -OFRAK has a system for inspecting and installing dependencies. See [the section on external dependencies](#handling-non-python-dependencies) for more info on that. +OFRAK has a system for inspecting and installing dependencies. The above `make install_XXX` commands will provide you with a listing of the optional dependencies you are missing. +See [the section on external dependencies](#handling-non-python-dependencies) for more info on that. ### Modifying OFRAK Source Code From 9c9c8567a63cdb437c7dd3a4d381d2b21da93dac Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Thu, 27 Jul 2023 14:06:02 -0400 Subject: [PATCH 16/52] Add more detailed instructions to documentation --- docs/environment-setup.md | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/docs/environment-setup.md b/docs/environment-setup.md index 2f0cb89a5..ae8973512 100644 --- a/docs/environment-setup.md +++ b/docs/environment-setup.md @@ -35,6 +35,22 @@ OFRAK has a system for inspecting and installing such dependencies. See [the sec ## From Source Code +### Prerequisites + +#### Git LFS +**OFRAK uses Git LFS. +This means that you must have Git LFS installed to completely clone the repository!** +Install Git LFS by following [the instructions here](https://git-lfs.github.com/). +You can install Git LFS before or after you clone OFRAK, but if you clone the OFRAK repo first, you will need to `cd` into the repository and run `git lfs install && git lfs pull`. + +#### Python +OFRAK Requires Python 3.7 or higher. Make sure to install the development files as well (e.g. on Ubuntu, run `apt install pythin3.X-dev` for some X>=7). + +#### NPM +OFRAK GUI requires NPM. + +### Pulling source Code + The OFRAK source code can be pulled from [the github page](https://github.com/redballoonsecurity/ofrak): ```shell @@ -43,14 +59,10 @@ cd ofrak ``` -**OFRAK uses Git LFS. -This means that you must have Git LFS installed to completely clone the repository!** -Install Git LFS by following [the instructions here](https://git-lfs.github.com/). -You can install Git LFS before or after you clone OFRAK, but if you clone the OFRAK repo first, you will need to `cd` into the repository and run `git lfs install && git lfs pull`. - +### Installing -Once cloned, make sure you have Python and NPM installed, then run `make install_develop` to create a development install if OFRAK linked to your cloned source tree. -You can also use `make install_tutorial` or `make install_core` to create a standalone install of just the portions of OFRAK needed for the tutorial, or just the core OFRAK respectively. +Once cloned, run `make install_develop` to create a development install if OFRAK linked to your cloned source tree. +You can also use `make install_tutorial` or `make install_core` to create a standalone install of just the portions of OFRAK needed for the tutorial, or just the core OFRAK respectively. If you have multiple instances of Python 3.X installed on your machine, then prior to running make, you can set the `OFRAK_INSTALL_PYTHON` environment variable to point to a particular version and/or path of python executable (or run `make OFRAK_INSTALL_PYTHON=python3.X install_...`) to have OFRAK installed for a partular Python instance. There are other installation options - run `./install.py --help` for more information. Installing OFRAK from source code will not install all of OFRAK's non-Python dependencies (for same reason as when installing OFRAK from PyPI - not all of its dependencies are pip-installable). From 597db82915114711a33f18cb77c105a7bbce699f Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Fri, 9 Feb 2024 16:23:59 -0800 Subject: [PATCH 17/52] Use $(PYTHON) to run mypy --- disassemblers/ofrak_angr/Makefile | 2 +- disassemblers/ofrak_binary_ninja/Makefile | 2 +- disassemblers/ofrak_capstone/Makefile | 2 +- ofrak_core/Makefile | 2 +- ofrak_io/Makefile | 2 +- ofrak_patch_maker/Makefile | 2 +- ofrak_tutorial/Makefile | 2 +- ofrak_type/Makefile | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/disassemblers/ofrak_angr/Makefile b/disassemblers/ofrak_angr/Makefile index f7da7aa5e..8de136b1c 100644 --- a/disassemblers/ofrak_angr/Makefile +++ b/disassemblers/ofrak_angr/Makefile @@ -11,7 +11,7 @@ develop: .PHONY: inspect inspect: - mypy + $(PYTHON) -m mypy .PHONY: test test: inspect diff --git a/disassemblers/ofrak_binary_ninja/Makefile b/disassemblers/ofrak_binary_ninja/Makefile index dd2052fa1..d79132149 100644 --- a/disassemblers/ofrak_binary_ninja/Makefile +++ b/disassemblers/ofrak_binary_ninja/Makefile @@ -11,7 +11,7 @@ develop: .PHONY: inspect inspect: - mypy + $(PYTHON) -m mypy .PHONY: test test: inspect diff --git a/disassemblers/ofrak_capstone/Makefile b/disassemblers/ofrak_capstone/Makefile index 71746f688..092bf9a46 100644 --- a/disassemblers/ofrak_capstone/Makefile +++ b/disassemblers/ofrak_capstone/Makefile @@ -11,7 +11,7 @@ develop: .PHONY: inspect inspect: - mypy + $(PYTHON) -m mypy .PHONY: test test: inspect diff --git a/ofrak_core/Makefile b/ofrak_core/Makefile index c1aee9ec4..e2d8f5ad3 100644 --- a/ofrak_core/Makefile +++ b/ofrak_core/Makefile @@ -11,7 +11,7 @@ develop: ofrak/gui/public .PHONY: inspect inspect: - mypy + $(PYTHON) -m mypy .PHONY: test test: inspect diff --git a/ofrak_io/Makefile b/ofrak_io/Makefile index 0f5724824..4ae28cc73 100644 --- a/ofrak_io/Makefile +++ b/ofrak_io/Makefile @@ -11,7 +11,7 @@ develop: .PHONY: inspect inspect: - mypy + $(PYTHON) -m mypy .PHONY: test test: inspect diff --git a/ofrak_patch_maker/Makefile b/ofrak_patch_maker/Makefile index 8c311007f..12799dd04 100644 --- a/ofrak_patch_maker/Makefile +++ b/ofrak_patch_maker/Makefile @@ -12,7 +12,7 @@ develop: .PHONY: inspect inspect: - mypy + $(PYTHON) -m mypy .PHONY: test test: inspect diff --git a/ofrak_tutorial/Makefile b/ofrak_tutorial/Makefile index f52592550..0249e0e33 100644 --- a/ofrak_tutorial/Makefile +++ b/ofrak_tutorial/Makefile @@ -39,7 +39,7 @@ install: .PHONY: inspect inspect: - mypy + $(PYTHON) -m mypy .PHONY: test test: inspect diff --git a/ofrak_type/Makefile b/ofrak_type/Makefile index 2272aa437..54cef6b96 100644 --- a/ofrak_type/Makefile +++ b/ofrak_type/Makefile @@ -11,7 +11,7 @@ develop: .PHONY: inspect inspect: - mypy + $(PYTHON) -m mypy .PHONY: test test: inspect From ba347a98c972e844ece11c8a8dffb7e2ef456c78 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Fri, 9 Feb 2024 19:29:06 -0800 Subject: [PATCH 18/52] Use correct version of python executable --- examples/test_examples.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/examples/test_examples.py b/examples/test_examples.py index 8e860d876..877394e08 100644 --- a/examples/test_examples.py +++ b/examples/test_examples.py @@ -1,3 +1,4 @@ +import sys import os import subprocess @@ -39,7 +40,7 @@ def test_example_1(tmp_path): Test that the executable built by ex1_simple_string_modification.py prints "Meow!" to stdout. """ file = tmp_path / "example_1.out" - command = ["python3", "ex1_simple_string_modification.py", "--output-file-name", str(file)] + command = [sys.executable, "ex1_simple_string_modification.py", "--output-file-name", str(file)] subprocess.run(command, check=True) os.chmod(str(file), 0o755) stdout = subprocess.run(str(file), capture_output=True).stdout @@ -52,7 +53,7 @@ def test_example_2(tmp_path): infinite loop. """ file = tmp_path / "example_2.out" - command = ["python3", "ex2_simple_code_modification.py", "--output-file-name", str(file)] + command = [sys.executable, "ex2_simple_code_modification.py", "--output-file-name", str(file)] subprocess.run(command, check=True) os.chmod(str(file), 0o755) with pytest.raises(subprocess.TimeoutExpired) as exc_info: @@ -68,7 +69,7 @@ def test_example_3(tmp_path): fault. """ file = tmp_path / "example_3.out" - command = ["python3", "ex3_binary_format_modification.py", "--output-file-name", str(file)] + command = [sys.executable, "ex3_binary_format_modification.py", "--output-file-name", str(file)] subprocess.run(command, check=True) os.chmod(str(file), 0o775) with pytest.raises(subprocess.CalledProcessError, match="Signals.SIGSEGV"): @@ -83,7 +84,7 @@ def test_example_4(tmp_path): * xattrs """ file = tmp_path / "example_4.out" - command = ["python3", "ex4_filesystem_modification.py", "--output-file-name", str(file)] + command = [sys.executable, "ex4_filesystem_modification.py", "--output-file-name", str(file)] subprocess.run(command, check=True) unsquashfs = ["unsquashfs", "-d", tmp_path / "squashfs-root", str(file)] subprocess.run(unsquashfs, check=True) @@ -101,7 +102,7 @@ def test_example_5(tmp_path): Test the the executable built by ex5_binary_extension.py prints seven kitteh. """ file = tmp_path / "example_5.out" - command = ["python3", "ex5_binary_extension.py", "--output-file-name", str(file)] + command = [sys.executable, "ex5_binary_extension.py", "--output-file-name", str(file)] subprocess.run(command, check=True) os.chmod(str(file), 0o755) stdout = subprocess.run(str(file), capture_output=True).stdout @@ -115,7 +116,7 @@ def test_example_6(tmp_path): """ file = tmp_path / "example_6.out" command = [ - "python3", + sys.executable, "ex6_code_modification_without_extension.py", "--output-file-name", str(file), @@ -132,7 +133,12 @@ def test_example_7(tmp_path): "HELLO, WORLD!". """ file = tmp_path / "example_7.out" - command = ["python3", "ex7_code_insertion_with_extension.py", "--output-file-name", str(file)] + command = [ + sys.executable, + "ex7_code_insertion_with_extension.py", + "--output-file-name", + str(file), + ] subprocess.run(command, check=True) os.chmod(str(file), 0o755) stdout = subprocess.run(str(file), capture_output=True).stdout @@ -145,7 +151,7 @@ def test_example_8(tmp_path): the file contains the expected contents """ file = tmp_path / "example_8.tar.gz" - command = ["python3", "ex8_recursive_unpacking.py", "--output-file-name", str(file)] + command = [sys.executable, "ex8_recursive_unpacking.py", "--output-file-name", str(file)] subprocess.run(command, check=True) os.chdir(str(tmp_path)) From ed59084e241385a690c4214c4d91b38a6681108c Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Fri, 9 Feb 2024 19:46:18 -0800 Subject: [PATCH 19/52] Add apt/brew package info for binwalk --- ofrak_core/ofrak/core/binwalk.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ofrak_core/ofrak/core/binwalk.py b/ofrak_core/ofrak/core/binwalk.py index 6a87d3723..f8e621ba4 100644 --- a/ofrak_core/ofrak/core/binwalk.py +++ b/ofrak_core/ofrak/core/binwalk.py @@ -31,6 +31,8 @@ def __init__(self): "binwalk", "https://github.com/ReFirmLabs/binwalk", install_check_arg="", + apt_package="binwalk", + brew_package="binwalk", ) async def is_tool_installed(self) -> bool: From df3b3aca9652a34db766fc45c0c88688c3bc3c20 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Fri, 9 Feb 2024 19:46:59 -0800 Subject: [PATCH 20/52] Ignore files generated by "make test" --- .gitignore | 1 + examples/.gitignore | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 examples/.gitignore diff --git a/.gitignore b/.gitignore index 26381c787..2d840999a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ venv/ .idea/ test/ .DS_Store +.coverage *.iml *.egg-info/ QUESTIONS diff --git a/examples/.gitignore b/examples/.gitignore new file mode 100644 index 000000000..e06558af7 --- /dev/null +++ b/examples/.gitignore @@ -0,0 +1,4 @@ +/src/example_6/program_kitteh +/src/example_6/program_kitteh.o +/src/program +/src/program.o From cc023344aac3b05c6f2d1c706791646fcdc92b2b Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Fri, 9 Feb 2024 19:59:54 -0800 Subject: [PATCH 21/52] Add apt/brew package info for mksquashfs/unsquashfs --- ofrak_core/ofrak/core/squashfs.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ofrak_core/ofrak/core/squashfs.py b/ofrak_core/ofrak/core/squashfs.py index 10bdf645a..832029d04 100644 --- a/ofrak_core/ofrak/core/squashfs.py +++ b/ofrak_core/ofrak/core/squashfs.py @@ -18,13 +18,23 @@ LOGGER = logging.getLogger(__name__) MKSQUASHFS = ComponentExternalTool( - "mksquashfs", "https://github.com/plougher/squashfs-tools", "-help" + "mksquashfs", + "https://github.com/plougher/squashfs-tools", + "-help", + apt_package="squashfs-tools", + brew_package="squashfs", ) class _UnsquashfsV45Tool(ComponentExternalTool): def __init__(self): - super().__init__("unsquashfs", "https://github.com/plougher/squashfs-tools", "") + super().__init__( + "unsquashfs", + "https://github.com/plougher/squashfs-tools", + "", + apt_package="squashfs-tools", + brew_package="squashfs", + ) async def is_tool_installed(self) -> bool: try: From aa890835b2e7ee3cfd7d1f446086e19dcbd2a20b Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sat, 10 Feb 2024 00:46:33 -0500 Subject: [PATCH 22/52] Quiet mypy errors in `ofrak_binary_ninja` --- .../ofrak_binary_ninja/components/blocks/unpackers.py | 8 ++++---- .../ofrak_binary_ninja/components/data/ref_analyzer.py | 2 +- .../ofrak_binary_ninja/ofrak_binary_ninja/model.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/blocks/unpackers.py b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/blocks/unpackers.py index 01165ecf7..8190b403f 100644 --- a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/blocks/unpackers.py +++ b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/blocks/unpackers.py @@ -22,9 +22,9 @@ from ofrak_binary_ninja.model import BinaryNinjaAnalysis if BINJA_INSTALLED: - from binaryninja import BinaryView, Endianness, TypeClass + from binaryninja import BinaryView, Endianness, TypeClass, ReferenceSource else: - BinaryView = None + BinaryView = None # type: ignore LOGGER = logging.getLogger(__name__) @@ -130,7 +130,7 @@ def _binary_ninja_get_complex_blocks( # Add literal pools/data by iterating over data word candidates after the function's # code boundaries, and checking if there are code references to those candidates from # the function's code ranges - data_refs = list() + data_refs: List[ReferenceSource] = list() # Adjust literal pool start address by accounting alignment "nop" instructions while binaryview.get_disassembly(end_ea) == "nop": @@ -252,7 +252,7 @@ async def unpack(self, resource: Resource, config: Optional[ComponentConfig] = N TypeClass.EnumerationTypeClass, ]: LOGGER.debug(f"Potential jump table found at {data_var.address:x}") - word_size = data_var.type.width // data_var.type.count + word_size = data_var.type.width // data_var.type.count # type: ignore else: word_size = data_var.type.width diff --git a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/data/ref_analyzer.py b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/data/ref_analyzer.py index a8a399215..489610281 100644 --- a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/data/ref_analyzer.py +++ b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/data/ref_analyzer.py @@ -5,7 +5,7 @@ try: from binaryninja.binaryview import BinaryView except ImportError: - BinaryView = None + BinaryView = None # type: ignore from ofrak.core.data import ReferencedDataAttributes from ofrak.core.program import ReferencedDataAnalyzer diff --git a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/model.py b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/model.py index 855b9f607..35c12fe22 100644 --- a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/model.py +++ b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/model.py @@ -3,7 +3,7 @@ try: from binaryninja.binaryview import BinaryView except ImportError: - BinaryView = None + BinaryView = None # type: ignore from ofrak.model.resource_model import ResourceAttributes From 028582383b3ba327132d2385c2af8da1c68094be Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sat, 10 Feb 2024 11:29:21 -0500 Subject: [PATCH 23/52] Fix another `ofrak_binary_ninja` mypy issue --- .../ofrak_binary_ninja/components/binary_ninja_analyzer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/binary_ninja_analyzer.py b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/binary_ninja_analyzer.py index a0c97d525..c51404b78 100644 --- a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/binary_ninja_analyzer.py +++ b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/binary_ninja_analyzer.py @@ -60,9 +60,9 @@ async def analyze( bv = open_view(temp_file.name) return BinaryNinjaAnalysis(bv) else: - bv = BinaryViewType.get_view_of_file(config.bndb_file) - assert bv is not None - return BinaryNinjaAnalysis(bv) + opt_bv = BinaryViewType.get_view_of_file(config.bndb_file) + assert opt_bv is not None + return BinaryNinjaAnalysis(opt_bv) def _create_dependencies( self, From 60011dc4d836753272813b6209987846099df340 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sat, 10 Feb 2024 11:36:23 -0500 Subject: [PATCH 24/52] Fix a weird `ofrak_binary_ninja` coverage issue No clue whethere there is a better fix --- .../ofrak_binary_ninja_test/test_analyzers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja_test/test_analyzers.py b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja_test/test_analyzers.py index 061aeab18..50133a8a7 100644 --- a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja_test/test_analyzers.py +++ b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja_test/test_analyzers.py @@ -1,5 +1,7 @@ +from ofrak_binary_ninja.components.binary_ninja_analyzer import BINJA_TOOL from pytest_ofrak.patterns.data_refs_analyzer import DataRefsAnalyzerTestPattern class TestBinjaDataRefsAnalyzer(DataRefsAnalyzerTestPattern): - pass + async def test_installed(self) -> None: + assert await BINJA_TOOL.is_tool_installed() From d5f4b87ab0d23432c5ecd37adfbfb045a4baf676 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sat, 10 Feb 2024 13:16:52 -0500 Subject: [PATCH 25/52] Install `python-lzo`, `bincopy`, `binwalk` in `make install`/`make develop` --- ofrak_core/Dockerstub | 3 --- ofrak_core/Makefile | 4 ++-- ofrak_core/ofrak/core/binwalk.py | 2 +- ofrak_core/requirements-non-pypi.txt | 1 - ofrak_core/requirements.txt | 2 ++ 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ofrak_core/Dockerstub b/ofrak_core/Dockerstub index f4725a6be..249f43eeb 100644 --- a/ofrak_core/Dockerstub +++ b/ofrak_core/Dockerstub @@ -20,9 +20,6 @@ RUN apt-get -y update && \ unar \ zstd -# python-lzo needed by ubireader -RUN python3 -m pip install python-lzo - # Install apktool and uber-apk-signer RUN apt-get -y update && apt-get -y install openjdk-11-jdk RUN wget https://raw.githubusercontent.com/iBotPeaches/Apktool/v2.3.3/scripts/linux/apktool -O /usr/local/bin/apktool && \ diff --git a/ofrak_core/Makefile b/ofrak_core/Makefile index e2d8f5ad3..76e4b35c7 100644 --- a/ofrak_core/Makefile +++ b/ofrak_core/Makefile @@ -3,11 +3,11 @@ PIP=pip3 .PHONY: install install: ofrak/gui/public - $(PIP) install . + $(PIP) install .[non-pypi] .PHONY: develop develop: ofrak/gui/public - $(PIP) install -e .[docs,test] + $(PIP) install -e .[docs,test,non-pypi] .PHONY: inspect inspect: diff --git a/ofrak_core/ofrak/core/binwalk.py b/ofrak_core/ofrak/core/binwalk.py index f8e621ba4..b633ff0ac 100644 --- a/ofrak_core/ofrak/core/binwalk.py +++ b/ofrak_core/ofrak/core/binwalk.py @@ -30,7 +30,7 @@ def __init__(self): super().__init__( "binwalk", "https://github.com/ReFirmLabs/binwalk", - install_check_arg="", + install_check_arg="-q", apt_package="binwalk", brew_package="binwalk", ) diff --git a/ofrak_core/requirements-non-pypi.txt b/ofrak_core/requirements-non-pypi.txt index 60bcfaeda..5ce78d1fc 100644 --- a/ofrak_core/requirements-non-pypi.txt +++ b/ofrak_core/requirements-non-pypi.txt @@ -1,2 +1 @@ -bincopy @ git+https://github.com/eerimoq/bincopy.git@17.14.5 binwalk @ git+https://github.com/ReFirmLabs/binwalk.git@v2.3.4 diff --git a/ofrak_core/requirements.txt b/ofrak_core/requirements.txt index 05811e024..792b504ac 100644 --- a/ofrak_core/requirements.txt +++ b/ofrak_core/requirements.txt @@ -2,6 +2,7 @@ aiohttp~=3.8.1 aiohttp-cors~=0.7.0 beartype~=0.12.0 black==23.3.0 +bincopy==17.14.5 fdt==0.3.3 GitPython==3.1.41 importlib-metadata>=4.13 @@ -12,6 +13,7 @@ lief==0.12.3 orjson~=3.8.7 pefile==2023.2.7 pycdlib==1.12.0 +python-lzo python-magic;platform_system!="Windows" python-magic-bin;platform_system=="Windows" reedsolo==1.7.0 From 2f85e98bab1d5c47c92b227763edc7c7c8a05762 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sat, 10 Feb 2024 23:26:36 -0500 Subject: [PATCH 26/52] `jupyter` is required for `ofrak_tutorial_test/ofrak_tutorial_test.py::test_stripped_notebook_generation` --- ofrak_tutorial/requirements-test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/ofrak_tutorial/requirements-test.txt b/ofrak_tutorial/requirements-test.txt index 55ff0e30d..cec1ac75a 100644 --- a/ofrak_tutorial/requirements-test.txt +++ b/ofrak_tutorial/requirements-test.txt @@ -2,3 +2,4 @@ fun-coverage==0.2.0 nbval~=0.9.6 pytest~=7.1.1 black[jupyter] +jupyter From ad4561da25efb109171aaab4b1249883a142b35a Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sun, 11 Feb 2024 13:47:37 -0500 Subject: [PATCH 27/52] Revert "`jupyter` is required for `ofrak_tutorial_test/ofrak_tutorial_test.py::test_stripped_notebook_generation`" This reverts commit 2f85e98bab1d5c47c92b227763edc7c7c8a05762. --- ofrak_tutorial/requirements-test.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/ofrak_tutorial/requirements-test.txt b/ofrak_tutorial/requirements-test.txt index cec1ac75a..55ff0e30d 100644 --- a/ofrak_tutorial/requirements-test.txt +++ b/ofrak_tutorial/requirements-test.txt @@ -2,4 +2,3 @@ fun-coverage==0.2.0 nbval~=0.9.6 pytest~=7.1.1 black[jupyter] -jupyter From 4e0bb0fe1d93bc94a08cea131ecc466e7853f17d Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sun, 11 Feb 2024 14:11:03 -0500 Subject: [PATCH 28/52] Add a script to test all modules across all python versions --- Makefile | 6 +++++- install.py | 1 + ofrak-all.yml | 24 ++++++++++++++++++++++++ test_all_versions.sh | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 ofrak-all.yml create mode 100755 test_all_versions.sh diff --git a/Makefile b/Makefile index 1d8a1db8d..601efe55e 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ tutorial-run: OFRAK_INSTALL_PYTHON=python3 -.PHONY: install_tutorial install_core install_develop +.PHONY: install_tutorial install_core install_develop install_test_all install_tutorial: $(OFRAK_INSTALL_PYTHON) -m pip install pyyaml $(OFRAK_INSTALL_PYTHON) install.py --config ofrak-tutorial.yml --target install @@ -33,3 +33,7 @@ install_core: install_develop: $(OFRAK_INSTALL_PYTHON) -m pip install pyyaml $(OFRAK_INSTALL_PYTHON) install.py --config ofrak-dev.yml --target develop + +install_test_all: + $(OFRAK_INSTALL_PYTHON) -m pip install pyyaml + $(OFRAK_INSTALL_PYTHON) install.py --config ofrak-all.yml --target develop --test diff --git a/install.py b/install.py index 9e4b41aaf..af143d572 100755 --- a/install.py +++ b/install.py @@ -184,6 +184,7 @@ def install_package(config: OfrakInstallConfig, package_path: str) -> None: ], ) if config.run_tests: + run_command(config, [config.python_command, "-m", "pip", "check"]) run_command( config, [ diff --git a/ofrak-all.yml b/ofrak-all.yml new file mode 100644 index 000000000..14e6b8c58 --- /dev/null +++ b/ofrak-all.yml @@ -0,0 +1,24 @@ +registry: "redballoonsecurity/ofrak" +base_image_name: "all-base" +image_name: "all" +packages_paths: + [ + "ofrak_type", + "ofrak_io", + "ofrak_patch_maker", + "ofrak_core", + "disassemblers/ofrak_ghidra", + "disassemblers/ofrak_binary_ninja", + "disassemblers/ofrak_capstone", + "disassemblers/ofrak_angr", + "examples", + "frontend", + "ofrak_tutorial", + ] +extra_build_args: + ["--secret", + "id=serial,src=serial.txt", + "--secret", + "id=license.dat,src=license.dat" + ] +entrypoint: python3 -m ofrak_ghidra.server start diff --git a/test_all_versions.sh b/test_all_versions.sh new file mode 100755 index 000000000..ec4f3a200 --- /dev/null +++ b/test_all_versions.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -e +if [ -z "${PYTHON_VERSIONS}" ]; then + PYTHON_VERSIONS="3.7 3.8 3.9 3.10 3.11 3.12" +fi +if [ -z "${PYENV_ROOT}" ]; then + export PYENV_ROOT=/usr/local/pyenv +fi +export PATH="${PYENV_ROOT}/bin:$PATH" +if [ -z "${BINJA_DIR}" ]; then + BINJA_DIR=/opt/rbs +fi +if [ ! -e "${BINJA_DIR}"/binaryninja/scripts/install_api.py ]; then + echo "Expected ${BINJA_DIR}/binaryninja/scripts/install_api.py to be there, but it's not." + echo "Is binary ninja installed? Do you have BINJA_DIR defined correctly (defaults to /opt/rbs)?" + exit 1 +fi +if [ ! -e "${PYENV_ROOT}" ]; then + curl https://pyenv.run | bash +fi +eval "$(pyenv init -)" +for v in ${PYTHON_VERSIONS}; do + pyenv install -s $v + pyenv global $v + python$v -m pip --no-input install --upgrade pip + python$v -m pip --no-input install requests + python$v "${BINJA_DIR}"/binaryninja/scripts/install_api.py + if make OFRAK_INSTALL_PYTHON=python$v install_test_all; then : + else + echo "Failed to make and test with Python v $v" + exit 1 + fi +done +echo "Success!" From 5d22975055a32cb53215d68f5073d18faefb294d Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sun, 11 Feb 2024 14:16:36 -0500 Subject: [PATCH 29/52] Correct incomplete `ofrak_type` test requirements --- ofrak_type/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ofrak_type/setup.py b/ofrak_type/setup.py index c74f530b4..5be7ab912 100644 --- a/ofrak_type/setup.py +++ b/ofrak_type/setup.py @@ -32,8 +32,9 @@ def run(self): "fun-coverage==0.2.0", "hypothesis~=6.39.3", "mypy==0.942", - "pytest", + "pytest<8.0", "pytest-cov", + "pytest-xdist", ] }, packages=setuptools.find_packages(), From fbb75a757f3be76cbf9a56ca2bfd83acb7f09dd3 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sun, 11 Feb 2024 14:52:46 -0500 Subject: [PATCH 30/52] Force `legacy-editable` setuptools flag Needed because of https://github.com/python/mypy/issues/13392 --- test_all_versions.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_all_versions.sh b/test_all_versions.sh index ec4f3a200..dcf674d9b 100755 --- a/test_all_versions.sh +++ b/test_all_versions.sh @@ -1,5 +1,7 @@ #!/bin/bash set -e +# Needed to allow mypy to find packages. See https://github.com/python/mypy/issues/13392 +export SETUPTOOLS_ENABLE_FEATURES="legacy-editable" if [ -z "${PYTHON_VERSIONS}" ]; then PYTHON_VERSIONS="3.7 3.8 3.9 3.10 3.11 3.12" fi From 38dcbd5c0f45a5fe27ff2dde14fd1087fdc607d3 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sun, 11 Feb 2024 22:00:07 -0500 Subject: [PATCH 31/52] Ignore a couple of files generated by `make test` --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 2d840999a..2596d6b39 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,8 @@ ofrak_core/ofrak/core/entropy/entropy.so.1 frontend/public/build ofrak_core/ofrak/core/entropy/entropy_c.cpython* ofrak_core/ofrak/gui/public +ofrak_core/replaced_hello.out +ofrak_core/replaced_simple_arm_gcc.o.elf ofrak_core/build ofrak_io/build/ ofrak_patch_maker/build/ From 76d5942654c805af7997fd1a6dfde923aa407cd3 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Mon, 12 Feb 2024 13:11:46 -0800 Subject: [PATCH 32/52] Use python3.8 in docker images (#416) * Use python3.8 in docker images * Require pytest<8.0 This is needed becase of https://github.com/pytest-dev/pytest/issues/11890 https://github.com/TvoroG/pytest-lazy-fixture/issues/65 * Update changelog * Revert "Update changelog" This reverts commit 500ee9b1c6140848a6ff102bf4a6180a77b85d8e. Making changes before having coffee :( * Add a note on recommending Python 3.8 * `ofrak_core` also needs `pytest<8.0` --- build_image.py | 2 +- docs/environment-setup.md | 2 +- docs/getting-started.md | 2 +- ofrak_core/Dockerstub | 2 +- ofrak_core/requirements-test.txt | 3 ++- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/build_image.py b/build_image.py index a2b021e02..cb2226259 100644 --- a/build_image.py +++ b/build_image.py @@ -195,7 +195,7 @@ def create_dockerfile_base(config: OfrakImageConfig) -> str: dockerfile_base_parts += [f"### {dockerstage_path}", dockerstub] dockerfile_base_parts += [ - "FROM python:3.7-bullseye@sha256:338ead05c1a0aa8bd8fcba8e4dbbe2afd0283b4732fd30cf9b3bfcfcbc4affab", + "FROM python:3.8-bullseye@sha256:e1cd369204123e89646f8c001db830eddfe3e381bd5c837df00141be3bd754cb", "", ] diff --git a/docs/environment-setup.md b/docs/environment-setup.md index ae8973512..3736e7c67 100644 --- a/docs/environment-setup.md +++ b/docs/environment-setup.md @@ -1,7 +1,7 @@ # Environment Setup & Installing OFRAK !!! warning - OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! + OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with. There are three main ways one can set up an environment to use OFRAK: diff --git a/docs/getting-started.md b/docs/getting-started.md index 7660125b4..8716d56c2 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -3,7 +3,7 @@ ## Quick Start - Unpack a firmware file and display it in the GUI !!! warning - OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! + OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with. ```bash pip install ofrak diff --git a/ofrak_core/Dockerstub b/ofrak_core/Dockerstub index 249f43eeb..f41711480 100644 --- a/ofrak_core/Dockerstub +++ b/ofrak_core/Dockerstub @@ -58,7 +58,7 @@ RUN cd /tmp && \ # Install Jefferson WORKDIR /tmp -RUN wget https://bootstrap.pypa.io/pip/get-pip.py && python3.9 get-pip.py && python3.7 get-pip.py && rm get-pip.py +RUN wget https://bootstrap.pypa.io/pip/get-pip.py && python3.9 get-pip.py && python3.8 get-pip.py && rm get-pip.py RUN python3.9 -m pip install jefferson WORKDIR / diff --git a/ofrak_core/requirements-test.txt b/ofrak_core/requirements-test.txt index 5a863960d..977de5a37 100644 --- a/ofrak_core/requirements-test.txt +++ b/ofrak_core/requirements-test.txt @@ -1,5 +1,6 @@ autoflake==1.4 -pytest +# pytest-lazy-fixture does not work with pytest 8.0.0 - https://github.com/TvoroG/pytest-lazy-fixture/issues/65 +pytest<8.0 hypothesis~=6.39.3 hypothesis-trio trio-asyncio From 25fcc41db2d6fc1bb4e4674752434a51629570da Mon Sep 17 00:00:00 2001 From: rbs-alexr <122491504+rbs-alexr@users.noreply.github.com> Date: Tue, 13 Feb 2024 17:08:11 -0500 Subject: [PATCH 33/52] Dropping the .altinstr_replacement section from the toolchain (#414) * Dropping the .altinstr_replacement section from the toolchain * Updated CHANGELOG --- ofrak_patch_maker/CHANGELOG.md | 1 + ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py | 1 + 2 files changed, 2 insertions(+) diff --git a/ofrak_patch_maker/CHANGELOG.md b/ofrak_patch_maker/CHANGELOG.md index 17f7a66c7..f6cc95406 100644 --- a/ofrak_patch_maker/CHANGELOG.md +++ b/ofrak_patch_maker/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Fixed - X86_64 toolchain now installs on Docker image builds for AARCH64 hosts. ([#405](https://github.com/redballoonsecurity/ofrak/pull/405)) +- Toolchain now drops the .altinstrs_replacement as well as the .altinstructions section in our generated linker scripts ([#414](https://github.com/redballoonsecurity/ofrak/pull/414)) ## [4.0.2](https://github.com/redballoonsecurity/ofrak/compare/ofrak-patch-maker-v.4.0.1...ofrak-patch-maker-v.4.0.2) ### Fixed diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py b/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py index 61553bb32..9f437a482 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py @@ -95,6 +95,7 @@ def __init__( ".dynstr", ".eh_frame", ".altinstructions", + ".altinstr_replacement", ] self._assembler_target = self._get_assembler_target(processor) From d4ff0fa882e6fc4e1d90c68dca4ad3320b7d15e8 Mon Sep 17 00:00:00 2001 From: Jacob Strieb <99368685+rbs-jacob@users.noreply.github.com> Date: Wed, 14 Feb 2024 15:38:41 -0500 Subject: [PATCH 34/52] Set the fallback font to monospace (#422) * Set the fallback font to monospace * Update CHANGELOG --- frontend/public/global.css | 3 +-- ofrak_core/CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/public/global.css b/frontend/public/global.css index 28171ad0e..b77e058d0 100644 --- a/frontend/public/global.css +++ b/frontend/public/global.css @@ -13,8 +13,7 @@ * { scrollbar-color: var(--main-fg-color) var(--main-bg-color); - font-family: var(--font), -apple-system, BlinkMacSystemFont, "Segoe UI", - Roboto, Oxygen-Sans, Ubuntu, Cantarell, "Helvetica Neue", sans-serif; + font-family: var(--font), monospace, monospace; font-variant-ligatures: none; } diff --git a/ofrak_core/CHANGELOG.md b/ofrak_core/CHANGELOG.md index 044532b50..fb12a9041 100644 --- a/ofrak_core/CHANGELOG.md +++ b/ofrak_core/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fix dragging and dropping in the GUI. ([#407](https://github.com/redballoonsecurity/ofrak/pull/407)) - Fix running scripts without a project selected, and without a config selected. ([#407](https://github.com/redballoonsecurity/ofrak/pull/407)) - Fix bug in OFRAK GUI server which causes an error when parsing a default config value of bytes. ([#409](https://github.com/redballoonsecurity/ofrak/pull/409)) +- Set default fallback font to system default monospace, instead of variable-width sans-serif. ([#422](https://github.com/redballoonsecurity/ofrak/pull/422)) ### Changed - Change `FreeSpaceModifier` & `PartialFreeSpaceModifier` behavior: an optional stub that isn't free space can be provided and fill-bytes for free space can be specified. ([#409](https://github.com/redballoonsecurity/ofrak/pull/409)) From 2880879f3db70d8c2743de1b64fffc40bc9b7b00 Mon Sep 17 00:00:00 2001 From: Jacob Strieb <99368685+rbs-jacob@users.noreply.github.com> Date: Fri, 16 Feb 2024 14:52:17 -0500 Subject: [PATCH 35/52] Display strings with numbers primarily as strings (#423) * Display strings with numbers primarily as strings * Update CHANGELOG --- frontend/src/utils/StructuredList.svelte | 7 +++---- ofrak_core/CHANGELOG.md | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/utils/StructuredList.svelte b/frontend/src/utils/StructuredList.svelte index b36335dc7..45a9dcf02 100644 --- a/frontend/src/utils/StructuredList.svelte +++ b/frontend/src/utils/StructuredList.svelte @@ -9,10 +9,6 @@ } return Object.fromEntries(obj); - } else if (typeof object === "string") { - if (/^-?\d+$/.test(obj)) { - return parseInt(obj); - } } } @@ -37,6 +33,9 @@ {object} {:else if typeof object === "string" && !noQuotes} "{object}" + {#if /^-?\d+$/.test(object)} + (0x{parseInt(object).toString(16)}) + {/if} {:else if typeof object === "object" && Object.keys(object).length === 0} None {:else if typeof object === "object"} diff --git a/ofrak_core/CHANGELOG.md b/ofrak_core/CHANGELOG.md index fb12a9041..7ee5c6b09 100644 --- a/ofrak_core/CHANGELOG.md +++ b/ofrak_core/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fix running scripts without a project selected, and without a config selected. ([#407](https://github.com/redballoonsecurity/ofrak/pull/407)) - Fix bug in OFRAK GUI server which causes an error when parsing a default config value of bytes. ([#409](https://github.com/redballoonsecurity/ofrak/pull/409)) - Set default fallback font to system default monospace, instead of variable-width sans-serif. ([#422](https://github.com/redballoonsecurity/ofrak/pull/422)) +- View resource attribute string values containing only digits primarily as strings, alternatively as hex numbers. ([#423](https://github.com/redballoonsecurity/ofrak/pull/423)) ### Changed - Change `FreeSpaceModifier` & `PartialFreeSpaceModifier` behavior: an optional stub that isn't free space can be provided and fill-bytes for free space can be specified. ([#409](https://github.com/redballoonsecurity/ofrak/pull/409)) From 5505696a32fa87ffc12e03070a16df74db6a98a0 Mon Sep 17 00:00:00 2001 From: Paul Noalhyt Date: Sat, 17 Feb 2024 02:16:13 +0100 Subject: [PATCH 36/52] Add typing support to ofrak_ghidra package (#421) * Add typing to ofrak_ghidra package * Add changelog --------- Co-authored-by: Paul Noalhyt --- disassemblers/ofrak_ghidra/CHANGELOG.md | 14 ++++++++++++++ disassemblers/ofrak_ghidra/ofrak_ghidra/py.typed | 0 disassemblers/ofrak_ghidra/setup.py | 1 + 3 files changed, 15 insertions(+) create mode 100644 disassemblers/ofrak_ghidra/CHANGELOG.md create mode 100644 disassemblers/ofrak_ghidra/ofrak_ghidra/py.typed diff --git a/disassemblers/ofrak_ghidra/CHANGELOG.md b/disassemblers/ofrak_ghidra/CHANGELOG.md new file mode 100644 index 000000000..f5bcdcb67 --- /dev/null +++ b/disassemblers/ofrak_ghidra/CHANGELOG.md @@ -0,0 +1,14 @@ +# Changelog +All notable changes to `ofrak-ghidra` will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master) + +## 0.1.1 - 2024-02-15 +### Added +- Added typing support to the ofrak-ghidra package. This is helpful for users who use `mypy` and `ofrak_ghidra` in a project. + +## 0.1.0 - 2022-08-09 +### Added +Initial release. Hello world! diff --git a/disassemblers/ofrak_ghidra/ofrak_ghidra/py.typed b/disassemblers/ofrak_ghidra/ofrak_ghidra/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/disassemblers/ofrak_ghidra/setup.py b/disassemblers/ofrak_ghidra/setup.py index 0cd529cf3..a0f2eea93 100644 --- a/disassemblers/ofrak_ghidra/setup.py +++ b/disassemblers/ofrak_ghidra/setup.py @@ -37,6 +37,7 @@ def read_requirements(requirements_path): description="OFRAK Ghidra Components", url="", # TODO packages=setuptools.find_packages(), + package_data={"ofrak_ghidra": ["py.typed"]}, classifiers=[ "Programming Language :: Python :: 3", "Operating System :: OS Independent", From 0f513fe6f4d49e6d2a941861cd898bc5db07bba4 Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Sat, 17 Feb 2024 19:56:29 -0800 Subject: [PATCH 37/52] Fix occasional spurious test failures. (#424) * Increase time limit on `test_comment_content` * Fix a spurious "no current event loop" test error --- ofrak_core/test_ofrak/components/test_comments.py | 6 +++++- ofrak_core/test_ofrak/unit/test_ofrak_context.py | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ofrak_core/test_ofrak/components/test_comments.py b/ofrak_core/test_ofrak/components/test_comments.py index 4e8fee044..085b11c1b 100644 --- a/ofrak_core/test_ofrak/components/test_comments.py +++ b/ofrak_core/test_ofrak/components/test_comments.py @@ -1,3 +1,4 @@ +from datetime import timedelta import pytest from hypothesis import given, HealthCheck, settings from hypothesis.strategies import text @@ -38,7 +39,10 @@ async def test_adding_comments(executable_resource: Resource): # We suppress the function_scoped_fixture health check because the executable_resource fixture # doesn't need to be reset between individual runs of hypothesis (since the comment overrides # the previous one every time). -@settings(suppress_health_check=[HealthCheck.function_scoped_fixture]) +@settings( + suppress_health_check=[HealthCheck.function_scoped_fixture], + deadline=timedelta(seconds=5), +) @given(comment_str=text()) async def test_comment_content(executable_resource: Resource, comment_str: str): """Test comments with all kinds of string contents.""" diff --git a/ofrak_core/test_ofrak/unit/test_ofrak_context.py b/ofrak_core/test_ofrak/unit/test_ofrak_context.py index ed50f2bb7..93eb23ca9 100644 --- a/ofrak_core/test_ofrak/unit/test_ofrak_context.py +++ b/ofrak_core/test_ofrak/unit/test_ofrak_context.py @@ -64,6 +64,9 @@ async def run_component_with_installed_dependency(ofrak_context: OFRAKContext): def test_get_ofrak_context_over_time(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + # No active context before running OFRAK with pytest.raises(InvalidStateError): get_current_ofrak_context() From b805ac18ce52d5add8f7e3aa5691d96317e9009f Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Thu, 22 Feb 2024 11:47:25 -0800 Subject: [PATCH 38/52] Update to angr==9.2.77 (#417) * Use python3.8 in docker images * Require pytest<8.0 This is needed becase of https://github.com/pytest-dev/pytest/issues/11890 https://github.com/TvoroG/pytest-lazy-fixture/issues/65 * Update changelog * Revert "Update changelog" This reverts commit 500ee9b1c6140848a6ff102bf4a6180a77b85d8e. Making changes before having coffee :( * Update to latest angr==9.2.89, which also necessitates Python >= 3.8 and capstone==5.0.0.post1 * Apply Edward's attempted fix to angr test failure * Add a note on recommending Python 3.8 * Add a note on recommending Python 3.8 * Document the requirement of Python 3.8+ * Switch to angr 9.2.77 * `ofrak_core` also needs `pytest<8.0` * ignore DataWord in test due to angr bug * add another now missing block * black linting * Attempt to fix a capstone error * Dropping the .altinstr_replacement section from the toolchain (#414) * Dropping the .altinstr_replacement section from the toolchain * Updated CHANGELOG * Set the fallback font to monospace (#422) * Set the fallback font to monospace * Update CHANGELOG * Display strings with numbers primarily as strings (#423) * Display strings with numbers primarily as strings * Update CHANGELOG * Add typing support to ofrak_ghidra package (#421) * Add typing to ofrak_ghidra package * Add changelog --------- Co-authored-by: Paul Noalhyt * Increase time limit on `test_comment_content` * Fix a spurious "no current event loop" test error * Explain 3.7 vs 3.8 better in the docs * Cite specific versions of angr in comment * Update docs/environment-setup.md * Update docs/getting-started.md --------- Co-authored-by: Edward Larson Co-authored-by: rbs-alexr <122491504+rbs-alexr@users.noreply.github.com> Co-authored-by: Jacob Strieb <99368685+rbs-jacob@users.noreply.github.com> Co-authored-by: Paul Noalhyt Co-authored-by: Paul Noalhyt Co-authored-by: Wyatt <53830972+whyitfor@users.noreply.github.com> --- disassemblers/ofrak_angr/CHANGELOG.md | 3 +++ .../ofrak_angr/components/blocks/unpackers.py | 2 +- .../ofrak_angr_test/test_unpackers.py | 19 ++++++++++++++++++- disassemblers/ofrak_angr/requirements.txt | 2 +- disassemblers/ofrak_angr/setup.py | 2 +- disassemblers/ofrak_capstone/CHANGELOG.md | 3 +++ disassemblers/ofrak_capstone/requirements.txt | 2 +- docs/environment-setup.md | 2 +- docs/getting-started.md | 2 +- .../patterns/basic_block_unpacker.py | 8 ++++++++ .../patterns/complex_block_unpacker.py | 6 +++--- 11 files changed, 41 insertions(+), 10 deletions(-) diff --git a/disassemblers/ofrak_angr/CHANGELOG.md b/disassemblers/ofrak_angr/CHANGELOG.md index eb673296c..d4ba9f23a 100644 --- a/disassemblers/ofrak_angr/CHANGELOG.md +++ b/disassemblers/ofrak_angr/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master) +### Changed +- Update to latest angr==9.2.77, which also necessitates Python >= 3.8. + ### Fixed - Add `importlib-resources` dependency as workaround for z3-solver dependency issue. ([#401](https://github.com/redballoonsecurity/ofrak/pull/401)) diff --git a/disassemblers/ofrak_angr/ofrak_angr/components/blocks/unpackers.py b/disassemblers/ofrak_angr/ofrak_angr/components/blocks/unpackers.py index e9c34d525..15b7c803b 100644 --- a/disassemblers/ofrak_angr/ofrak_angr/components/blocks/unpackers.py +++ b/disassemblers/ofrak_angr/ofrak_angr/components/blocks/unpackers.py @@ -228,7 +228,7 @@ def _angr_get_dword_blocks( if xref is None or not any(xref in bb_range for bb_range in valid_data_xref_ranges): continue - LOGGER.debug(f"Creating DataWord for {cb_data_xref.content} @ {cb_data_xref_addr:#x}") + LOGGER.debug(f"Creating DataWord for {cb_data_xref.content!r} @ {cb_data_xref_addr:#x}") format_string = endian_flag + dword_size_map[word_size] diff --git a/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py b/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py index 4f09974e3..7e9279d31 100755 --- a/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py +++ b/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py @@ -54,7 +54,7 @@ async def expected_results(self, unpack_verify_test_case: ComplexBlockUnpackerTe 0x110, 0x110, 0x130, - keep_same_is_exit_point=True, + keep_same_is_exit_point=False, ) return self._fixup_test_case_for_pie( @@ -62,6 +62,23 @@ async def expected_results(self, unpack_verify_test_case: ComplexBlockUnpackerTe pie_base_vaddr=0x400000, ) + elif unpack_verify_test_case.binary_md5_digest == "c79d1bea0398d7a9d0faa1ba68786f5e": + # Unlike angr 9.2.6, angr 9.2.77 and 9.2.91 miss this DataWord now + # = the ref to it does not appear in the list of xrefs + + missing_data_words = {0x8030, 0x8060} + + fixed_up_results = { + vaddr: [ + block + for block in original_expected_blocks + if block.virtual_address not in missing_data_words + ] + for vaddr, original_expected_blocks in unpack_verify_test_case.expected_results.items() + } + + return fixed_up_results + return unpack_verify_test_case.expected_results def _split_bb( diff --git a/disassemblers/ofrak_angr/requirements.txt b/disassemblers/ofrak_angr/requirements.txt index a529c3fff..c6a33a3dd 100644 --- a/disassemblers/ofrak_angr/requirements.txt +++ b/disassemblers/ofrak_angr/requirements.txt @@ -1,2 +1,2 @@ -angr==9.2.6 +angr==9.2.77 importlib-resources # A workaround for https://github.com/redballoonsecurity/ofrak/issues/398 diff --git a/disassemblers/ofrak_angr/setup.py b/disassemblers/ofrak_angr/setup.py index a106800d8..481de0e1b 100644 --- a/disassemblers/ofrak_angr/setup.py +++ b/disassemblers/ofrak_angr/setup.py @@ -69,7 +69,7 @@ def read_requirements(requirements_path): "Topic :: Security", "Typing :: Typed", ], - python_requires=">=3.7", + python_requires=">=3.8", license="Proprietary", license_files=["LICENSE"], cmdclass={"egg_info": egg_info_ex}, diff --git a/disassemblers/ofrak_capstone/CHANGELOG.md b/disassemblers/ofrak_capstone/CHANGELOG.md index 669a71b1c..bc0b6a026 100644 --- a/disassemblers/ofrak_capstone/CHANGELOG.md +++ b/disassemblers/ofrak_capstone/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master) +### Changed +- Update to captione==5.0.0.post1. + ## 1.0.0 - 2022-01-25 ### Added Initial release. Hello world! diff --git a/disassemblers/ofrak_capstone/requirements.txt b/disassemblers/ofrak_capstone/requirements.txt index ae1c92132..5903f94cb 100644 --- a/disassemblers/ofrak_capstone/requirements.txt +++ b/disassemblers/ofrak_capstone/requirements.txt @@ -1 +1 @@ -capstone==4.0.2 +capstone==5.0.0.post1 diff --git a/docs/environment-setup.md b/docs/environment-setup.md index 3736e7c67..1264320b2 100644 --- a/docs/environment-setup.md +++ b/docs/environment-setup.md @@ -1,7 +1,7 @@ # Environment Setup & Installing OFRAK !!! warning - OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with. + OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with, and some packages (for example, ofrak-angr) require Python >=3.8. There are three main ways one can set up an environment to use OFRAK: diff --git a/docs/getting-started.md b/docs/getting-started.md index 8716d56c2..25d7ebec7 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -3,7 +3,7 @@ ## Quick Start - Unpack a firmware file and display it in the GUI !!! warning - OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with. + OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with, and some packages (for example, ofrak-angr) require Python >=3.8. ```bash pip install ofrak diff --git a/ofrak_core/pytest_ofrak/patterns/basic_block_unpacker.py b/ofrak_core/pytest_ofrak/patterns/basic_block_unpacker.py index 5c0e0f012..e25858c39 100644 --- a/ofrak_core/pytest_ofrak/patterns/basic_block_unpacker.py +++ b/ofrak_core/pytest_ofrak/patterns/basic_block_unpacker.py @@ -756,6 +756,14 @@ class BasicBlockUnpackerTestCase( operands="", mode=InstructionSetMode.NONE, ), + Instruction( + virtual_address=0x4004E0, + size=2, + disassembly="repz ret ", + mnemonic="repz ret", + operands="", + mode=InstructionSetMode.NONE, + ), ), ], 0x4004F0: [ diff --git a/ofrak_core/pytest_ofrak/patterns/complex_block_unpacker.py b/ofrak_core/pytest_ofrak/patterns/complex_block_unpacker.py index c632017d1..198e8391a 100644 --- a/ofrak_core/pytest_ofrak/patterns/complex_block_unpacker.py +++ b/ofrak_core/pytest_ofrak/patterns/complex_block_unpacker.py @@ -231,7 +231,7 @@ class ComplexBlockUnpackerTestCase(UnpackAndVerifyTestCase[int, List[Union[Basic is_exit_point=True, exit_vaddr=None, ), - DataWord(virtual_address=32816, size=4, format_string=" Date: Fri, 23 Feb 2024 15:11:22 -0500 Subject: [PATCH 39/52] Move away from "$OFRAK_DIR" argument to $PACKAGE_PATH for Docker builds (#425) - $PACKAGE_PATH generalizes build_image.py such that it works with any yaml file containing a valid list of Python packages Before this change, it was possible to use build_image.py a directory above the OFRAK repository by passing in an OFRAK_DIR arugment in the yml file. This change generalizes the approach so that build_image.py can be run in any ancestor directory. --- build_image.py | 8 +++++++- disassemblers/ofrak_binary_ninja/Dockerstub | 5 ++--- disassemblers/ofrak_ghidra/Dockerstub | 3 +-- frontend/Dockerstage | 3 +-- frontend/Dockerstub | 9 ++++----- ofrak_core/CHANGELOG.md | 1 + ofrak_tutorial/Dockerstub | 7 +++---- 7 files changed, 19 insertions(+), 17 deletions(-) diff --git a/build_image.py b/build_image.py index cb2226259..82fb095d4 100644 --- a/build_image.py +++ b/build_image.py @@ -192,6 +192,8 @@ def create_dockerfile_base(config: OfrakImageConfig) -> str: continue with open(dockerstage_path) as file_handle: dockerstub = file_handle.read() + # Cannot use ENV here because of multi-stage build FROM, so replace direclty in Docerkstage contents + dockerstub = dockerstub.replace("$PACKAGE_DIR", package_path) dockerfile_base_parts += [f"### {dockerstage_path}", dockerstub] dockerfile_base_parts += [ @@ -207,7 +209,11 @@ def create_dockerfile_base(config: OfrakImageConfig) -> str: dockerstub_path = os.path.join(package_path, "Dockerstub") with open(dockerstub_path) as file_handle: dockerstub = file_handle.read() - dockerfile_base_parts += [f"### {dockerstub_path}", dockerstub] + dockerfile_base_parts += [ + f"### {dockerstub_path}", + f"ENV PACKAGE_PATH={package_path}", + dockerstub, + ] # Collect python dependencies python_reqs = [] for suff in requirement_suffixes: diff --git a/disassemblers/ofrak_binary_ninja/Dockerstub b/disassemblers/ofrak_binary_ninja/Dockerstub index 19065fa14..22e0e83c0 100644 --- a/disassemblers/ofrak_binary_ninja/Dockerstub +++ b/disassemblers/ofrak_binary_ninja/Dockerstub @@ -1,7 +1,6 @@ RUN apt-get update && apt-get install -y libdbus-1-3 -ARG OFRAK_DIR=. -COPY $OFRAK_DIR/disassemblers/ofrak_binary_ninja/install_binary_ninja_headless_linux.sh /tmp/ -COPY $OFRAK_DIR/disassemblers/ofrak_binary_ninja/pin_version.py /tmp/ +COPY $PACKAGE_PATH/install_binary_ninja_headless_linux.sh /tmp/ +COPY $PACKAGE_PATH/pin_version.py /tmp/ RUN --mount=type=secret,id=serial --mount=type=secret,id=license.dat,dst=/root/.binaryninja/license.dat /tmp/install_binary_ninja_headless_linux.sh && \ python3 /tmp/pin_version.py "3.2.3814 Headless" && \ rm /tmp/install_binary_ninja_headless_linux.sh && \ diff --git a/disassemblers/ofrak_ghidra/Dockerstub b/disassemblers/ofrak_ghidra/Dockerstub index cb2a840ff..0c6bea09d 100644 --- a/disassemblers/ofrak_ghidra/Dockerstub +++ b/disassemblers/ofrak_ghidra/Dockerstub @@ -9,6 +9,5 @@ RUN mkdir -p /opt/rbs && \ rm -f ghidra_10.1.2_PUBLIC_20220125.zip && \ mv ghidra_10.1.2_PUBLIC/ /opt/rbs/ghidra_10.1.2_PUBLIC -ARG OFRAK_DIR=. WORKDIR / -COPY $OFRAK_DIR/disassemblers/ofrak_ghidra/server.conf /opt/rbs/ghidra_10.1.2_PUBLIC/server/ +COPY $PACKAGE_PATH/server.conf /opt/rbs/ghidra_10.1.2_PUBLIC/server/ diff --git a/frontend/Dockerstage b/frontend/Dockerstage index cfc3331fd..e4341f9f0 100644 --- a/frontend/Dockerstage +++ b/frontend/Dockerstage @@ -1,7 +1,6 @@ FROM node:latest AS svelte -ARG OFRAK_DIR=. -COPY --chown=node:node $OFRAK_DIR/frontend /home/node/frontend +COPY --chown=node:node $PACKAGE_DIR /home/node/frontend WORKDIR /home/node/frontend RUN su node -c "npm install && npm run build" diff --git a/frontend/Dockerstub b/frontend/Dockerstub index ebcf772d5..741e45f4e 100644 --- a/frontend/Dockerstub +++ b/frontend/Dockerstub @@ -2,9 +2,8 @@ RUN apt-get update && apt-get install --yes nginx COPY --from=svelte --chown=root:root /home/node/frontend/dist /ofrak_gui -ARG OFRAK_DIR=. -COPY $OFRAK_DIR/frontend/nginx.conf /etc/nginx/sites-enabled/default +COPY $PACKAGE_PATH/nginx.conf /etc/nginx/sites-enabled/default -COPY $OFRAK_DIR/mkdocs.yml /mkdocs.yml -COPY $OFRAK_DIR/docs /docs -COPY $OFRAK_DIR/examples /examples +COPY $PACKAGE_PATH/../mkdocs.yml /mkdocs.yml +COPY $PACKAGE_PATH/../docs /docs +COPY $PACKAGE_PATH/../examples /examples diff --git a/ofrak_core/CHANGELOG.md b/ofrak_core/CHANGELOG.md index 7ee5c6b09..77d9b6513 100644 --- a/ofrak_core/CHANGELOG.md +++ b/ofrak_core/CHANGELOG.md @@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Changed - Change `FreeSpaceModifier` & `PartialFreeSpaceModifier` behavior: an optional stub that isn't free space can be provided and fill-bytes for free space can be specified. ([#409](https://github.com/redballoonsecurity/ofrak/pull/409)) - `Resource.flush_to_disk` method renamed to `Resource.flush_data_to_disk`. ([#373](https://github.com/redballoonsecurity/ofrak/pull/373)) +- `build_image.py` supports building Docker images with OFRAK packages from any ancestor directory. ([#425](https://github.com/redballoonsecurity/ofrak/pull/425)) ## [3.2.0](https://github.com/redballoonsecurity/ofrak/compare/ofrak-v3.1.0...ofrak-v3.2.0) ### Added diff --git a/ofrak_tutorial/Dockerstub b/ofrak_tutorial/Dockerstub index cd504da1a..deaec2896 100644 --- a/ofrak_tutorial/Dockerstub +++ b/ofrak_tutorial/Dockerstub @@ -1,4 +1,3 @@ -ARG OFRAK_DIR=. -COPY $OFRAK_DIR/mkdocs.yml /mkdocs.yml -COPY $OFRAK_DIR/docs /docs -COPY $OFRAK_DIR/examples /examples +COPY $PACKAGE_PATH/../mkdocs.yml /mkdocs.yml +COPY $PACKAGE_PATH/../docs /docs +COPY $PACKAGE_PATH/../examples /examples From af5bf505ec4ebfad58b0dbb0528dd97a87ae5607 Mon Sep 17 00:00:00 2001 From: dannyp303 Date: Wed, 28 Feb 2024 17:29:32 -0500 Subject: [PATCH 40/52] Refactor the GUI hex view and pane components (#427) --------- Co-authored-by: Dan Pesce Co-authored-by: Jacob Strieb <99368685+rbs-jacob@users.noreply.github.com> --- .../ofrak_angr_test/assets/hello.x64.elf | Bin 0 -> 75568 bytes frontend/src/App.svelte | 32 +- .../src/{views => hex}/ByteclassView.svelte | 39 +- .../src/{views => hex}/EntropyView.svelte | 39 +- frontend/src/{views => hex}/HexView.svelte | 361 ++++++++++-------- .../src/{utils => hex}/JumpToOffset.svelte | 29 +- .../src/{views => hex}/MagnitudeView.svelte | 41 +- frontend/src/hex/MinimapView.svelte | 47 +++ frontend/src/hex/stores.js | 3 + frontend/src/ofrak/remote_resource.js | 7 +- frontend/src/resource/ResourceTreeView.svelte | 7 +- frontend/src/stores.js | 2 + frontend/src/utils/Pane.svelte | 116 +----- frontend/src/utils/Script.svelte | 3 + frontend/src/utils/SearchBar.svelte | 8 +- frontend/src/views/AssemblyView.svelte | 3 + frontend/src/views/AttributesView.svelte | 39 +- frontend/src/views/ComponentsView.svelte | 70 ++-- frontend/src/views/RunScriptView.svelte | 4 + frontend/src/views/ScriptView.svelte | 1 + frontend/src/views/TextView.svelte | 3 + ofrak_core/CHANGELOG.md | 1 + 22 files changed, 417 insertions(+), 438 deletions(-) create mode 100755 disassemblers/ofrak_angr/ofrak_angr_test/assets/hello.x64.elf rename frontend/src/{views => hex}/ByteclassView.svelte (84%) rename frontend/src/{views => hex}/EntropyView.svelte (83%) rename frontend/src/{views => hex}/HexView.svelte (54%) rename frontend/src/{utils => hex}/JumpToOffset.svelte (56%) rename frontend/src/{views => hex}/MagnitudeView.svelte (83%) create mode 100644 frontend/src/hex/MinimapView.svelte create mode 100644 frontend/src/hex/stores.js diff --git a/disassemblers/ofrak_angr/ofrak_angr_test/assets/hello.x64.elf b/disassemblers/ofrak_angr/ofrak_angr_test/assets/hello.x64.elf new file mode 100755 index 0000000000000000000000000000000000000000..e45a08e4a95a30668f43eb530d46e3cb3c8ef0e9 GIT binary patch literal 75568 zcmeI2e{5UT702(hR_{2p$PVajS2pow12w&u_29Zl(I%h(^RAhF^y^21fzq3T7Ie1O1*R5yT`F# z-F4EW38e3pp3lAKo^$TKpLcVA*m-DRc$3dClq8?JU2*ATi>v9;6-Te0v*}UmRio-r z*Q#q&J=NFIBX(YSPP>T=k}W4cJ?y8#{6Wbhm%CaeQd(yj&~>#$&xfv7q{L-Er#)wM zY*QIgsim%4SyuY-7*$x?dt}7tI<{*meq3a~u(tQglRBJ8;lh37XSwF#mHkx5NhI5H z+V+;6_v*-9pvYX&vm=(r@W@i8o~esKmY_l00ck)1V8`;KmY_l;4?*F zbviZDy>@jv*^y3V3ez3a>$^L;*GBW%=o;r9x`2AiclE(7yVTo_=8dO@*FN;Zsm-ZB z`tF(R{bB13e2?TiAzx?Nh7;S%b(rR5aOf7)%&b6D<1<8N3|njJfGd~7V*yy6>N;k5s2 z$NuZ@JEC|de(e2?@nh%wapOe%)eCk@IYNVu;PpSa;c4oNDlFO+FTQo^Yczrrfp3z= z*nQmXpQpWdA7jIK@xgQPM+eT$INt>w*?rtm9y%QXfoaYY44-cG=7iUhr zLSY{rIA1&wKfIy&J*AvpojP`*RC02!R6vN`HGFu(+vFX)bEx=QY}ZinfupUi$w4jUx6P&6bDow*THMTh zT|Rl%%`DMn6?Q$B=`#N|%Sqw{bv^%H%Sq}#zEH|?`9A)Um{PzMYF3x)b@`w!%l8t_ zZ@pC4r0W-J>HJ2l^DO*2f2PIlpHG*?UVgLY-^e*Fa)p}K;}v~em!;qQ+cl>~S15^h zq2!LIUE7y*S-76j<%q7=?*BIt*6+dF;E7B0XRxpD)<}E5HIhnXB3+%)HPNn)?(20f zvZixQS7%pOr@E2ixoBNLKhx7`r?(wl{wK*gCm%F!CpqsU`uGVwwH$px=cj&OT?5ZP zpAqCqz{NiLTurC;@^Wn$8Qiif(${sfcY;MaChbDT+AwZq ztXyi8iizCl#D?kh-5uR)BOT+Bjh-MB&`*Ct%WLZWs;8#DuDotm#|bD|FRS(S<#n}M-=J!(M}AtR zWgV-wZ&b1#R_m`Qubb8SCMEkvwSJyDT4UYutNH3=O?^nozEd5inToQ)&eiw{@*8L8 z%98V|78UtGN^{b$XkDnWKKfOglKpGWn*V4Ps9Ni(UoBL#dcUc*zf#FQRIOj6!mYLX zzgR_T>dU)sWfS=*<^gSN^HCiv{5V60RG6wCg!a*qYIr@v=%l$fzoNFWw|5+L}-tTuvU#tI@8rP4U zI4$Z+A4=poSt}0jX^PtQ&Yu@ZYt)!Ozt#5MdHA}czj!{s>%_TuUhxv%O8lp2f8ZqC zcv`Yv)^XXec91?oryuV;TtfOV>4m==jm}*h$9M7HV}$i$R3UG=g#Hd4-`nmU(nl)% zB|Ay_7T+SZ*zXuwT#U@qfIzz<*e~k-nMe4 zJ!y{821|RJX(qGgcse_hNSjGJo6DPt!nEQID{a|UGTK?&!W>IwQf4BTOYAqTjGf!B z#&U^C%S;v~C-;*}MTs`(lwD;i+ns4{+8*0HU=D2QH%)S?GE6G7|EpVKn}_F(M$M@dxtSyRybkXiaZGdP=01%`Mf3Y7=}IMPtZhZdGll3#A(c*cq>@?{>m8!=%(!yuCKCAx6;1BXP&`-KIk#r7 zmCMsHrK(_39rZO49nac2H7>ZdhQ7$=uRW}IfOE~YYLS;;3xMsn6(DWy{xOY%fX z|Hb`pf_FmtQ`46f%E-A&OYe@zwv~ym5_~6=d!(G#M6Rqfro5~}qVA?lrGJlZS7byC zwe^S#J^4pou7{y(QWZb{mgdq7#b-6%(p_b-YUDZ^)2{u^+T+!wU;+DlpF z@5qTyDdI2Z!m@OumOsT$B%cF$?uftKKSZ9?{g?SAe)1k4B40KUf4QfK6gx6;lK6}K zD`_kJTeM|+`byg}V zFaFng^x`k_0@X{t(sWRtBfW9Gw)08h)p4AC+W*tapp^I8@_F2=V;8&4)job1|LZQ{ z?_c2hZ@r8^pX1rmJI?Le-}&mOB8D4vuFqVSzr6}Gb@xKoGOj0XgetiWd7rYb^7 - + {#if useAssemblyView} {:else if useTextView} @@ -227,25 +218,6 @@ Answer by running riddle.answer('your answer here') from the console.`); Named slot must be outside {#if} because of: https://github.com/sveltejs/svelte/issues/5604 --> - - - {#if carouselSelection === "Entropy"} - - {:else if carouselSelection === "Byteclass"} - - {:else if carouselSelection === "Magnitude"} - - {/if} -
- -
- {/await} diff --git a/frontend/src/views/ByteclassView.svelte b/frontend/src/hex/ByteclassView.svelte similarity index 84% rename from frontend/src/views/ByteclassView.svelte rename to frontend/src/hex/ByteclassView.svelte index 693738ef3..a2bd2cf35 100644 --- a/frontend/src/views/ByteclassView.svelte +++ b/frontend/src/hex/ByteclassView.svelte @@ -28,11 +28,12 @@ import LoadingTextVertical from "../utils/LoadingTextVertical.svelte"; import { hexToByteArray } from "../helpers.js"; + import { screenHeight } from "./stores.js"; import { selectedResource, settings } from "../stores.js"; import { onMount } from "svelte"; - export let scrollY; + export let dataLength, currentPosition; let data = undefined; $: colorArray = [ @@ -124,17 +125,13 @@ context.strokeStyle = "red"; context.lineWidth = Math.ceil(canvas.height / 512); - if ( - data !== undefined && - data.length > alignment * 3 && - $scrollY.viewHeight !== 1 - ) { + if (data !== undefined && data.length > alignment * 3) { // Offset Y by 0.5 because of: https://stackoverflow.com/a/48970774 context.strokeRect( 0, - Math.ceil($scrollY.top * canvas.height) - 0.5, + Math.ceil((currentPosition / dataLength) * canvas.height) - 0.5, alignment, - Math.ceil(($scrollY.viewHeight * canvas.height) / 2) + Math.ceil(($screenHeight / dataLength) * canvas.height) ); } } @@ -144,11 +141,11 @@ diff --git a/frontend/src/views/EntropyView.svelte b/frontend/src/hex/EntropyView.svelte similarity index 83% rename from frontend/src/views/EntropyView.svelte rename to frontend/src/hex/EntropyView.svelte index b3819c46f..66e89d6d0 100644 --- a/frontend/src/views/EntropyView.svelte +++ b/frontend/src/hex/EntropyView.svelte @@ -27,12 +27,12 @@ -{#await dataLenPromise} - -{:then dataLength} - {#if dataLength > 0} - -
-
-
- -
- -
- {#await chunkDataPromise} - - {:then chunks} -
- {#each chunks as _, chunkIndex} + +
+ +
+ +
+
+ {#await dataLenPromise} + + {:then dataLength} + {#if dataLength > 0} + +
+
+
+ {#await chunkDataPromise} + + {:then chunks}
- {(chunkIndex * alignment + start) - .toString(16) - .padStart(8, "0") + ": "} + {#each chunks as _, chunkIndex} +
+ {(chunkIndex * alignment + start) + .toString(16) + .padStart(8, "0") + ": "} +
+ {/each}
- {/each} -
- + - {#await childRangesPromise} -
- {#each chunks as hexes} + {#await childRangesPromise}
- {#each hexes as byte} - {byte} + {#each chunks as hexes} +
+ {#each hexes as byte} + {byte} + {/each} +
{/each}
- {/each} -
- {:then childRangesResult} -
- {#each chunks as hexes, chunkIndex} + {:then childRangesResult}
- {#each hexes as byte, byteIndex} - {@const rangeInfo = getRangeInfo( - chunkIndex * alignment + byteIndex + start, - childRangesResult, - byte - )} - {byte} + {#each chunks as hexes, chunkIndex} +
+ {#each hexes as byte, byteIndex} + {@const rangeInfo = getRangeInfo( + chunkIndex * alignment + byteIndex + start, + childRangesResult, + byte + )} + + {byte} + {/each} +
{/each}
- {/each} -
- {/await} + {/await} - + -
- {#each chunks as hexes} -
- {hexes.map(hexToChar).join("") + " "} +
+ {#each chunks as hexes} +
+ {hexes.map(hexToChar).join("") + " "} +
+ {/each}
- {/each} + {/await}
- {/await} +
-
+ {:else} + Resource has no data! + {/if} + {/await} +
+ {#if resourceData != undefined} +
+
- {:else} -
- -
- - Resource has no data! {/if} -{/await} +
diff --git a/frontend/src/utils/JumpToOffset.svelte b/frontend/src/hex/JumpToOffset.svelte similarity index 56% rename from frontend/src/utils/JumpToOffset.svelte rename to frontend/src/hex/JumpToOffset.svelte index 170b5a071..c139dc7e9 100644 --- a/frontend/src/utils/JumpToOffset.svelte +++ b/frontend/src/hex/JumpToOffset.svelte @@ -16,21 +16,13 @@ diff --git a/frontend/src/views/MagnitudeView.svelte b/frontend/src/hex/MagnitudeView.svelte similarity index 83% rename from frontend/src/views/MagnitudeView.svelte rename to frontend/src/hex/MagnitudeView.svelte index 8d52291bd..73119056a 100644 --- a/frontend/src/views/MagnitudeView.svelte +++ b/frontend/src/hex/MagnitudeView.svelte @@ -26,13 +26,14 @@ + +
+ + {#if carouselSelection === "Entropy"} + + {:else if carouselSelection === "Byteclass"} + + {:else if carouselSelection === "Magnitude"} + + {/if} +
+ +
+
diff --git a/frontend/src/hex/stores.js b/frontend/src/hex/stores.js new file mode 100644 index 000000000..c47596e02 --- /dev/null +++ b/frontend/src/hex/stores.js @@ -0,0 +1,3 @@ +import { writable } from "svelte/store"; + +export const screenHeight = writable(0); diff --git a/frontend/src/ofrak/remote_resource.js b/frontend/src/ofrak/remote_resource.js index 035b94b28..da690e14b 100644 --- a/frontend/src/ofrak/remote_resource.js +++ b/frontend/src/ofrak/remote_resource.js @@ -170,10 +170,9 @@ export class RemoteResource extends Resource { if (this.data_id === null) { return null; } - let result = await fetch(`${this.uri}/get_data_length`).then((r) => - r.json() - ); - return result; + return await fetch(`${this.uri}/get_data_length`).then(async (r) => { + return await r.json(); + }); } async get_data_range_within_parent() { diff --git a/frontend/src/resource/ResourceTreeView.svelte b/frontend/src/resource/ResourceTreeView.svelte index 50eb12ab7..def1d0b99 100644 --- a/frontend/src/resource/ResourceTreeView.svelte +++ b/frontend/src/resource/ResourceTreeView.svelte @@ -28,12 +28,12 @@ .resources { flex-grow: 1; + overflow: auto; } .treebox { flex-grow: 1; padding-left: 1em; - overflow-x: auto; white-space: nowrap; text-align: left; } @@ -41,7 +41,12 @@ .searchbar { flex-grow: 1; padding-left: 1em; + padding-right: 1em; padding-bottom: 0.5em; + position: sticky; + top: 0; + background-color: var(--main-bg-color); + z-index: 10; } diff --git a/frontend/src/stores.js b/frontend/src/stores.js index dd6973e44..0af6397d0 100644 --- a/frontend/src/stores.js +++ b/frontend/src/stores.js @@ -49,3 +49,5 @@ export function loadSettings(forceReset) { } export let settings = writable(loadSettings()); + +export let resourceNodeDataMap = writable({}); diff --git a/frontend/src/utils/Pane.svelte b/frontend/src/utils/Pane.svelte index ca56fdf04..2a37d0c43 100644 --- a/frontend/src/utils/Pane.svelte +++ b/frontend/src/utils/Pane.svelte @@ -10,117 +10,31 @@ flex-flow: row; justify-content: space-between; align-items: stretch; + overflow: hidden; } .inner { - overflow: auto; + overflow: hidden; flex-grow: 1; } - - .minimap-container { - background: var(--main-bg-color); - display: flex; - z-index: 1; - } - - .minimap { - width: 64px; - max-width: 64px; - display: flex; - flex-flow: column; - justify-content: space-between; - align-items: center; - } - -
- {#if scrollY !== undefined} -
- -
- {:else} -
- -
- {/if} - - {#if displayMinimap} -
-
- -
-
- {/if} +
+ +
diff --git a/frontend/src/utils/Script.svelte b/frontend/src/utils/Script.svelte index a618c2afa..a51914121 100644 --- a/frontend/src/utils/Script.svelte +++ b/frontend/src/utils/Script.svelte @@ -7,6 +7,9 @@ align-items: stretch; line-height: var(--line-height); font-size: var(--font-size, medium); + overflow: auto; + max-height: 100%; + min-height: 100%; } .spacer { diff --git a/frontend/src/utils/SearchBar.svelte b/frontend/src/utils/SearchBar.svelte index da674b2fb..69d3880cf 100644 --- a/frontend/src/utils/SearchBar.svelte +++ b/frontend/src/utils/SearchBar.svelte @@ -41,6 +41,7 @@ width: 100%; padding-bottom: 1em; min-height: 2.25em; + background-color: var(--main-bg-color); } .resultwidgets { @@ -93,6 +94,9 @@ prevOptions = {}; function nextMatch() { + if (searchResults.matches === undefined) { + return searchResults; + } let nextIndex = searchResults.index + 1; if (nextIndex >= searchResults.matches.length) { nextIndex = 0; @@ -113,7 +117,8 @@ searchQuery == prevQuery && searchOptions.searchType == prevOptions.searchType && searchOptions.regex == prevOptions.regex && - searchOptions.caseIgnore == prevOptions.caseIgnore + searchOptions.caseIgnore == prevOptions.caseIgnore && + searchResults.matches != undefined // Search results are the only attribute of the search we have control of from other components, so we use it to clear the search when changing selected resources. ); } @@ -163,6 +168,7 @@ {/each} +
{cleanOfrakType(tag)}{i !== resource.get_tags().length - 1 ? ", " : ""} - {/each} - -

Attributes:

- -{/if} +
+ {#if resource !== undefined} +

Tags:

+ {#each resource.get_tags() as tag, i} + {cleanOfrakType(tag)}{i !== resource.get_tags().length - 1 ? ", " : ""} + {/each} +

Attributes:

+ + {/if} +
diff --git a/frontend/src/views/ComponentsView.svelte b/frontend/src/views/ComponentsView.svelte index 9229efa58..185f56786 100644 --- a/frontend/src/views/ComponentsView.svelte +++ b/frontend/src/views/ComponentsView.svelte @@ -1,12 +1,19 @@ diff --git a/frontend/src/views/TextView.svelte b/frontend/src/views/TextView.svelte index bb30c0382..e732a801a 100644 --- a/frontend/src/views/TextView.svelte +++ b/frontend/src/views/TextView.svelte @@ -14,6 +14,9 @@ align-items: stretch; line-height: var(--line-height); font-size: 0.95em; + min-height: calc(100% - 6em); + max-height: calc(100% - 6em); + overflow: auto; } .spacer { diff --git a/ofrak_core/CHANGELOG.md b/ofrak_core/CHANGELOG.md index 77d9b6513..b94c65309 100644 --- a/ofrak_core/CHANGELOG.md +++ b/ofrak_core/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master) ### Added +- Refactor HexView and related components to use mousewheel instead of scroll and compartmentalize all comonents to src/hex. ([#427](https://github.com/redballoonsecurity/ofrak/pull/427)) - Add an improved ISO9660 packer that leverages `mkisofs` instead of PyCdLib. ([#393](https://github.com/redballoonsecurity/ofrak/pull/393)) - Add UEFI binary unpacker. ([#399](https://github.com/redballoonsecurity/ofrak/pull/399)) From db3abb1dd17b1f32ab11a3ff5301e1e3047fd15f Mon Sep 17 00:00:00 2001 From: rbs-afflitto <74425743+rbs-afflitto@users.noreply.github.com> Date: Wed, 28 Feb 2024 18:46:12 -0500 Subject: [PATCH 41/52] Revert using SUBALIGN(0) for .bss sections (#431) * Remove SUBALIGN(0) from bss linker script * Update changelog --- ofrak_patch_maker/CHANGELOG.md | 3 +++ ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ofrak_patch_maker/CHANGELOG.md b/ofrak_patch_maker/CHANGELOG.md index f6cc95406..fe2f9b5ce 100644 --- a/ofrak_patch_maker/CHANGELOG.md +++ b/ofrak_patch_maker/CHANGELOG.md @@ -14,6 +14,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - X86_64 toolchain now installs on Docker image builds for AARCH64 hosts. ([#405](https://github.com/redballoonsecurity/ofrak/pull/405)) - Toolchain now drops the .altinstrs_replacement as well as the .altinstructions section in our generated linker scripts ([#414](https://github.com/redballoonsecurity/ofrak/pull/414)) +### Changed +- Removed `SUBALIGN(0)` for `.bss` sections + ## [4.0.2](https://github.com/redballoonsecurity/ofrak/compare/ofrak-patch-maker-v.4.0.1...ofrak-patch-maker-v.4.0.2) ### Fixed - Remove option to read or install toolchain.conf from/to "/etc/toolchain.conf" ([#342](https://github.com/redballoonsecurity/ofrak/pull/342)) diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu.py b/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu.py index 51ca0b2fd..93120b515 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu.py +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu.py @@ -185,7 +185,7 @@ def ld_generate_bss_section( ) -> str: bss_section_name = ".bss" return ( - f" {bss_section_name} : SUBALIGN(0) {{\n" + f" {bss_section_name} : {{\n" f" *.o({bss_section_name}, {bss_section_name}.*)\n" f" }} > {memory_region_name}" ) From 59c4387e5c8ad866a2d24e91a9ad362cd1eea3dc Mon Sep 17 00:00:00 2001 From: dannyp303 Date: Thu, 29 Feb 2024 14:36:46 -0500 Subject: [PATCH 42/52] move resourceTreeNode and dataLength to stores (#434) --------- Co-authored-by: Dan Pesce --- frontend/src/App.svelte | 22 +- frontend/src/hex/ByteclassView.svelte | 13 +- frontend/src/hex/EntropyView.svelte | 12 +- frontend/src/hex/HexView.svelte | 194 +++++++++--------- frontend/src/hex/MagnitudeView.svelte | 13 +- frontend/src/hex/MinimapView.svelte | 17 +- frontend/src/resource/ResourceTreeNode.svelte | 58 +++--- .../src/resource/ResourceTreeToolbar.svelte | 67 +++--- frontend/src/resource/ResourceTreeView.svelte | 3 - frontend/src/stores.js | 2 + frontend/src/views/AddTagView.svelte | 13 +- frontend/src/views/BinaryModifyView.svelte | 26 +-- frontend/src/views/CarveView.svelte | 23 ++- frontend/src/views/CommentView.svelte | 25 ++- frontend/src/views/ComponentsView.svelte | 22 +- frontend/src/views/FindReplaceView.svelte | 14 +- frontend/src/views/RunScriptView.svelte | 17 +- frontend/src/views/SearchView.svelte | 12 +- frontend/src/views/StartView.svelte | 20 +- 19 files changed, 280 insertions(+), 293 deletions(-) diff --git a/frontend/src/App.svelte b/frontend/src/App.svelte index d64a5e33e..3cdb45b33 100644 --- a/frontend/src/App.svelte +++ b/frontend/src/App.svelte @@ -43,7 +43,12 @@ import ProjectManagerView from "./project/ProjectManagerView.svelte"; import { printConsoleArt } from "./console-art.js"; - import { selected, selectedResource, settings } from "./stores.js"; + import { + selected, + selectedResource, + settings, + dataLength, + } from "./stores.js"; import { keyEventToString, shortcuts } from "./keyboard.js"; import { writable } from "svelte/store"; @@ -57,13 +62,8 @@ useAssemblyView = false, useTextView = false, rootResourceLoadPromise = new Promise((resolve) => {}), - resourceNodeDataMap = {}, resources = {}; - let carouselSelection, - currentResource, - rootResource, - modifierView, - bottomLeftPane; + let currentResource, rootResource, modifierView, bottomLeftPane; // TODO: Move to settings let riddleAnswered = JSON.parse(window.localStorage.getItem("riddleSolved")); @@ -71,6 +71,10 @@ riddleAnswered = false; } + $: dataLenPromise.then((r) => { + $dataLength = r; + }); + $: if ($selected !== undefined) { currentResource = resources[$selected]; if (currentResource === undefined) { @@ -177,13 +181,11 @@ Answer by running riddle.answer('your answer here') from the console.`); this="{modifierView}" dataLenPromise="{dataLenPromise}" bind:modifierView="{modifierView}" - bind:resourceNodeDataMap="{resourceNodeDataMap}" /> {:else} {/if} -
-
-
- {#await chunkDataPromise} - - {:then chunks} + {#if $dataLength > 0} + +
+
+
+
+ {#each chunks as _, chunkIndex}
- {#each chunks as _, chunkIndex} -
- {(chunkIndex * alignment + start) - .toString(16) - .padStart(8, "0") + ": "} -
- {/each} + {(chunkIndex * alignment + start) + .toString(16) + .padStart(8, "0") + ": "}
+ {/each} +
- + - {#await childRangesPromise} + {#await childRangesPromise} +
+ {#each chunks as hexes}
- {#each chunks as hexes} -
- {#each hexes as byte} - {byte} - {/each} -
+ {#each hexes as byte} + {byte} {/each}
- {:then childRangesResult} + {/each} +
+ {:then childRangesResult} +
+ {#each chunks as hexes, chunkIndex}
- {#each chunks as hexes, chunkIndex} -
- {#each hexes as byte, byteIndex} - {@const rangeInfo = getRangeInfo( - chunkIndex * alignment + byteIndex + start, - childRangesResult, - byte - )} - - {byte} - {/each} -
+ {#each hexes as byte, byteIndex} + {@const rangeInfo = getRangeInfo( + chunkIndex * alignment + byteIndex + start, + childRangesResult, + byte + )} + + {byte} {/each}
- {/await} + {/each} +
+ {/await} - + -
- {#each chunks as hexes} -
- {hexes.map(hexToChar).join("") + " "} -
- {/each} +
+ {#each chunks as hexes} +
+ {hexes.map(hexToChar).join("") + " "}
- {/await} + {/each}
- {:else} - Resource has no data! - {/if} - {/await} +
+ {:else} + Resource has no data! + {/if}
{#if resourceData != undefined}
- +
{/if}
diff --git a/frontend/src/hex/MagnitudeView.svelte b/frontend/src/hex/MagnitudeView.svelte index 73119056a..ccf322e61 100644 --- a/frontend/src/hex/MagnitudeView.svelte +++ b/frontend/src/hex/MagnitudeView.svelte @@ -28,11 +28,11 @@ import LoadingTextVertical from "../utils/LoadingTextVertical.svelte"; import { screenHeight } from "./stores.js"; import { hexToByteArray } from "../helpers.js"; - import { selectedResource, settings } from "../stores.js"; + import { selectedResource, settings, dataLength } from "../stores.js"; import { onMount } from "svelte"; - export let dataLength, currentPosition; + export let currentPosition; let data = undefined; @@ -112,9 +112,9 @@ // Offset Y by 0.5 because of: https://stackoverflow.com/a/48970774 context.strokeRect( 0, - Math.ceil((currentPosition / dataLength) * canvas.height) - 0.5, + Math.ceil((currentPosition / $dataLength) * canvas.height) - 0.5, alignment, - Math.ceil(($screenHeight / dataLength) * canvas.height) + Math.ceil(($screenHeight / $dataLength) * canvas.height) ); } @@ -131,7 +131,8 @@ on:mousedown="{(e) => { currentPosition = Math.floor( - Math.floor(dataLength * (e.offsetY / canvas.offsetHeight)) / alignment + Math.floor($dataLength * (e.offsetY / canvas.offsetHeight)) / + alignment ) * alignment; clicking = true; }}" @@ -145,7 +146,7 @@ if (clicking) { currentPosition = Math.floor( - Math.floor(dataLength * (e.offsetY / canvas.offsetHeight)) / + Math.floor($dataLength * (e.offsetY / canvas.offsetHeight)) / alignment ) * alignment; clicking = true; diff --git a/frontend/src/hex/MinimapView.svelte b/frontend/src/hex/MinimapView.svelte index 6db16e5b3..c6d3c9d76 100644 --- a/frontend/src/hex/MinimapView.svelte +++ b/frontend/src/hex/MinimapView.svelte @@ -16,27 +16,18 @@ import ByteclassView from "./ByteclassView.svelte"; import MagnitudeView from "./MagnitudeView.svelte"; import CarouselSelector from "../utils/CarouselSelector.svelte"; - export let dataLength, currentPosition; + export let currentPosition; let carouselSelection;
{#if carouselSelection === "Entropy"} - + {:else if carouselSelection === "Byteclass"} - + {:else if carouselSelection === "Magnitude"} - + {/if}
{}, selectPreviousSibling = () => {}, collapsed = true, @@ -131,31 +130,31 @@ kiddoChunksize = 512; $: { - if (resourceNodeDataMap[self_id] === undefined) { - resourceNodeDataMap[self_id] = {}; + if ($resourceNodeDataMap[self_id] === undefined) { + $resourceNodeDataMap[self_id] = {}; } - if (resourceNodeDataMap[self_id].collapsed === undefined) { - resourceNodeDataMap[self_id].collapsed = collapsed; + if ($resourceNodeDataMap[self_id].collapsed === undefined) { + $resourceNodeDataMap[self_id].collapsed = collapsed; } - if (resourceNodeDataMap[self_id].childrenPromise === undefined) { - resourceNodeDataMap[self_id].childrenPromise = + if ($resourceNodeDataMap[self_id].childrenPromise === undefined) { + $resourceNodeDataMap[self_id].childrenPromise = rootResource.get_children(); } - if (resourceNodeDataMap[self_id].commentsPromise === undefined) { - resourceNodeDataMap[self_id].commentsPromise = + if ($resourceNodeDataMap[self_id].commentsPromise === undefined) { + $resourceNodeDataMap[self_id].commentsPromise = rootResource.get_comments(); } - if (resourceNodeDataMap[self_id].lastModified === undefined) { - resourceNodeDataMap[self_id].lastModified = false; + if ($resourceNodeDataMap[self_id].lastModified === undefined) { + $resourceNodeDataMap[self_id].lastModified = false; } - if (resourceNodeDataMap[self_id].allModified === undefined) { - resourceNodeDataMap[self_id].allModified = false; + if ($resourceNodeDataMap[self_id].allModified === undefined) { + $resourceNodeDataMap[self_id].allModified = false; } - childrenPromise = resourceNodeDataMap[self_id].childrenPromise; - commentsPromise = resourceNodeDataMap[self_id].commentsPromise; - collapsed = resourceNodeDataMap[self_id].collapsed; - lastModified = resourceNodeDataMap[self_id].lastModified; - allModified = resourceNodeDataMap[self_id].allModified; + childrenPromise = $resourceNodeDataMap[self_id].childrenPromise; + commentsPromise = $resourceNodeDataMap[self_id].commentsPromise; + collapsed = $resourceNodeDataMap[self_id].collapsed; + lastModified = $resourceNodeDataMap[self_id].lastModified; + allModified = $resourceNodeDataMap[self_id].allModified; } function updateRootModel() { rootResource.update(); @@ -171,10 +170,10 @@ $: if ($selected === self_id) { shortcuts["h"] = () => { - resourceNodeDataMap[self_id].collapsed = true; + $resourceNodeDataMap[self_id].collapsed = true; }; shortcuts["l"] = () => { - resourceNodeDataMap[self_id].collapsed = false; + $resourceNodeDataMap[self_id].collapsed = false; }; shortcuts["j"] = () => { if (!collapsed && firstChild) { @@ -204,7 +203,7 @@ } function onDoubleClick(e) { - resourceNodeDataMap[self_id].collapsed = !collapsed; + $resourceNodeDataMap[self_id].collapsed = !collapsed; // Expand children recursively on double click if (!collapsed) { childrenCollapsed = false; @@ -217,15 +216,15 @@ // As a side effect, the corresponding resource gets selected. $selected = self_id; await rootResource.delete_comment(optional_range); - resourceNodeDataMap[$selected].commentsPromise = + $resourceNodeDataMap[$selected].commentsPromise = rootResource.get_comments(); } // Swap "just modified" indication to "previously modified" indication onDestroy(() => { - if (resourceNodeDataMap[self_id].lastModified) { - resourceNodeDataMap[self_id].allModified = - resourceNodeDataMap[self_id].lastModified; + if ($resourceNodeDataMap[self_id].lastModified) { + $resourceNodeDataMap[self_id].allModified = + $resourceNodeDataMap[self_id].lastModified; } }); @@ -235,7 +234,7 @@ {#if children?.length > 0}
diff --git a/frontend/src/resource/ResourceTreeToolbar.svelte b/frontend/src/resource/ResourceTreeToolbar.svelte index 4b4439fb0..16aef12f5 100644 --- a/frontend/src/resource/ResourceTreeToolbar.svelte +++ b/frontend/src/resource/ResourceTreeToolbar.svelte @@ -15,13 +15,10 @@ selected, settings, selectedProject, + resourceNodeDataMap, } from "../stores.js"; - export let resourceNodeDataMap, - modifierView, - bottomLeftPane, - showProjectManager, - showRootResource; + export let modifierView, bottomLeftPane, showProjectManager, showRootResource; $: rootResource = $selectedResource; function refreshResource() { @@ -80,12 +77,12 @@ shortcut: "i", onclick: async (e) => { await rootResource.identify(); - if (!resourceNodeDataMap[$selected]) { - resourceNodeDataMap[$selected] = {}; + if (!$resourceNodeDataMap[$selected]) { + $resourceNodeDataMap[$selected] = {}; } - resourceNodeDataMap[$selected].collapsed = - !!resourceNodeDataMap[$selected]?.collapsed; - resourceNodeDataMap[$selected].childrenPromise = + $resourceNodeDataMap[$selected].collapsed = + !!$resourceNodeDataMap[$selected]?.collapsed; + $resourceNodeDataMap[$selected].childrenPromise = rootResource.get_children(); refreshResource(); }, @@ -97,11 +94,11 @@ shortcut: "u", onclick: async (e) => { await rootResource.unpack(); - if (!resourceNodeDataMap[$selected]) { - resourceNodeDataMap[$selected] = {}; + if (!$resourceNodeDataMap[$selected]) { + $resourceNodeDataMap[$selected] = {}; } - resourceNodeDataMap[$selected].collapsed = false; - resourceNodeDataMap[$selected].childrenPromise = + $resourceNodeDataMap[$selected].collapsed = false; + $resourceNodeDataMap[$selected].childrenPromise = rootResource.get_children(); refreshResource(); }, @@ -121,12 +118,12 @@ shortcut: "a", onclick: async (e) => { await rootResource.analyze(); - if (!resourceNodeDataMap[$selected]) { - resourceNodeDataMap[$selected] = {}; + if (!$resourceNodeDataMap[$selected]) { + $resourceNodeDataMap[$selected] = {}; } - resourceNodeDataMap[$selected].collapsed = - !!resourceNodeDataMap[$selected]?.collapsed; - resourceNodeDataMap[$selected].childrenPromise = + $resourceNodeDataMap[$selected].collapsed = + !!$resourceNodeDataMap[$selected]?.collapsed; + $resourceNodeDataMap[$selected].childrenPromise = rootResource.get_children(); refreshResource(); }, @@ -148,11 +145,11 @@ const descendants = await $selectedResource.get_descendants(); clearModified(descendants); await rootResource.pack(); - if (!resourceNodeDataMap[$selected]) { - resourceNodeDataMap[$selected] = {}; + if (!$resourceNodeDataMap[$selected]) { + $resourceNodeDataMap[$selected] = {}; } - resourceNodeDataMap[$selected].collapsed = false; - resourceNodeDataMap[$selected].childrenPromise = + $resourceNodeDataMap[$selected].collapsed = false; + $resourceNodeDataMap[$selected].childrenPromise = rootResource.get_children(); refreshResource(); }, @@ -244,11 +241,11 @@ shortcut: "u+Shift", onclick: async (e) => { await rootResource.unpack_recursively(); - if (!resourceNodeDataMap[$selected]) { - resourceNodeDataMap[$selected] = {}; + if (!$resourceNodeDataMap[$selected]) { + $resourceNodeDataMap[$selected] = {}; } - resourceNodeDataMap[$selected].collapsed = false; - resourceNodeDataMap[$selected].childrenPromise = + $resourceNodeDataMap[$selected].collapsed = false; + $resourceNodeDataMap[$selected].childrenPromise = rootResource.get_children(); refreshResource(); }, @@ -262,11 +259,11 @@ const descendants = await $selectedResource.get_descendants(); clearModified(descendants); await rootResource.pack_recursively(); - if (!resourceNodeDataMap[$selected]) { - resourceNodeDataMap[$selected] = {}; + if (!$resourceNodeDataMap[$selected]) { + $resourceNodeDataMap[$selected] = {}; } - resourceNodeDataMap[$selected].collapsed = false; - resourceNodeDataMap[$selected].childrenPromise = + $resourceNodeDataMap[$selected].collapsed = false; + $resourceNodeDataMap[$selected].childrenPromise = rootResource.get_children(); refreshResource(); }, @@ -316,11 +313,11 @@ function clearModified(descendants) { for (const descendant of descendants) { - if (!resourceNodeDataMap[descendant["resource_id"]]) { - resourceNodeDataMap[descendant["resource_id"]] = {}; + if (!$resourceNodeDataMap[descendant["resource_id"]]) { + $resourceNodeDataMap[descendant["resource_id"]] = {}; } - resourceNodeDataMap[descendant["resource_id"]].lastModified = undefined; - resourceNodeDataMap[descendant["resource_id"]].allModified = undefined; + $resourceNodeDataMap[descendant["resource_id"]].lastModified = undefined; + $resourceNodeDataMap[descendant["resource_id"]].allModified = undefined; } } diff --git a/frontend/src/resource/ResourceTreeView.svelte b/frontend/src/resource/ResourceTreeView.svelte index def1d0b99..8c3416777 100644 --- a/frontend/src/resource/ResourceTreeView.svelte +++ b/frontend/src/resource/ResourceTreeView.svelte @@ -58,7 +58,6 @@ export let rootResource, modifierView, bottomLeftPane, - resourceNodeDataMap = {}, showProjectManager, showRootResource; @@ -81,7 +80,6 @@
diff --git a/frontend/src/stores.js b/frontend/src/stores.js index 0af6397d0..0cd75bc04 100644 --- a/frontend/src/stores.js +++ b/frontend/src/stores.js @@ -51,3 +51,5 @@ export function loadSettings(forceReset) { export let settings = writable(loadSettings()); export let resourceNodeDataMap = writable({}); + +export let dataLength = writable(undefined); diff --git a/frontend/src/views/AddTagView.svelte b/frontend/src/views/AddTagView.svelte index 9ec564cbd..51461472a 100644 --- a/frontend/src/views/AddTagView.svelte +++ b/frontend/src/views/AddTagView.svelte @@ -68,21 +68,26 @@