Add an ability to install OFRAK from source

.gitattributes

@@ -8,3 +8,4 @@ docs/user-guide/gui/assets/* filter=lfs diff=lfs merge=lfs -text
 ofrak_core/test_ofrak/components/assets/elf/* filter=lfs diff=lfs merge=lfs -text
 ofrak_core/test_ofrak/components/assets/elf/edge-cases/* filter=lfs diff=lfs merge=lfs -text
 frontend/public/themes/**/* filter=lfs diff=lfs merge=lfs -text
+disassemblers/ofrak_angr/ofrak_angr_test/assets/* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -3,6 +3,7 @@ venv/
 .idea/
 test/
 .DS_Store
+.coverage
 *.iml
 *.egg-info/
 QUESTIONS
@@ -17,4 +18,10 @@ ofrak_core/ofrak/core/entropy/entropy.so.1
 frontend/public/build
 ofrak_core/ofrak/core/entropy/entropy_c.cpython*
 ofrak_core/ofrak/gui/public
+ofrak_core/replaced_hello.out
+ofrak_core/replaced_simple_arm_gcc.o.elf
 ofrak_core/build
+ofrak_io/build/
+ofrak_patch_maker/build/
+ofrak_type/build/
+disassemblers/ofrak_ghidra/ofrak_ghidra/config/ofrak_ghidra.conf.yml

Makefile

@@ -18,3 +18,22 @@ tutorial-image:
 
 tutorial-run:
 	make -C ofrak_tutorial run
+
+OFRAK_INSTALL_PYTHON=python3
+
+.PHONY: install_tutorial install_core install_develop install_test_all
+install_tutorial:
+	$(OFRAK_INSTALL_PYTHON) -m pip install pyyaml
+	$(OFRAK_INSTALL_PYTHON) install.py --config ofrak-tutorial.yml --target install
+
+install_core:
+	$(OFRAK_INSTALL_PYTHON) -m pip install pyyaml
+	$(OFRAK_INSTALL_PYTHON) install.py --config ofrak-core-dev.yml --target install
+
+install_develop:
+	$(OFRAK_INSTALL_PYTHON) -m pip install pyyaml
+	$(OFRAK_INSTALL_PYTHON) install.py --config ofrak-dev.yml --target develop
+
+install_test_all:
+	$(OFRAK_INSTALL_PYTHON) -m pip install pyyaml
+	$(OFRAK_INSTALL_PYTHON) install.py --config ofrak-all.yml --target develop --test

build_image.py

@@ -192,10 +192,12 @@ def create_dockerfile_base(config: OfrakImageConfig) -> str:
             continue
         with open(dockerstage_path) as file_handle:
             dockerstub = file_handle.read()
+        # Cannot use ENV here because of multi-stage build FROM, so replace direclty in Docerkstage contents
+        dockerstub = dockerstub.replace("$PACKAGE_DIR", package_path)
         dockerfile_base_parts += [f"### {dockerstage_path}", dockerstub]
 
     dockerfile_base_parts += [
-        "FROM python:3.7-bullseye@sha256:338ead05c1a0aa8bd8fcba8e4dbbe2afd0283b4732fd30cf9b3bfcfcbc4affab",
+        "FROM python:3.8-bullseye@sha256:e1cd369204123e89646f8c001db830eddfe3e381bd5c837df00141be3bd754cb",
         "",
     ]
 
@@ -207,7 +209,11 @@ def create_dockerfile_base(config: OfrakImageConfig) -> str:
         dockerstub_path = os.path.join(package_path, "Dockerstub")
         with open(dockerstub_path) as file_handle:
             dockerstub = file_handle.read()
-        dockerfile_base_parts += [f"### {dockerstub_path}", dockerstub]
+        dockerfile_base_parts += [
+            f"### {dockerstub_path}",
+            f"ENV PACKAGE_PATH={package_path}",
+            dockerstub,
+        ]
         # Collect python dependencies
         python_reqs = []
         for suff in requirement_suffixes:
@@ -247,19 +253,24 @@ def create_dockerfile_finish(config: OfrakImageConfig) -> str:
         [
             "$INSTALL_TARGET:",
             "\\n\\\n".join(
-                [f"\tmake -C {package_name} $INSTALL_TARGET" for package_name in package_names]
+                [f"\t\\$(MAKE) -C {package_name} $INSTALL_TARGET" for package_name in package_names]
             ),
             "\\n",
         ]
     )
     dockerfile_finish_parts.append(f'RUN printf "{develop_makefile}" >> Makefile\n')
     dockerfile_finish_parts.append("RUN make $INSTALL_TARGET\n\n")
+    test_names = " ".join([f"test_{package_name}" for package_name in package_names])
     finish_makefile = "\\n\\\n".join(
         [
-            "test:",
-            "\\n\\\n".join([f"\tmake -C {package_name} test" for package_name in package_names]),
-            "\\n",
+            ".PHONY: test " + test_names,
+            "test: " + test_names,
+        ]
+        + [
+            f"test_{package_name}:\\n\\\n\t\\$(MAKE) -C {package_name} test"
+            for package_name in package_names
         ]
+        + ["\\n"]
     )
     dockerfile_finish_parts.append(f'RUN printf "{finish_makefile}" >> Makefile\n')
     if config.entrypoint is not None:

disassemblers/ofrak_angr/CHANGELOG.md

@@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master)
 
+### Changed
+- Update to latest angr==9.2.77, which also necessitates Python >= 3.8.
+
 ### Fixed
 - Add `importlib-resources` dependency as workaround for z3-solver dependency issue. ([#401](https://github.com/redballoonsecurity/ofrak/pull/401))
 

disassemblers/ofrak_angr/Makefile

@@ -11,7 +11,7 @@ develop:
 
 .PHONY: inspect
 inspect:
-	mypy
+	$(PYTHON) -m mypy
 
 .PHONY: test
 test: inspect

disassemblers/ofrak_angr/ofrak_angr/components/angr_decompilation_analyzer.py

@@ -0,0 +1,73 @@
+from dataclasses import dataclass
+import angr
+from angr.analyses.decompiler import Decompiler
+from ofrak.component.analyzer import Analyzer
+from ofrak.component.identifier import Identifier
+from ofrak.resource_view import ResourceView
+
+from ofrak.resource import Resource
+from ofrak.core.complex_block import ComplexBlock
+from ofrak.service.resource_service_i import ResourceFilter
+from ofrak_angr.model import AngrAnalysis, AngrAnalysisResource
+
+
+@dataclass
+class AngrDecompilationAnalysis(ResourceView):
+    decompilation: str
+
+
+class AngrDecompilationAnalysisIdentifier(Identifier):
+    id = b"AngrDecompilationAnalysisIdentifier"
+    targets = (ComplexBlock,)
+
+    async def identify(self, resource: Resource, config=None):
+        resource.add_tag(AngrDecompilationAnalysis)
+
+
+class AngrDecompilatonAnalyzer(Analyzer[None, AngrDecompilationAnalysis]):
+    id = b"AngrDecompilationAnalyzer"
+    targets = (ComplexBlock,)
+    outputs = (AngrDecompilationAnalysis,)
+
+    async def analyze(self, resource: Resource, config: None) -> AngrDecompilationAnalysis:
+        # Run / fetch angr analyzer
+        try:
+            root_resource = await resource.get_only_ancestor(
+                ResourceFilter(tags=[AngrAnalysisResource], include_self=True)
+            )
+            complex_block = await resource.view_as(ComplexBlock)
+            angr_analysis = await root_resource.analyze(AngrAnalysis)
+
+            cfg = angr_analysis.project.analyses[angr.analyses.CFGFast].prep()(
+                data_references=True, normalize=True
+            )
+
+            function_s = [
+                func
+                for addr, func in angr_analysis.project.kb.functions.items()
+                if func.addr == complex_block.virtual_address
+            ]
+            if len(function_s) == 0:
+                # Check for thumb
+                function_s = [
+                    func
+                    for addr, func in angr_analysis.project.kb.functions.items()
+                    if func.addr == complex_block.virtual_address + 1
+                ]
+            if len(function_s) != 1:
+                raise ValueError(
+                    f"Could not find angr function for function at address {complex_block.virtual_address}"
+                )
+            function = function_s[0]
+            dec: Decompiler = angr_analysis.project.analyses[angr.analyses.Decompiler].prep()(
+                function, cfg=cfg.model, options=None
+            )
+            if dec.codegen is not None:
+                decomp = dec.codegen.text
+            else:
+                decomp = "No Decompilation available"
+            return AngrDecompilationAnalysis(decomp)
+        except Exception as e:
+            return AngrDecompilationAnalysis(
+                f"The decompilation for this Complex Block has failed with the error {e}"
+            )

disassemblers/ofrak_angr/ofrak_angr/components/blocks/unpackers.py

@@ -228,7 +228,7 @@ def _angr_get_dword_blocks(
             if xref is None or not any(xref in bb_range for bb_range in valid_data_xref_ranges):
                 continue
 
-            LOGGER.debug(f"Creating DataWord for {cb_data_xref.content} @ {cb_data_xref_addr:#x}")
+            LOGGER.debug(f"Creating DataWord for {cb_data_xref.content!r} @ {cb_data_xref_addr:#x}")
 
             format_string = endian_flag + dword_size_map[word_size]
 

disassemblers/ofrak_angr/ofrak_angr_test/test_decompilation.py

@@ -0,0 +1,36 @@
+from typing import List
+import os
+from ofrak_angr.components.angr_decompilation_analyzer import AngrDecompilationAnalysis
+from ofrak.ofrak_context import OFRAKContext
+from ofrak.core.complex_block import ComplexBlock
+from ofrak.service.resource_service_i import ResourceFilter
+
+
+async def test_angr_decompilation(ofrak_context: OFRAKContext):
+    root_resource = await ofrak_context.create_root_resource_from_file(
+        os.path.join(os.path.dirname(__file__), "assets/hello.x64.elf")
+    )
+    await root_resource.unpack_recursively(
+        do_not_unpack=[
+            ComplexBlock,
+        ]
+    )
+    complex_blocks: List[ComplexBlock] = await root_resource.get_descendants_as_view(
+        ComplexBlock,
+        r_filter=ResourceFilter(
+            tags=[
+                ComplexBlock,
+            ]
+        ),
+    )
+    decomps = []
+    for complex_block in complex_blocks:
+        await complex_block.resource.identify()
+        angr_resource: AngrDecompilationAnalysis = await complex_block.resource.view_as(
+            AngrDecompilationAnalysis
+        )
+        decomps.append(angr_resource.decompilation)
+    assert len(decomps) == 11
+    assert "" not in decomps
+    assert "main" in " ".join(decomps)
+    assert "print" in " ".join(decomps)

disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py

@@ -54,14 +54,31 @@ async def expected_results(self, unpack_verify_test_case: ComplexBlockUnpackerTe
                 0x110,
                 0x110,
                 0x130,
-                keep_same_is_exit_point=True,
+                keep_same_is_exit_point=False,
             )
 
             return self._fixup_test_case_for_pie(
                 unpack_verify_test_case.expected_results,
                 pie_base_vaddr=0x400000,
             )
 
+        elif unpack_verify_test_case.binary_md5_digest == "c79d1bea0398d7a9d0faa1ba68786f5e":
+            # Unlike angr 9.2.6, angr 9.2.77 and 9.2.91 miss this DataWord now
+            # = the ref to it does not appear in the list of xrefs
+
+            missing_data_words = {0x8030, 0x8060}
+
+            fixed_up_results = {
+                vaddr: [
+                    block
+                    for block in original_expected_blocks
+                    if block.virtual_address not in missing_data_words
+                ]
+                for vaddr, original_expected_blocks in unpack_verify_test_case.expected_results.items()
+            }
+
+            return fixed_up_results
+
         return unpack_verify_test_case.expected_results
 
     def _split_bb(

disassemblers/ofrak_angr/requirements.txt

@@ -1,2 +1,2 @@
-angr==9.2.6
+angr==9.2.77
 importlib-resources  # A workaround for https://github.com/redballoonsecurity/ofrak/issues/398

disassemblers/ofrak_angr/setup.py

@@ -69,7 +69,7 @@ def read_requirements(requirements_path):
         "Topic :: Security",
         "Typing :: Typed",
     ],
-    python_requires=">=3.7",
+    python_requires=">=3.8",
     license="Proprietary",
     license_files=["LICENSE"],
     cmdclass={"egg_info": egg_info_ex},

disassemblers/ofrak_binary_ninja/Dockerstub

@@ -1,7 +1,6 @@
 RUN apt-get update && apt-get install -y libdbus-1-3
-ARG OFRAK_DIR=.
-COPY $OFRAK_DIR/disassemblers/ofrak_binary_ninja/install_binary_ninja_headless_linux.sh /tmp/
-COPY $OFRAK_DIR/disassemblers/ofrak_binary_ninja/pin_version.py /tmp/
+COPY $PACKAGE_PATH/install_binary_ninja_headless_linux.sh /tmp/
+COPY $PACKAGE_PATH/pin_version.py /tmp/
 RUN --mount=type=secret,id=serial --mount=type=secret,id=license.dat,dst=/root/.binaryninja/license.dat /tmp/install_binary_ninja_headless_linux.sh && \
   python3 /tmp/pin_version.py "3.2.3814 Headless" && \
   rm /tmp/install_binary_ninja_headless_linux.sh && \

disassemblers/ofrak_binary_ninja/Makefile

@@ -11,7 +11,7 @@ develop:
 
 .PHONY: inspect
 inspect:
-	mypy
+	$(PYTHON) -m mypy
 
 .PHONY: test
 test: inspect

disassemblers/ofrak_binary_ninja/install_binary_ninja_headless_linux.sh

@@ -5,12 +5,13 @@ else
     echo "Error: BinaryNinja license serial number not found." >&2 
     exit 1
 fi
+PACKAGE_NAME="binaryninja-headless.zip"
 INSTALL_DIR=/opt/rbs
 mkdir -p $INSTALL_DIR
 cd $INSTALL_DIR
 curl -O https://raw.githubusercontent.com/Vector35/binaryninja-api/dev/scripts/download_headless.py
 python3 -m pip --no-input install requests
-python3 download_headless.py --serial $SERIAL
-unzip BinaryNinja-headless.zip
-rm download_headless.py BinaryNinja-headless.zip
+python3 download_headless.py --serial $SERIAL --output "$PACKAGE_NAME"
+unzip "$PACKAGE_NAME"
+rm download_headless.py "$PACKAGE_NAME"
 python3 binaryninja/scripts/install_api.py

disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/binary_ninja_analyzer.py

@@ -2,11 +2,17 @@
 import tempfile
 from dataclasses import dataclass
 from typing import Optional, List
+from ofrak.component.abstract import ComponentMissingDependencyError
 
-from binaryninja import open_view, BinaryViewType
+try:
+    from binaryninja import open_view, BinaryViewType
+
+    BINJA_INSTALLED = True
+except ImportError:
+    BINJA_INSTALLED = False
 
 from ofrak.component.analyzer import Analyzer
-from ofrak.model.component_model import ComponentConfig
+from ofrak.model.component_model import ComponentConfig, ComponentExternalTool
 from ofrak.model.resource_model import ResourceAttributeDependency
 from ofrak_binary_ninja.components.identifiers import BinaryNinjaAnalysisResource
 from ofrak_binary_ninja.model import BinaryNinjaAnalysis
@@ -15,6 +21,21 @@
 LOGGER = logging.getLogger(__file__)
 
 
+class _BinjaExternalTool(ComponentExternalTool):
+    def __init__(self):
+        super().__init__(
+            "binary_ninja",
+            "https://ofrak.com/docs/user-guide/disassembler-backends/binary_ninja.html",
+            install_check_arg="",
+        )
+
+    async def is_tool_installed(self) -> bool:
+        return BINJA_INSTALLED
+
+
+BINJA_TOOL = _BinjaExternalTool()
+
+
 @dataclass
 class BinaryNinjaAnalyzerConfig(ComponentConfig):
     bndb_file: str  # Path to BinaryNinja DB pre-analyzed file
@@ -24,10 +45,13 @@ class BinaryNinjaAnalyzer(Analyzer[Optional[BinaryNinjaAnalyzerConfig], BinaryNi
     id = b"BinaryNinjaAnalyzer"
     targets = (BinaryNinjaAnalysisResource,)
     outputs = (BinaryNinjaAnalysis,)
+    external_dependencies = (BINJA_TOOL,)
 
     async def analyze(
         self, resource: Resource, config: Optional[BinaryNinjaAnalyzerConfig] = None
     ) -> BinaryNinjaAnalysis:
+        if not BINJA_INSTALLED:
+            raise ComponentMissingDependencyError(self, BINJA_TOOL)
         if not config:
             resource_data = await resource.get_data()
             temp_file = tempfile.NamedTemporaryFile()
@@ -36,9 +60,9 @@ async def analyze(
             bv = open_view(temp_file.name)
             return BinaryNinjaAnalysis(bv)
         else:
-            bv = BinaryViewType.get_view_of_file(config.bndb_file)
-            assert bv is not None
-            return BinaryNinjaAnalysis(bv)
+            opt_bv = BinaryViewType.get_view_of_file(config.bndb_file)
+            assert opt_bv is not None
+            return BinaryNinjaAnalysis(opt_bv)
 
     def _create_dependencies(
         self,