From b3c1af4603d6a01f24fa8c42086c3ecc44a13162 Mon Sep 17 00:00:00 2001 From: Paul Noalhyt Date: Mon, 26 Jun 2023 20:07:51 +0200 Subject: [PATCH 01/13] Support for PPC VLE as a sub-isa. Patchmaker support, plus automated tests. Ghidra analysis and unpackers support. --- .../components/blocks/bb_unpacker.py | 18 ++- .../components/ghidra_analyzer.py | 24 +++- .../ghidra_scripts/PreAnalyzePPCVLE.java | 51 +++++++ ofrak_core/ofrak/core/elf/analyzer.py | 3 +- ofrak_core/ofrak/core/elf/model.py | 25 +++- ofrak_patch_maker/Dockerstub | 13 ++ .../ofrak_patch_maker/toolchain.conf | 7 + .../ofrak_patch_maker/toolchain/abstract.py | 7 +- .../ofrak_patch_maker/toolchain/gnu_ppc.py | 83 ++++++++++- .../example_2/bom1.vle.S | 7 + .../example_2/bom2.vle.S | 7 + .../example_2/bom3.vle.S | 7 + .../test_alignment/patch_vle.as | 2 + .../test_ppc_vle_toolchain.py | 129 ++++++++++++++++++ .../test_symbol_parsing.py | 12 +- .../ofrak_patch_maker_test/toolchain_c.py | 4 + ofrak_type/ofrak_type/architecture.py | 1 + 17 files changed, 390 insertions(+), 10 deletions(-) create mode 100644 disassemblers/ofrak_ghidra/ofrak_ghidra/ghidra_scripts/PreAnalyzePPCVLE.java create mode 100644 ofrak_patch_maker/ofrak_patch_maker_test/example_2/bom1.vle.S create mode 100644 ofrak_patch_maker/ofrak_patch_maker_test/example_2/bom2.vle.S create mode 100644 ofrak_patch_maker/ofrak_patch_maker_test/example_2/bom3.vle.S create mode 100644 ofrak_patch_maker/ofrak_patch_maker_test/test_alignment/patch_vle.as create mode 100644 ofrak_patch_maker/ofrak_patch_maker_test/test_ppc_vle_toolchain.py diff --git a/disassemblers/ofrak_ghidra/ofrak_ghidra/components/blocks/bb_unpacker.py b/disassemblers/ofrak_ghidra/ofrak_ghidra/components/blocks/bb_unpacker.py index 9f2adfe88..15f9d04ac 100644 --- a/disassemblers/ofrak_ghidra/ofrak_ghidra/components/blocks/bb_unpacker.py +++ b/disassemblers/ofrak_ghidra/ofrak_ghidra/components/blocks/bb_unpacker.py @@ -5,7 +5,7 @@ from typing import Tuple, Dict, Union, List, Iterable from ofrak.core.architecture import ProgramAttributes -from ofrak_type.architecture import InstructionSet, InstructionSetMode +from ofrak_type.architecture import InstructionSet, InstructionSetMode, SubInstructionSet from ofrak.core.basic_block import BasicBlockUnpacker, BasicBlock from ofrak.core.instruction import Instruction from ofrak.resource import ResourceFactory, Resource @@ -174,6 +174,22 @@ def _asm_fixups( operand = re.sub(r"a([0-7])", r"%A\1", operand) operand = re.sub(r"d([0-7])[bw]?", r"%D\1", operand) operands += operand + elif program_attrs.sub_isa is SubInstructionSet.PPCVLE: + # in Ghidra, offsets from a register like in `se_stw r0,0x9(r1)` are expressed in words. + # so in this example r0 is stored at r1+0x9*4=r1+0x24 + # But it is more natural to express it in bytes, to get the instruction `se_stw r0,0x24(r1)` + # (this is also the convention used by the VLE assembler) + + def replace_offset(match): + new_operand = match.group(1) + new_operand += f"0x{int(match.group(2), 0)*4:x}" + new_operand += match.group(3) + return new_operand + + mnemonic = base_mnemonic + operands = re.sub( + r"(.*, )(0x[0-9]+)(\(r[0-9]+\))", lambda match: replace_offset(match), operands + ) else: mnemonic = base_mnemonic return mnemonic, operands diff --git a/disassemblers/ofrak_ghidra/ofrak_ghidra/components/ghidra_analyzer.py b/disassemblers/ofrak_ghidra/ofrak_ghidra/components/ghidra_analyzer.py index c596785ef..d95cb75c8 100644 --- a/disassemblers/ofrak_ghidra/ofrak_ghidra/components/ghidra_analyzer.py +++ b/disassemblers/ofrak_ghidra/ofrak_ghidra/components/ghidra_analyzer.py @@ -8,7 +8,7 @@ from typing import Optional, List from ofrak import ResourceFilter -from ofrak.core import CodeRegion +from ofrak.core import CodeRegion, ProgramAttributes, Elf, ElfUnpacker from ofrak.component.analyzer import Analyzer from ofrak.component.modifier import Modifier from ofrak.model.component_model import ComponentConfig @@ -33,6 +33,7 @@ OfrakGhidraMixin, GhidraComponentException, ) +from ofrak_type import InstructionSet, SubInstructionSet LOGGER = logging.getLogger(__name__) @@ -106,7 +107,16 @@ async def analyze( ghidra_project = f"{GHIDRA_REPOSITORY_HOST}:{GHIDRA_REPOSITORY_PORT}/ofrak" - program_name = await self._do_ghidra_import(ghidra_project, full_fname) + program_attributes = None + if resource.has_tag(Elf): + await resource.run(ElfUnpacker) + program_attributes = await resource.analyze(ProgramAttributes) + else: + logging.warning( + f"Could not get ProgramAttributes for resource {resource.get_id()} because it doesn't have the Elf tag." + ) + + program_name = await self._do_ghidra_import(ghidra_project, full_fname, program_attributes) await self._do_ghidra_analyze_and_serve( ghidra_project, program_name, skip_analysis=config is not None ) @@ -116,7 +126,7 @@ async def analyze( return GhidraProject(ghidra_project, f"{GHIDRA_SERVER_HOST}:{GHIDRA_SERVER_PORT}") - async def _do_ghidra_import(self, ghidra_project: str, full_fname: str): + async def _do_ghidra_import(self, ghidra_project: str, full_fname: str, program_attributes): args = [ ghidra_project, "-connect", @@ -127,6 +137,14 @@ async def _do_ghidra_import(self, ghidra_project: str, full_fname: str): "-overwrite", ] + if ( + program_attributes is not None + and program_attributes.isa == InstructionSet.PPC + and program_attributes.sub_isa == SubInstructionSet.PPCVLE + ): + args.extend(["-scriptPath", "'" + (";".join(self._script_directories)) + "'"]) + args.extend(["-preScript", "PreAnalyzePPCVLE.java"]) + cmd_str = " ".join([GHIDRA_HEADLESS_EXEC] + args) LOGGER.debug(f"Running command: {cmd_str}") ghidra_proc = await asyncio.create_subprocess_exec( diff --git a/disassemblers/ofrak_ghidra/ofrak_ghidra/ghidra_scripts/PreAnalyzePPCVLE.java b/disassemblers/ofrak_ghidra/ofrak_ghidra/ghidra_scripts/PreAnalyzePPCVLE.java new file mode 100644 index 000000000..472a9472d --- /dev/null +++ b/disassemblers/ofrak_ghidra/ofrak_ghidra/ghidra_scripts/PreAnalyzePPCVLE.java @@ -0,0 +1,51 @@ +import ghidra.app.script.GhidraScript; +import ghidra.program.model.mem.*; +import ghidra.program.model.lang.*; +import ghidra.program.model.pcode.*; +import ghidra.program.model.util.*; +import ghidra.program.model.reloc.*; +import ghidra.program.model.data.*; +import ghidra.program.model.block.*; +import ghidra.program.model.symbol.*; +import ghidra.program.model.scalar.*; +import ghidra.program.model.listing.*; +import ghidra.program.model.address.*; + +import java.math.BigInteger; + +public class PreAnalyzePPCVLE extends GhidraScript { + + @Override + public void run() throws Exception { + try { + // Set the language to PPC VLE + Language language = (Language) getLanguage(new LanguageID("PowerPC:BE:64:VLE-32addr")); + Program p = currentProgram; + p.setLanguage(language, language.getDefaultCompilerSpec().getCompilerSpecID(), false, monitor); + ProgramContext programContext = p.getProgramContext(); + // Set the vle bit (Ghidra has a "vle" register for that, but on real devices, the VLE + // bit is defined per memory page, as a page attribute bit) so that instructions are + // decoded correctly. + for (Register register : programContext.getContextRegisters()) { + if (register.getName().equals("vle")){ + RegisterValue newValue = new RegisterValue(programContext.getBaseContextRegister()); + BigInteger value = BigInteger.ONE; + newValue = setRegisterValue(newValue, register, value); + programContext.setDefaultDisassemblyContext(newValue); + println("Set the vle bit."); + } + } + } catch(Exception e) { + println(e.toString()); + e.printStackTrace(System.out); + throw e; + } + } + + private RegisterValue setRegisterValue(RegisterValue registerValue, Register register, + BigInteger value) { + RegisterValue newValue = new RegisterValue(register, value); + return registerValue.combineValues(newValue); + } + +} diff --git a/ofrak_core/ofrak/core/elf/analyzer.py b/ofrak_core/ofrak/core/elf/analyzer.py index c4f9a38cf..fccd44a79 100644 --- a/ofrak_core/ofrak/core/elf/analyzer.py +++ b/ofrak_core/ofrak/core/elf/analyzer.py @@ -370,6 +370,7 @@ class ElfProgramAttributesAnalyzer(Analyzer[None, ProgramAttributes]): async def analyze( self, resource: Resource, config: Optional[ComponentConfig] = None ) -> ProgramAttributes: + elf_resource = await resource.view_as(Elf) elf_header = await resource.get_only_descendant_as_view( ElfHeader, r_filter=ResourceFilter.with_tags(ElfHeader) ) @@ -379,7 +380,7 @@ async def analyze( return ProgramAttributes( elf_header.get_isa(), - None, + await elf_resource.get_sub_isa(), elf_basic_header.get_bitwidth(), elf_basic_header.get_endianness(), None, diff --git a/ofrak_core/ofrak/core/elf/model.py b/ofrak_core/ofrak/core/elf/model.py index 313fdb069..43e71a430 100644 --- a/ofrak_core/ofrak/core/elf/model.py +++ b/ofrak_core/ofrak/core/elf/model.py @@ -5,7 +5,7 @@ from ofrak.model.viewable_tag_model import AttributesType -from ofrak_type.architecture import InstructionSet +from ofrak_type.architecture import InstructionSet, SubInstructionSet from ofrak.core.program import Program from ofrak.core.program_section import NamedProgramSection, ProgramSegment from ofrak.model.resource_model import index @@ -863,5 +863,28 @@ async def get_program_header_by_index(self, index: int) -> ElfProgramHeader: ), ) + async def get_sub_isa(self) -> Optional[SubInstructionSet]: + elf_header = await self.get_header() + isa = elf_header.get_isa() + if isa == InstructionSet.PPC: + # We can detect whether the elf is PPC VLE by looking at the section header flags, as described here: https://reverseengineering.stackexchange.com/questions/20863/powerpc-elf32-detecting-vle + PF_PPC_VLE = 0x10000000 + SHF_PPC_VLE = 0x10000000 + ppc_vle = False + program_headers = await self.get_program_headers() + for program_header in program_headers: + if program_header.p_flags & PF_PPC_VLE != 0: + ppc_vle = True + break + if not ppc_vle: + section_headers = await self.get_section_headers() + for section_header in section_headers: + if section_header.sh_flags & SHF_PPC_VLE != 0: + ppc_vle = True + break + if ppc_vle: + return SubInstructionSet.PPCVLE + return None + MagicDescriptionIdentifier.register(Elf, lambda s: s.startswith("ELF ")) diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index c6d3ac340..f5cfac991 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -75,3 +75,16 @@ RUN apt-get update && apt-get install -y gcc-avr binutils-avr avr-libc RUN if [ "$TARGETARCH" = "amd64" ]; then \ apt-get update && apt-get install -y gcc-10-powerpc-linux-gnu; \ fi; + +#PPCVLE 4 NXP GCC Fork +# Download the toolchain into your OFRAK directory from here (requires sign up): +# https://www.nxp.com/design/software/development-software/s32-design-studio-ide/s32-design-studio-for-power-architecture:S32DS-PA +ARG OFRAK_DIR=. +COPY $OFRAK_DIR/gcc-4.9.4-Ee200-eabivle-x86_64-linux-g2724867.zip /tmp +RUN cd /tmp && \ + unzip gcc-4.9.4-Ee200-eabivle-x86_64-linux-g2724867.zip && \ + cd powerpc-eabivle-4_9 && \ + for f in *; do test -d "${f}" && cp -r "${f}" /usr/; done && \ + dpkg --add-architecture i386 && \ + apt-get update && \ + apt-get install -y libc6:i386 libncurses5:i386 libstdc++6:i386 lib32z1 diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf index bac54c64b..6d019d1fd 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf @@ -50,6 +50,12 @@ COMPILER = /usr/bin/powerpc-linux-gnu-gcc-10 LINKER = /usr/bin/powerpc-linux-gnu-ld BIN_PARSER = /usr/bin/powerpc-linux-gnu-objdump +[GNU_PPCVLE_4] +PREPROCESSOR = /usr/bin/powerpc-eabivle-gcc +COMPILER = /usr/bin/powerpc-eabivle-gccbloop +LINKER = /usr/bin/powerpc-eabivle-ld +BIN_PARSER = /usr/bin/powerpc-eabivle-objdump + [ASM] ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as X86_64_ASM_PATH = /opt/rbs/toolchain/binutils-2.34/gas/as-new @@ -57,3 +63,4 @@ M68K_ASM_PATH = /usr/bin/m68k-linux-gnu-as AARCH64_ASM_PATH = /usr/bin/aarch64-linux-gnu-as AVR_ASM_PATH = /usr/bin/avr-as PPC_ASM_PATH = /usr/bin/powerpc-linux-gnu-as +PPCVLE_ASM_PATH = /usr/bin/powerpc-eabivle-as diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py b/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py index 61553bb32..a14c07d72 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py @@ -13,7 +13,7 @@ from ofrak_patch_maker.binary_parser.abstract import AbstractBinaryFileParser from ofrak_patch_maker.toolchain.model import Segment, ToolchainConfig from ofrak_patch_maker.toolchain.utils import get_repository_config -from ofrak_type.architecture import InstructionSet +from ofrak_type.architecture import InstructionSet, SubInstructionSet from ofrak_type.bit_width import BitWidth from ofrak_type.memory_permissions import MemoryPermissions from ofrak_type.symbol_type import LinkableSymbolType @@ -156,6 +156,11 @@ def _assembler_path(self) -> str: and self._processor.bit_width == BitWidth.BIT_64 ): assembler_path = "X86_64_ASM_PATH" + elif ( + self._processor.isa == InstructionSet.PPC + and self._processor.sub_isa == SubInstructionSet.PPCVLE + ): + assembler_path = "PPCVLE_ASM_PATH" else: assembler_path = f"{self._processor.isa.value.upper()}_ASM_PATH" return get_repository_config("ASM", assembler_path) diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py b/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py index a92ef7dc3..079dd421f 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py @@ -2,9 +2,9 @@ from typing import Optional from ofrak_patch_maker.binary_parser.gnu import GNU_V10_ELF_Parser -from ofrak_patch_maker.toolchain.gnu import GNU_10_Toolchain +from ofrak_patch_maker.toolchain.gnu import GNU_10_Toolchain, Abstract_GNU_Toolchain from ofrak_patch_maker.toolchain.model import ToolchainConfig -from ofrak_type import ArchInfo, InstructionSet, MemoryPermissions +from ofrak_type import ArchInfo, InstructionSet, MemoryPermissions, SubInstructionSet class GNU_PPC_LINUX_10_Toolchain(GNU_10_Toolchain): @@ -83,3 +83,82 @@ def ld_generate_placeholder_reloc_sections(self): regions.append(got_region) sections.append(self._ld_generate_got_section(got_name)) return regions, sections + + +class GNU_PPCVLE_4_Toolchain(Abstract_GNU_Toolchain): + binary_file_parsers = [GNU_V10_ELF_Parser()] + + def __init__( + self, + processor: ArchInfo, + toolchain_config: ToolchainConfig, + logger: logging.Logger = logging.getLogger(__name__), + ): + super().__init__(processor, toolchain_config, logger=logger) + logging.warning(f"PLOUF: {self._config}") + if self._config.hard_float: + self._compiler_flags.append("-mhard-float") + else: + self._compiler_flags.append("-msoft-float") + self._assembler_flags.append("-mvle") + + @property + def segment_alignment(self) -> int: + return 4 # TODO: Check + + @property + def name(self) -> str: + return "GNU_PPCVLE_4" + + def _get_assembler_target(self, processor: ArchInfo) -> Optional[str]: + if processor.isa != InstructionSet.PPC or processor.sub_isa != SubInstructionSet.PPCVLE: + raise ValueError( + f"The GNU PPCVLE toolchain does not support ISAs that are not PPCVLE. " + f"(Got: {processor.isa.name})" + ) + if self._config.assembler_target: + return self._config.assembler_target + + # PPCVLE GNU 4 does not implement -march + return None + + @staticmethod + def _ld_generate_rel_dyn_section( + memory_region_name: str, + ) -> str: + rel_dyn_section_name = ".rela.dyn" + return ( + f" {rel_dyn_section_name} : {{\n" + f" *.o({rel_dyn_section_name})\n" + f" }} > {memory_region_name}" + ) + + @staticmethod + def _ld_generate_got_plt_section( + memory_region_name: str, + ) -> str: + got_plt_section_name = ".got2" + return ( + f" {got_plt_section_name} : {{\n" + f" *.o({got_plt_section_name})\n" + f" }} > {memory_region_name}" + ) + + def _ld_generate_got_region(self, vm_address, length): + region_name = '".got_mem"' + perms_string = self._ld_perm2str(MemoryPermissions.R) + return ( + f" {region_name} ({perms_string}) : ORIGIN = {hex(vm_address)}, " + f"LENGTH = {hex(length)}", + region_name, + ) + + def ld_generate_placeholder_reloc_sections(self): + regions, sections = super().ld_generate_placeholder_reloc_sections() + ( + got_region, + got_name, + ) = self._ld_generate_got_region(0xDEADBEEF + 0x30000, 0x1000) + regions.append(got_region) + sections.append(self._ld_generate_got_section(got_name)) + return regions, sections diff --git a/ofrak_patch_maker/ofrak_patch_maker_test/example_2/bom1.vle.S b/ofrak_patch_maker/ofrak_patch_maker_test/example_2/bom1.vle.S new file mode 100644 index 000000000..f418190de --- /dev/null +++ b/ofrak_patch_maker/ofrak_patch_maker_test/example_2/bom1.vle.S @@ -0,0 +1,7 @@ +.global _main1 +_main1: + add 1, 1, 2 + add 1, 1, 2 + add 1, 1, 2 + add 1, 1, 2 + add 1, 1, 2 diff --git a/ofrak_patch_maker/ofrak_patch_maker_test/example_2/bom2.vle.S b/ofrak_patch_maker/ofrak_patch_maker_test/example_2/bom2.vle.S new file mode 100644 index 000000000..be797fe14 --- /dev/null +++ b/ofrak_patch_maker/ofrak_patch_maker_test/example_2/bom2.vle.S @@ -0,0 +1,7 @@ +.global _main2 +_main2: + add 2, 2, 3 + add 2, 2, 3 + add 2, 2, 3 + add 2, 2, 3 + add 2, 2, 3 diff --git a/ofrak_patch_maker/ofrak_patch_maker_test/example_2/bom3.vle.S b/ofrak_patch_maker/ofrak_patch_maker_test/example_2/bom3.vle.S new file mode 100644 index 000000000..641a56e04 --- /dev/null +++ b/ofrak_patch_maker/ofrak_patch_maker_test/example_2/bom3.vle.S @@ -0,0 +1,7 @@ +.global _main3 +_main3: + add 3, 3, 4 + add 3, 3, 4 + add 3, 3, 4 + add 3, 3, 4 + add 3, 3, 4 diff --git a/ofrak_patch_maker/ofrak_patch_maker_test/test_alignment/patch_vle.as b/ofrak_patch_maker/ofrak_patch_maker_test/test_alignment/patch_vle.as new file mode 100644 index 000000000..412bc43ce --- /dev/null +++ b/ofrak_patch_maker/ofrak_patch_maker_test/test_alignment/patch_vle.as @@ -0,0 +1,2 @@ +.align 1 +se_mflr 0 diff --git a/ofrak_patch_maker/ofrak_patch_maker_test/test_ppc_vle_toolchain.py b/ofrak_patch_maker/ofrak_patch_maker_test/test_ppc_vle_toolchain.py new file mode 100644 index 000000000..5ef65c4db --- /dev/null +++ b/ofrak_patch_maker/ofrak_patch_maker_test/test_ppc_vle_toolchain.py @@ -0,0 +1,129 @@ +import os +import tempfile + +import pytest + +from ofrak_patch_maker.model import PatchRegionConfig +from ofrak_patch_maker.patch_maker import PatchMaker +from ofrak_patch_maker.toolchain.model import ( + ToolchainConfig, + BinFileType, + CompilerOptimizationLevel, + Segment, +) +from ofrak_patch_maker.toolchain.utils import get_file_format +from ofrak_patch_maker_test import ToolchainUnderTest, CURRENT_DIRECTORY + +from ofrak_patch_maker.toolchain.gnu_ppc import GNU_PPCVLE_4_Toolchain +from ofrak_type import ( + ArchInfo, + InstructionSet, + BitWidth, + Endianness, + SubInstructionSet, + MemoryPermissions, +) + +from ofrak_patch_maker_test.toolchain_asm import ( + run_monkey_patch_test, +) + +from ofrak_patch_maker_test.toolchain_c import ( + run_bounds_check_test, + run_hello_world_test, +) + +PPC_EXTENSION = ".vle" + + +@pytest.fixture( + params=[ + ToolchainUnderTest( + GNU_PPCVLE_4_Toolchain, + ArchInfo( + InstructionSet.PPC, + SubInstructionSet.PPCVLE, + BitWidth.BIT_32, + Endianness.BIG_ENDIAN, + None, + ), + PPC_EXTENSION, + ), + ] +) +def toolchain_under_test(request) -> ToolchainUnderTest: + return request.param + + +# ASM Tests +# def test_challenge_3_reloc_toy_example(toolchain_under_test: ToolchainUnderTest): +# # TODO +# pass + + +def test_monkey_patch(toolchain_under_test: ToolchainUnderTest): + run_monkey_patch_test(toolchain_under_test) + + +# C Tests +def test_bounds_check(toolchain_under_test: ToolchainUnderTest): + run_bounds_check_test(toolchain_under_test) + + +def test_hello_world(toolchain_under_test: ToolchainUnderTest): + run_hello_world_test(toolchain_under_test) + + +def test_vle_alignment(toolchain_under_test: ToolchainUnderTest): + tc_config = ToolchainConfig( + file_format=BinFileType.ELF, + force_inlines=True, + relocatable=False, + no_std_lib=True, + no_jump_tables=True, + no_bss_section=True, + create_map_files=True, + compiler_optimization_level=CompilerOptimizationLevel.NONE, + debug_info=True, + check_overlap=False, + hard_float=True, + ) + + build_dir = tempfile.mkdtemp() + + patch_maker = PatchMaker( + toolchain=toolchain_under_test.toolchain(toolchain_under_test.proc, tc_config), + build_dir=build_dir, + ) + patch_source = os.path.join(CURRENT_DIRECTORY, "test_alignment/patch_vle.as") + patch_bom = patch_maker.make_bom("patch", [patch_source], [], []) + + # Grab the resulting object paths and re-map them to the segments we chose for each source file. + patch_object = patch_bom.object_map[patch_source] + text_segment_patch = Segment( + segment_name=".text", + vm_address=0x51A, + offset=0, + is_entry=False, + length=2, + access_perms=MemoryPermissions.RX, + ) + segment_dict = { + patch_object.path: (text_segment_patch,), + } + + exec_path = os.path.join(build_dir, "patch_exec") + # Generate a PatchRegionConfig from your segment Dict. + # This data structure informs ld script generation which regions to create for every segment. + p = PatchRegionConfig(patch_bom.name + "_patch", segment_dict) + fem = patch_maker.make_fem([(patch_bom, p)], exec_path) + assert os.path.exists(exec_path) + assert get_file_format(exec_path) == tc_config.file_format + code_segments = [s for s in fem.executable.segments if s.access_perms == MemoryPermissions.RX] + assert len(code_segments) == 1 + assert code_segments[0].vm_address == 0x51A + assert code_segments[0].length == 2 + with open(exec_path, "rb") as f: + dat = f.read() + code_offset = code_segments[0].offset + assert dat[code_offset : code_offset + 2] == b"\x00\x80" diff --git a/ofrak_patch_maker/ofrak_patch_maker_test/test_symbol_parsing.py b/ofrak_patch_maker/ofrak_patch_maker_test/test_symbol_parsing.py index c1794fed7..220bbb6b4 100644 --- a/ofrak_patch_maker/ofrak_patch_maker_test/test_symbol_parsing.py +++ b/ofrak_patch_maker/ofrak_patch_maker_test/test_symbol_parsing.py @@ -9,7 +9,7 @@ from ofrak_patch_maker.toolchain.gnu_aarch64 import GNU_AARCH64_LINUX_10_Toolchain from ofrak_patch_maker.toolchain.gnu_arm import GNU_ARM_NONE_EABI_10_2_1_Toolchain from ofrak_patch_maker.toolchain.gnu_avr import GNU_AVR_5_Toolchain -from ofrak_patch_maker.toolchain.gnu_ppc import GNU_PPC_LINUX_10_Toolchain +from ofrak_patch_maker.toolchain.gnu_ppc import GNU_PPC_LINUX_10_Toolchain, GNU_PPCVLE_4_Toolchain from ofrak_patch_maker.toolchain.gnu_x64 import GNU_X86_64_LINUX_EABI_10_3_0_Toolchain from ofrak_patch_maker.toolchain.llvm_12 import LLVM_12_0_1_Toolchain from ofrak_patch_maker.toolchain.model import ( @@ -89,6 +89,16 @@ def full_label(self) -> str: GNU_PPC_LINUX_10_Toolchain, ArchInfo(InstructionSet.PPC, None, BitWidth.BIT_32, Endianness.BIG_ENDIAN, None), ), + ( + GNU_PPCVLE_4_Toolchain, + ArchInfo( + InstructionSet.PPC, + SubInstructionSet.PPCVLE, + BitWidth.BIT_32, + Endianness.BIG_ENDIAN, + None, + ), + ), ( GNU_AVR_5_Toolchain, ArchInfo( diff --git a/ofrak_patch_maker/ofrak_patch_maker_test/toolchain_c.py b/ofrak_patch_maker/ofrak_patch_maker_test/toolchain_c.py index 7ced85369..1b9050a56 100644 --- a/ofrak_patch_maker/ofrak_patch_maker_test/toolchain_c.py +++ b/ofrak_patch_maker/ofrak_patch_maker_test/toolchain_c.py @@ -2,6 +2,7 @@ import os import tempfile from ofrak_patch_maker.toolchain.gnu_avr import GNU_AVR_5_Toolchain +from ofrak_patch_maker.toolchain.gnu_ppc import GNU_PPCVLE_4_Toolchain from ofrak_patch_maker.toolchain.gnu_x64 import GNU_X86_64_LINUX_EABI_10_3_0_Toolchain from ofrak_patch_maker_test import ToolchainUnderTest from ofrak_type.architecture import InstructionSet @@ -95,6 +96,9 @@ def run_hello_world_test(toolchain_under_test: ToolchainUnderTest): if toolchain_under_test.toolchain == GNU_AVR_5_Toolchain: relocatable = False base_symbols = {"__mulhi3": 0x1234} # Dummy address to fix missing symbol + elif toolchain_under_test.toolchain == GNU_PPCVLE_4_Toolchain: + relocatable = True + base_symbols = {"__eabi": 0x1234} else: relocatable = True base_symbols = None diff --git a/ofrak_type/ofrak_type/architecture.py b/ofrak_type/ofrak_type/architecture.py index 4a34b42bc..fa9eca94f 100644 --- a/ofrak_type/ofrak_type/architecture.py +++ b/ofrak_type/ofrak_type/architecture.py @@ -87,6 +87,7 @@ class SubInstructionSet(Enum): AVRXMEGA7 = "avrxmega7" AVRTINY = "avrtiny" AVR1 = "avr1" # assembler only + PPCVLE = "ppc-vle" # PowerPC VLE (Variable Length Encoding) class InstructionSetMode(Enum): From 538d5e29137d06f91d76c80d33395068928303cc Mon Sep 17 00:00:00 2001 From: Paul Noalhyt Date: Tue, 27 Jun 2023 18:07:28 +0200 Subject: [PATCH 02/13] Check if the VLE toolchain is available before installing and testing --- ofrak_patch_maker/Dockerstub | 16 +++++---- .../ofrak_patch_maker/toolchain.conf | 10 +++--- .../test_ppc_vle_toolchain.py | 35 +++++++++++++++++++ 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index f5cfac991..e6a9bfb45 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -79,12 +79,16 @@ fi; #PPCVLE 4 NXP GCC Fork # Download the toolchain into your OFRAK directory from here (requires sign up): # https://www.nxp.com/design/software/development-software/s32-design-studio-ide/s32-design-studio-for-power-architecture:S32DS-PA +# if the file `gcc-4.9.4-Ee200-eabivle-x86_64-linux-g2724867.zip` doesn't exist, the toolchain won't be installed, and corresponding tests will be skipped. ARG OFRAK_DIR=. -COPY $OFRAK_DIR/gcc-4.9.4-Ee200-eabivle-x86_64-linux-g2724867.zip /tmp -RUN cd /tmp && \ - unzip gcc-4.9.4-Ee200-eabivle-x86_64-linux-g2724867.zip && \ - cd powerpc-eabivle-4_9 && \ - for f in *; do test -d "${f}" && cp -r "${f}" /usr/; done && \ +COPY $OFRAK_DIR/gcc-4.9.4-Ee200-eabivle-x86_64-linux-g2724867.zi[p] /tmp +RUN test -f /tmp/gcc-4.9.4-Ee200-eabivle-x86_64-linux-g2724867.zip && \ + cd /tmp && \ + unzip -q gcc-4.9.4-Ee200-eabivle-x86_64-linux-g2724867.zip && \ + cp -r powerpc-eabivle-4_9 /opt/rbs/toolchain/ && \ + rm -rf powerpc-eabivle-4_9 && \ + rm gcc-4.9.4-Ee200-eabivle-x86_64-linux-g2724867.zip && \ dpkg --add-architecture i386 && \ apt-get update && \ - apt-get install -y libc6:i386 libncurses5:i386 libstdc++6:i386 lib32z1 + apt-get install -y libc6:i386 libncurses5:i386 libstdc++6:i386 lib32z1 \ + || true diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf index 6d019d1fd..1b86c0041 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf @@ -51,10 +51,10 @@ LINKER = /usr/bin/powerpc-linux-gnu-ld BIN_PARSER = /usr/bin/powerpc-linux-gnu-objdump [GNU_PPCVLE_4] -PREPROCESSOR = /usr/bin/powerpc-eabivle-gcc -COMPILER = /usr/bin/powerpc-eabivle-gccbloop -LINKER = /usr/bin/powerpc-eabivle-ld -BIN_PARSER = /usr/bin/powerpc-eabivle-objdump +PREPROCESSOR = /opt/rbs/toolchain/powerpc-eabivle-4_9/bin/powerpc-eabivle-gcc +COMPILER = /opt/rbs/toolchain/powerpc-eabivle-4_9/bin/powerpc-eabivle-gcc +LINKER = /opt/rbs/toolchain/powerpc-eabivle-4_9/bin/powerpc-eabivle-ld +BIN_PARSER = /opt/rbs/toolchain/powerpc-eabivle-4_9/bin/powerpc-eabivle-objdump [ASM] ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as @@ -63,4 +63,4 @@ M68K_ASM_PATH = /usr/bin/m68k-linux-gnu-as AARCH64_ASM_PATH = /usr/bin/aarch64-linux-gnu-as AVR_ASM_PATH = /usr/bin/avr-as PPC_ASM_PATH = /usr/bin/powerpc-linux-gnu-as -PPCVLE_ASM_PATH = /usr/bin/powerpc-eabivle-as +PPCVLE_ASM_PATH = /opt/rbs/toolchain/powerpc-eabivle-4_9/bin/powerpc-eabivle-as diff --git a/ofrak_patch_maker/ofrak_patch_maker_test/test_ppc_vle_toolchain.py b/ofrak_patch_maker/ofrak_patch_maker_test/test_ppc_vle_toolchain.py index 5ef65c4db..c5d74fa73 100644 --- a/ofrak_patch_maker/ofrak_patch_maker_test/test_ppc_vle_toolchain.py +++ b/ofrak_patch_maker/ofrak_patch_maker_test/test_ppc_vle_toolchain.py @@ -1,5 +1,6 @@ import os import tempfile +from warnings import warn import pytest @@ -36,6 +37,32 @@ PPC_EXTENSION = ".vle" +INSTALL_TOOLCHAIN_MESSAGE = f""" +The NXP PPC VLE toolchain was not installed as part af the container build, because it requires signing-up and manually downloading the toolchain. +Download the toolchain into your OFRAK directory from here: +https://www.nxp.com/design/software/development-software/s32-design-studio-ide/s32-design-studio-for-power-architecture:S32DS-PA +Then rebuild the docker container, or refer to the Dockerfile for installation instructions. +""" + + +def check_toolchain_installed(toolchain_under_test: ToolchainUnderTest) -> bool: + tc_config = ToolchainConfig( + file_format=BinFileType.ELF, + force_inlines=True, + relocatable=False, + no_std_lib=True, + no_jump_tables=True, + no_bss_section=True, + compiler_optimization_level=CompilerOptimizationLevel.FULL, + ) + compiler_path_exist = os.path.exists( + toolchain_under_test.toolchain(toolchain_under_test.proc, tc_config)._compiler_path + ) + if not compiler_path_exist: + warn(INSTALL_TOOLCHAIN_MESSAGE) + return compiler_path_exist + + @pytest.fixture( params=[ ToolchainUnderTest( @@ -62,19 +89,27 @@ def toolchain_under_test(request) -> ToolchainUnderTest: def test_monkey_patch(toolchain_under_test: ToolchainUnderTest): + if not check_toolchain_installed(toolchain_under_test): + pytest.skip(INSTALL_TOOLCHAIN_MESSAGE) run_monkey_patch_test(toolchain_under_test) # C Tests def test_bounds_check(toolchain_under_test: ToolchainUnderTest): + if not check_toolchain_installed(toolchain_under_test): + pytest.skip(INSTALL_TOOLCHAIN_MESSAGE) run_bounds_check_test(toolchain_under_test) def test_hello_world(toolchain_under_test: ToolchainUnderTest): + if not check_toolchain_installed(toolchain_under_test): + pytest.skip(INSTALL_TOOLCHAIN_MESSAGE) run_hello_world_test(toolchain_under_test) def test_vle_alignment(toolchain_under_test: ToolchainUnderTest): + if not check_toolchain_installed(toolchain_under_test): + pytest.skip(INSTALL_TOOLCHAIN_MESSAGE) tc_config = ToolchainConfig( file_format=BinFileType.ELF, force_inlines=True, From ef4a988af40b8960c426711bfadf492fcf9549bb Mon Sep 17 00:00:00 2001 From: Jacob Strieb <99368685+rbs-jacob@users.noreply.github.com> Date: Wed, 28 Jun 2023 11:49:28 -0400 Subject: [PATCH 03/13] Add script to download PPC VLE toolchain --- ofrak_patch_maker/download_ppcvle.py | 51 ++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 ofrak_patch_maker/download_ppcvle.py diff --git a/ofrak_patch_maker/download_ppcvle.py b/ofrak_patch_maker/download_ppcvle.py new file mode 100644 index 000000000..5ce03432c --- /dev/null +++ b/ofrak_patch_maker/download_ppcvle.py @@ -0,0 +1,51 @@ +import argparse +import os + +from playwright.sync_api import sync_playwright + + +START_URL = "https://www.nxp.com/design/software/development-software/s32-design-studio-ide/s32-design-studio-for-power-architecture:S32DS-PA" +DOWNLOAD_PATH = "gcc-4.9.4-Ee200-eabivle-x86_64-linux-g2724867.zip" + + +def run(page, email, password) -> None: + print("Going to page") + page.goto() + page.get_by_role("listitem").filter( + has_text="Build Tools NXP Embedded GCC for Power Architecture" + ).filter(has_text="Linux").get_by_role("link", name="Download", exact=True).click() + + print("Signing in") + page.locator("#username").click() + page.keyboard.type(email) + page.locator("#password").click() + page.keyboard.type(password) + page.get_by_role("button", name="SIGN IN").click() + + print("Accepting terms and conditions") + page.get_by_role("button", name="I Accept").click() + + print("Waiting for download") + with page.expect_download() as download_info: + # Download begins when the page is loaded + pass + os.rename(download_info.value.path(), DOWNLOAD_PATH) + + print(f"Complete! Saved to {DOWNLOAD_PATH}") + + +def main(args): + with sync_playwright() as playwright: + browser = playwright.chromium.launch(headless=True) + context = browser.new_context() + page = context.new_page() + run(page, args.email, args.password) + context.close() + browser.close() + + +if __name__ == "__main__": + argument_parser = argparse.ArgumentParser() + argument_parser.add_argument("email") + argument_parser.add_argument("password") + main(argument_parser.parse_args()) From 4d59be3ec95f7854939ab03bd48ed9aca6c33f5c Mon Sep 17 00:00:00 2001 From: Jacob Strieb <99368685+rbs-jacob@users.noreply.github.com> Date: Wed, 28 Jun 2023 12:39:10 -0400 Subject: [PATCH 04/13] Pass output file as CLI argument --- ofrak_patch_maker/download_ppcvle.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ofrak_patch_maker/download_ppcvle.py b/ofrak_patch_maker/download_ppcvle.py index 5ce03432c..a47578d3c 100644 --- a/ofrak_patch_maker/download_ppcvle.py +++ b/ofrak_patch_maker/download_ppcvle.py @@ -5,10 +5,9 @@ START_URL = "https://www.nxp.com/design/software/development-software/s32-design-studio-ide/s32-design-studio-for-power-architecture:S32DS-PA" -DOWNLOAD_PATH = "gcc-4.9.4-Ee200-eabivle-x86_64-linux-g2724867.zip" -def run(page, email, password) -> None: +def run(page, email: str, password: str, outfile: str) -> None: print("Going to page") page.goto() page.get_by_role("listitem").filter( @@ -29,9 +28,9 @@ def run(page, email, password) -> None: with page.expect_download() as download_info: # Download begins when the page is loaded pass - os.rename(download_info.value.path(), DOWNLOAD_PATH) + os.rename(download_info.value.path(), outfile) - print(f"Complete! Saved to {DOWNLOAD_PATH}") + print(f"Complete! Saved to {outfile}") def main(args): @@ -39,7 +38,7 @@ def main(args): browser = playwright.chromium.launch(headless=True) context = browser.new_context() page = context.new_page() - run(page, args.email, args.password) + run(page, args.email, args.password, args.outfile) context.close() browser.close() @@ -48,4 +47,7 @@ def main(args): argument_parser = argparse.ArgumentParser() argument_parser.add_argument("email") argument_parser.add_argument("password") + argument_parser.add_argument( + "-o", "--outfile", default="/tmp/gcc-4.9.4-Ee200-eabivle-x86_64-linux-g2724867.zip" + ) main(argument_parser.parse_args()) From da785c868425724ea9eafdf70e8c7845150bbe8f Mon Sep 17 00:00:00 2001 From: Jacob Strieb <99368685+rbs-jacob@users.noreply.github.com> Date: Wed, 28 Jun 2023 13:04:14 -0400 Subject: [PATCH 05/13] Run NXP download script in Docker --- build_image.py | 29 ++++++++++++++++++++++++++++ ofrak_patch_maker/Dockerstub | 17 +++++++++------- ofrak_patch_maker/download_ppcvle.py | 12 ++++++------ 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/build_image.py b/build_image.py index 1ebe4d70d..8c6e9ec9e 100644 --- a/build_image.py +++ b/build_image.py @@ -1,3 +1,4 @@ +import tempfile from dataclasses import dataclass from enum import Enum from typing import List, Optional @@ -35,6 +36,8 @@ class OfrakImageConfig: install_target: InstallTarget cache_from: List[str] entrypoint: Optional[str] + nxp_email: Optional[str] + nxp_password: Optional[str] def validate_serial_txt_existence(self): """ @@ -81,11 +84,27 @@ def main(): for cache in config.cache_from: cache_args.append("--cache-from") cache_args.append(cache) + nxp_args = [] + email_file = password_file = None + if config.nxp_email and config.nxp_password: + email_file = tempfile.NamedTemporaryFile(suffix=".txt", mode="w+") + email_file.write(config.nxp_email) + email_file.flush() + password_file = tempfile.NamedTemporaryFile(suffix=".txt", mode="w+") + password_file.write(config.nxp_password) + password_file.flush() + nxp_args = [ + "--secret", + f"id=nxp_email,src={email_file.name}", + "--secret", + f"id=nxp_password,src={password_file.name}", + ] base_command = [ "docker", "build", "--build-arg", "BUILDKIT_INLINE_CACHE=1", + *nxp_args, "--cache-from", f"{full_base_image_name}:master", *cache_args, @@ -108,6 +127,9 @@ def main(): print(f"Error running command: '{' '.join(error.cmd)}'") print(f"Exit status: {error.returncode}") sys.exit(error.returncode) + if email_file and password_file: + email_file.close() + password_file.close() if config.build_finish: full_image_name = "/".join((config.registry, config.image_name)) @@ -146,7 +168,12 @@ def parse_args() -> OfrakImageConfig: default=InstallTarget.DEVELOP.value, ) parser.add_argument("--cache-from", action="append") + parser.add_argument("--nxp-email") + parser.add_argument("--nxp-password") args = parser.parse_args() + if (not not args.nxp_email) ^ (not not args.nxp_password): + raise RuntimeError("Must include the NXP email and password!") + with open(args.config) as file_handle: config_dict = yaml.safe_load(file_handle) image_config = OfrakImageConfig( @@ -161,6 +188,8 @@ def parse_args() -> OfrakImageConfig: InstallTarget(args.target), args.cache_from, config_dict.get("entrypoint"), + args.nxp_email, + args.nxp_password, ) image_config.validate_serial_txt_existence() return image_config diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index e6a9bfb45..217db529a 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -77,16 +77,19 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ fi; #PPCVLE 4 NXP GCC Fork -# Download the toolchain into your OFRAK directory from here (requires sign up): -# https://www.nxp.com/design/software/development-software/s32-design-studio-ide/s32-design-studio-for-power-architecture:S32DS-PA -# if the file `gcc-4.9.4-Ee200-eabivle-x86_64-linux-g2724867.zip` doesn't exist, the toolchain won't be installed, and corresponding tests will be skipped. +# Only runs if the NXP email and password to log in and download the toolchain are passed to build_image.py via the CLI flags ARG OFRAK_DIR=. -COPY $OFRAK_DIR/gcc-4.9.4-Ee200-eabivle-x86_64-linux-g2724867.zi[p] /tmp -RUN test -f /tmp/gcc-4.9.4-Ee200-eabivle-x86_64-linux-g2724867.zip && \ +COPY $OFRAK_DIR/ofrak_patch_maker/download_ppcvle.py /tmp/ +RUN --mount=type=secret,id=nxp_email,dst=/tmp/nxp_email.txt \ + --mount=type=secret,id=nxp_password,dst=/tmp/nxp_password.txt \ + test -f /tmp/nxp_email.txt && \ + test -f /tmp/nxp_password.txt && \ + python3 -m pip install playwright && \ + playwright install --with-deps chromium && \ + python3 /tmp/download_ppcvle.py "$(cat /tmp/nxp_email.txt)" "$(cat /tmp/nxp_password.txt)" && \ cd /tmp && \ unzip -q gcc-4.9.4-Ee200-eabivle-x86_64-linux-g2724867.zip && \ - cp -r powerpc-eabivle-4_9 /opt/rbs/toolchain/ && \ - rm -rf powerpc-eabivle-4_9 && \ + mv powerpc-eabivle-4_9 /opt/rbs/toolchain/ && \ rm gcc-4.9.4-Ee200-eabivle-x86_64-linux-g2724867.zip && \ dpkg --add-architecture i386 && \ apt-get update && \ diff --git a/ofrak_patch_maker/download_ppcvle.py b/ofrak_patch_maker/download_ppcvle.py index a47578d3c..08b0ae863 100644 --- a/ofrak_patch_maker/download_ppcvle.py +++ b/ofrak_patch_maker/download_ppcvle.py @@ -8,29 +8,29 @@ def run(page, email: str, password: str, outfile: str) -> None: - print("Going to page") - page.goto() + print(f"Going to page {START_URL}", flush=True) + page.goto(START_URL) page.get_by_role("listitem").filter( has_text="Build Tools NXP Embedded GCC for Power Architecture" ).filter(has_text="Linux").get_by_role("link", name="Download", exact=True).click() - print("Signing in") + print("Signing in", flush=True) page.locator("#username").click() page.keyboard.type(email) page.locator("#password").click() page.keyboard.type(password) page.get_by_role("button", name="SIGN IN").click() - print("Accepting terms and conditions") + print("Accepting terms and conditions", flush=True) page.get_by_role("button", name="I Accept").click() - print("Waiting for download") + print("Waiting for download", flush=True) with page.expect_download() as download_info: # Download begins when the page is loaded pass os.rename(download_info.value.path(), outfile) - print(f"Complete! Saved to {outfile}") + print(f"Complete! Saved to {outfile}", flush=True) def main(args): From a424ae75a6a1000c8e4908df5635b8733b46a35c Mon Sep 17 00:00:00 2001 From: Jacob Strieb <99368685+rbs-jacob@users.noreply.github.com> Date: Wed, 28 Jun 2023 13:12:14 -0400 Subject: [PATCH 06/13] Add NXP creds to Actions workflow as secrets --- .github/workflows/test-all.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index 89363a849..7bed279b7 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -77,7 +77,7 @@ jobs: if: inputs.flush_cache || steps.cache-image.outputs.cache-hit != 'true' || github.event_name == 'schedule' run: | python3 -m pip install PyYAML - DOCKER_BUILDKIT=1 python3 build_image.py --config ofrak-core-dev.yml --base + DOCKER_BUILDKIT=1 python3 build_image.py --config ofrak-core-dev.yml --base --nxp-email '${{ secrets.nxp_email }}' --nxp-password '${{ secrets.nxp_password }}' - name: Export base image if: inputs.flush_cache || steps.cache-image.outputs.cache-hit != 'true' || github.event_name == 'schedule' run: | From eb3d7fb3d6213fd757dc6d3a15f1d0fe45dfa209 Mon Sep 17 00:00:00 2001 From: Jacob Strieb <99368685+rbs-jacob@users.noreply.github.com> Date: Wed, 28 Jun 2023 13:27:02 -0400 Subject: [PATCH 07/13] Try to fix Actions secrets --- .github/workflows/test-all.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index 7bed279b7..8b6bed9dc 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -75,9 +75,12 @@ jobs: - name: Build base image # Always rebuild the base image when the scheduled workflow runs if: inputs.flush_cache || steps.cache-image.outputs.cache-hit != 'true' || github.event_name == 'schedule' + env: + NXP_EMAIL: ${{ secrets.NXP_EMAIL }} + NXP_PASSWORD: ${{ secrets.NXP_PASSWORD }} run: | python3 -m pip install PyYAML - DOCKER_BUILDKIT=1 python3 build_image.py --config ofrak-core-dev.yml --base --nxp-email '${{ secrets.nxp_email }}' --nxp-password '${{ secrets.nxp_password }}' + DOCKER_BUILDKIT=1 python3 build_image.py --config ofrak-core-dev.yml --base --nxp-email "${NXP_EMAIL}" --nxp-password "${NXP_PASSWORD}" - name: Export base image if: inputs.flush_cache || steps.cache-image.outputs.cache-hit != 'true' || github.event_name == 'schedule' run: | From 72e2fbc96f2db4553479a7257bf58f54499eb43f Mon Sep 17 00:00:00 2001 From: Jacob Strieb <99368685+rbs-jacob@users.noreply.github.com> Date: Wed, 28 Jun 2023 13:30:03 -0400 Subject: [PATCH 08/13] Use NXP creds when building Ghidra image --- .github/workflows/test-all.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index 8b6bed9dc..39fdc11e3 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -116,6 +116,9 @@ jobs: | docker load docker images - name: Build Ghidra image + env: + NXP_EMAIL: ${{ secrets.NXP_EMAIL }} + NXP_PASSWORD: ${{ secrets.NXP_PASSWORD }} run: | python3 -m pip install PyYAML DOCKER_BUILDKIT=1 \ @@ -123,7 +126,9 @@ jobs: --config ofrak-ghidra.yml \ --base \ --finish \ - --cache-from redballoonsecurity/ofrak/core-dev-base:latest + --cache-from redballoonsecurity/ofrak/core-dev-base:latest \ + --nxp-email "${NXP_EMAIL}" \ + --nxp-password "${NXP_PASSWORD}" - name: Test documentation run: | docker run \ From ba1ad0e653aba409c39a01e4c170fca633bb002d Mon Sep 17 00:00:00 2001 From: Paul Noalhyt Date: Wed, 28 Jun 2023 20:57:43 +0200 Subject: [PATCH 09/13] Avoid defining a function to handle the regex --- .../ofrak_ghidra/components/blocks/bb_unpacker.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/disassemblers/ofrak_ghidra/ofrak_ghidra/components/blocks/bb_unpacker.py b/disassemblers/ofrak_ghidra/ofrak_ghidra/components/blocks/bb_unpacker.py index 15f9d04ac..3e588896f 100644 --- a/disassemblers/ofrak_ghidra/ofrak_ghidra/components/blocks/bb_unpacker.py +++ b/disassemblers/ofrak_ghidra/ofrak_ghidra/components/blocks/bb_unpacker.py @@ -180,15 +180,11 @@ def _asm_fixups( # But it is more natural to express it in bytes, to get the instruction `se_stw r0,0x24(r1)` # (this is also the convention used by the VLE assembler) - def replace_offset(match): - new_operand = match.group(1) - new_operand += f"0x{int(match.group(2), 0)*4:x}" - new_operand += match.group(3) - return new_operand - mnemonic = base_mnemonic operands = re.sub( - r"(.*, )(0x[0-9]+)(\(r[0-9]+\))", lambda match: replace_offset(match), operands + r"(.*, )(0x[0-9]+)(\(r[0-9]+\))", + lambda match: match.group(1) + f"0x{int(match.group(2), 0)*4:x}" + match.group(3), + operands, ) else: mnemonic = base_mnemonic From 3c415eff6be4e1e03478ff0a076a94c4cf80a36f Mon Sep 17 00:00:00 2001 From: Paul Noalhyt Date: Thu, 29 Jun 2023 15:48:01 +0200 Subject: [PATCH 10/13] Remove useless print and add C99 flag --- ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py b/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py index 079dd421f..0d8c82a97 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py @@ -95,11 +95,11 @@ def __init__( logger: logging.Logger = logging.getLogger(__name__), ): super().__init__(processor, toolchain_config, logger=logger) - logging.warning(f"PLOUF: {self._config}") if self._config.hard_float: self._compiler_flags.append("-mhard-float") else: self._compiler_flags.append("-msoft-float") + self._compiler_flags.append("-std=c99") self._assembler_flags.append("-mvle") @property From 823fb419fc846386eccd9683a3fbec9602953b8d Mon Sep 17 00:00:00 2001 From: Paul Noalhyt Date: Wed, 5 Jul 2023 14:06:29 +0200 Subject: [PATCH 11/13] Remove c99 standard flag --- ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py b/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py index 0d8c82a97..6ea405673 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py @@ -99,7 +99,6 @@ def __init__( self._compiler_flags.append("-mhard-float") else: self._compiler_flags.append("-msoft-float") - self._compiler_flags.append("-std=c99") self._assembler_flags.append("-mvle") @property From 9fbfcf75719632cfb39f965dcccffd588fdd557f Mon Sep 17 00:00:00 2001 From: Paul Noalhyt Date: Mon, 10 Jul 2023 16:40:54 +0200 Subject: [PATCH 12/13] Remove PPC VLE toolchain functions from coverage --- .../ofrak_patch_maker/toolchain/gnu_ppc.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py b/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py index 6ea405673..853015133 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain/gnu_ppc.py @@ -101,9 +101,9 @@ def __init__( self._compiler_flags.append("-msoft-float") self._assembler_flags.append("-mvle") - @property + @property # pragma: no cover def segment_alignment(self) -> int: - return 4 # TODO: Check + return 4 @property def name(self) -> str: @@ -121,7 +121,7 @@ def _get_assembler_target(self, processor: ArchInfo) -> Optional[str]: # PPCVLE GNU 4 does not implement -march return None - @staticmethod + @staticmethod # pragma: no cover def _ld_generate_rel_dyn_section( memory_region_name: str, ) -> str: @@ -132,7 +132,7 @@ def _ld_generate_rel_dyn_section( f" }} > {memory_region_name}" ) - @staticmethod + @staticmethod # pragma: no cover def _ld_generate_got_plt_section( memory_region_name: str, ) -> str: @@ -143,7 +143,7 @@ def _ld_generate_got_plt_section( f" }} > {memory_region_name}" ) - def _ld_generate_got_region(self, vm_address, length): + def _ld_generate_got_region(self, vm_address, length): # pragma: no cover region_name = '".got_mem"' perms_string = self._ld_perm2str(MemoryPermissions.R) return ( @@ -152,7 +152,7 @@ def _ld_generate_got_region(self, vm_address, length): region_name, ) - def ld_generate_placeholder_reloc_sections(self): + def ld_generate_placeholder_reloc_sections(self): # pragma: no cover regions, sections = super().ld_generate_placeholder_reloc_sections() ( got_region, From dcb77a8e291c17b0018a4c60ce5cbc4f3037dff1 Mon Sep 17 00:00:00 2001 From: Paul Noalhyt Date: Wed, 4 Oct 2023 16:31:53 +0200 Subject: [PATCH 13/13] Add memory permissions for VLE ELFs (VLE execution bit) --- ofrak_type/ofrak_type/memory_permissions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ofrak_type/ofrak_type/memory_permissions.py b/ofrak_type/ofrak_type/memory_permissions.py index 92af59123..a24f46e7d 100644 --- a/ofrak_type/ofrak_type/memory_permissions.py +++ b/ofrak_type/ofrak_type/memory_permissions.py @@ -13,6 +13,10 @@ class MemoryPermissions(Enum): RW = R + W RX = R + X RWX = R + W + X + VLE = 0x10000000 + X_VLE = X + VLE + RX_VLE = R + X + VLE + RWX_VLE = R + W + X + VLE def as_str(self) -> str: string = ""