From 41952d86550f3d8b61eab7512ceead2047d57800 Mon Sep 17 00:00:00 2001 From: Steven Hugg Date: Wed, 16 Oct 2024 16:28:43 -0500 Subject: [PATCH] oscar64 server, base64 data --- scripts/docker/Dockerfile | 4 + src/worker/server/buildenv.ts | 148 +++++++++++++++++++++++++++++----- src/worker/tools/remote.ts | 2 +- 3 files changed, 134 insertions(+), 20 deletions(-) diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile index 90b0492a..a40af1f8 100644 --- a/scripts/docker/Dockerfile +++ b/scripts/docker/Dockerfile @@ -27,6 +27,10 @@ RUN apt-get install -y nodejs # Fetch the LLVM-Mos tarball and extract it RUN curl -L https://github.com/llvm-mos/llvm-mos-sdk/releases/latest/download/llvm-mos-linux.tar.xz | xz -d | tar x -C /app +# Fetch Oscar64 +RUN curl -L https://github.com/drmortalwombat/oscar64/archive/refs/tags/v1.31.255.tar.gz | tar xz -C /app +RUN cd /app/oscar64 && make -f make/makefile compiler + # Fetch the SDCC tarball #RUN apt-get install -y bzip2 #RUN curl -L https://cytranet.dl.sourceforge.net/project/sdcc/sdcc-linux-amd64/4.3.0/sdcc-4.3.0-amd64-unknown-linux2.5.tar.bz2 | tar xj -C /app diff --git a/src/worker/server/buildenv.ts b/src/worker/server/buildenv.ts index b69c925c..2d43521f 100644 --- a/src/worker/server/buildenv.ts +++ b/src/worker/server/buildenv.ts @@ -2,8 +2,8 @@ import fs from 'fs'; import path from 'path'; import { spawn } from 'child_process'; -import { WorkerBuildStep, WorkerErrorResult, WorkerFileUpdate, WorkerResult, isOutputResult } from '../../common/workertypes'; -import { getRootBasePlatform, replaceAll } from '../../common/util'; +import { CodeListing, CodeListingMap, Segment, WorkerBuildStep, WorkerErrorResult, WorkerFileUpdate, WorkerResult, isOutputResult } from '../../common/workertypes'; +import { getFilenamePrefix, getRootBasePlatform, replaceAll } from '../../common/util'; import { parseObjDump } from './clang'; import { BuildStep } from '../builder'; import { makeErrorMatcher } from '../listingutils'; @@ -15,6 +15,8 @@ interface ServerBuildTool { archs: string[]; platforms: string[]; platform_configs: { [platform: string]: ServerBuildToolPlatformConfig }; + processErrors(step: WorkerBuildStep, errorData: string): Promise; + processOutput(step: WorkerBuildStep, outfile: string): Promise; } interface ServerBuildToolPlatformConfig { @@ -31,6 +33,8 @@ const LLVM_MOS_TOOL: ServerBuildTool = { extensions: ['.c', '.cpp', '.s', '.S', '.C'], archs: ['6502'], platforms: ['atari8', 'c64', 'nes', 'pce', 'vcs'], + processOutput: basicProcessOutput, + processErrors: llvmMosProcessErrors, platform_configs: { default: { binpath: 'llvm-mos/bin', @@ -61,12 +65,31 @@ const LLVM_MOS_TOOL: ServerBuildTool = { } } +async function basicProcessOutput(step: WorkerBuildStep, outfile: string): Promise { + let output = await fs.promises.readFile(outfile, { encoding: 'base64' }); + return { output }; +} + +async function llvmMosProcessErrors(step: WorkerBuildStep, errorData: string): Promise { + errorData = errorData.replace(/(\/var\/folders\/.+?\/).+?:/g, ''); // TODO? + let errors = []; + // split errorData into lines + let errorMatcher = makeErrorMatcher(errors, /([^:/]+):(\d+):(\d+):\s*(.+)/, 2, 4, step.path, 1); + for (let line of errorData.split('\n')) { + errorMatcher(line); + } + return { errors }; +} + + const OSCAR64_TOOL: ServerBuildTool = { name: 'oscar64', version: '', extensions: ['.c', '.cc', '.cpp'], archs: ['6502'], platforms: ['atari8', 'c64', 'nes'], + processOutput: oscar64ProcessOutput, + processErrors: oscar64ProcessErrors, platform_configs: { default: { binpath: 'oscar64/bin', @@ -79,6 +102,93 @@ const OSCAR64_TOOL: ServerBuildTool = { } } +async function oscar64ProcessErrors(step: WorkerBuildStep, errorData: string): Promise { + let errors = []; + // split errorData into lines + let errorMatcher = makeErrorMatcher(errors, /\/([^(]+)\((\d+), (\d+)\) : \s*(.+)/, 2, 4, step.path, 1); + for (let line of errorData.split('\n')) { + errorMatcher(line); + } + return { errors }; +} + +async function oscar64ProcessOutput(step: WorkerBuildStep, outpath: string): Promise { + let prefix_path = outpath.replace(/\.\w+$/, ''); + let output = await fs.promises.readFile(outpath, { encoding: 'base64' }); + let listings: CodeListingMap = {}; + let symbolmap: { [sym: string]: number } = {}; + let debuginfo = {}; + let segments: Segment[] = []; + // read segments + { + let txt = await fs.promises.readFile(prefix_path + '.map', { encoding: 'utf-8' }); + for (let line of txt.split("\n")) { + // 0880 - 0887 : DATA, code + const m1 = line.match(/([0-9a-f]+) - ([0-9a-f]+) : ([A-Z_]+), (.+)/); + if (m1) { + const name = m1[4]; + const start = parseInt(m1[1], 16); + const end = parseInt(m1[2], 16); + segments.push({ + name, start, size: end - start, + }); + } + // 0801 (0062) : startup, NATIVE_CODE:startup + const m2 = line.match(/([0-9a-f]+) \(([0-9a-f]+)\) : ([^,]+), (.+)/); + if (m2) { + const addr = parseInt(m2[1], 16); + const name = m2[3]; + symbolmap[name] = addr; + } + } + } + // read listings + { + let txt = await fs.promises.readFile(prefix_path + '.asm', { encoding: 'utf-8' }); + let lst : CodeListing = { lines: [], text: txt }; + let asm_lineno = 0; + let c_lineno = 0; + let c_path = ''; + const path = step.path; + for (let line of txt.split("\n")) { + asm_lineno++; + //; 4, "/Users/sehugg/PuzzlingPlans/8bitworkshop/server-root/oscar64/main.c" + let m2 = line.match(/;\s*(\d+), "(.+?)"/); + if (m2) { + c_lineno = parseInt(m2[1]); + c_path = m2[2].split('/').pop(); // TODO + } + //0807 : 30 36 __ BMI $083f ; (startup + 62) + let m = line.match(/([0-9a-f]+) : ([0-9a-f _]{8}) (.+)/); + if (m) { + let offset = parseInt(m[1], 16); + let hex = m[2]; + let asm = m[3]; + if (c_path) { + lst.lines.push({ + line: c_lineno, + path: c_path, + offset, + iscode: true + }); + c_path = ''; + c_lineno = 0; + } + /* + lst.asmlines.push({ + line: asm_lineno, + path, + offset, + insns: hex + ' ' + asm, + iscode: true }); + */ + } + } + listings[getFilenamePrefix(step.path) + '.lst'] = lst; + } + return { output, listings, symbolmap, segments, debuginfo }; +} + export function findBestTool(step: BuildStep) { if (!step?.tool) throw new Error('No tool specified'); const [name, version] = step.tool.split('@'); @@ -123,7 +233,16 @@ export class ServerBuildEnv { if (file.path.match(/[\\\/]/)) { throw new Error(`Invalid file path: ${file.path}`); } - await fs.promises.writeFile(path.join(this.sessionDir, file.path), file.data); + let data = file.data; + if (typeof data === 'string' && data.startsWith('data:base64,')) { + // convert data URL to base64 + let parts = data.split(','); + if (parts.length !== 2) { + throw new Error(`Invalid data URL: ${data}`); + } + data = Buffer.from(parts[1], 'base64'); + } + await fs.promises.writeFile(path.join(this.sessionDir, file.path), data); } async build(step: WorkerBuildStep, platform?: string): Promise { @@ -174,8 +293,10 @@ export class ServerBuildEnv { let childProcess = spawn(command, args, { shell: true, cwd: this.rootdir, - env: { PATH: path.join(this.rootdir, config.binpath) - } }); + env: { + PATH: path.join(this.rootdir, config.binpath) + } + }); let outputData = ''; let errorData = ''; // TODO? @@ -191,14 +312,13 @@ export class ServerBuildEnv { if (platform === 'debug') { resolve(this.processDebugInfo(step)); } else { - resolve(this.processOutput(step, outfile)); + resolve(this.tool.processOutput(step, outfile)); } } else { errorData = replaceAll(errorData, this.sessionDir, ''); errorData = replaceAll(errorData, this.rootdir, ''); // remove folder paths - errorData = errorData.replace(/(\/var\/folders\/.+?\/).+?:/g, ''); - let errorResult = await this.processErrors(step, errorData); + let errorResult = await this.tool.processErrors(step, errorData); if (errorResult.errors.length === 0) { errorResult.errors.push({ line: 0, msg: `Build failed.\n\n${errorData}` }); } @@ -208,16 +328,6 @@ export class ServerBuildEnv { }); } - async processErrors(step: WorkerBuildStep, errorData: string): Promise { - let errors = []; - // split errorData into lines - let errorMatcher = makeErrorMatcher(errors, /([^:/]+):(\d+):(\d+):\s*(.+)/, 2, 4, step.path, 1); - for (let line of errorData.split('\n')) { - errorMatcher(line); - } - return { errors }; - } - async processOutput(step: WorkerBuildStep, outfile: string): Promise { let output = await fs.promises.readFile(outfile, { encoding: 'base64' }); return { output }; @@ -247,7 +357,7 @@ export class ServerBuildEnv { } return result; } catch (err) { - return { errors: [{line:0, msg: err.toString()}] }; + return { errors: [{ line: 0, msg: err.toString() }] }; } } } diff --git a/src/worker/tools/remote.ts b/src/worker/tools/remote.ts index be6647b0..8a31adfd 100644 --- a/src/worker/tools/remote.ts +++ b/src/worker/tools/remote.ts @@ -20,7 +20,7 @@ export async function buildRemote(step: BuildStep): Promise { let path = step.files[i]; let entry = store.workfs[path]; // convert to base64 - let data = typeof entry.data === 'string' ? entry.data : btoa(byteArrayToString(entry.data)); + let data = typeof entry.data === 'string' ? entry.data : "data:base64," + btoa(byteArrayToString(entry.data)); updates.push({ path, data }); } // build the command