refactor(cli): ship one Eval runtime in the npm CLI package

packages/cli/src/__tests__/eval-bundle-path.test.ts

@@ -28,7 +28,7 @@ describe('installed Eval bundle', () => {
   test('points Eval containers at the installed package root', async (t) => {
     const packageRoot = await mkdtemp(join(tmpdir(), 'maka-cli-eval-bundle-'));
     t.after(() => rm(packageRoot, { recursive: true, force: true }));
-    await mkdir(join(packageRoot, 'packages/eval'), { recursive: true });
+    await mkdir(join(packageRoot, 'node_modules/@maka/eval'), { recursive: true });
     const environment: NodeJS.ProcessEnv = {};
 
     configureInstalledEvalBundle(environment, packageRoot);
@@ -44,7 +44,7 @@ describe('installed Eval bundle', () => {
     assert.equal(environment.MAKA_EVAL_MAKA_BUNDLE_PATH, '/explicit/bundle');
   });
 
-  test('does not change source-checkout behavior without a packaged Eval mirror', () => {
+  test('does not change source-checkout behavior without a packaged Eval runtime', () => {
     const environment: NodeJS.ProcessEnv = {};
 
     configureInstalledEvalBundle(environment, '/missing/package');

packages/cli/src/eval-bundle-path.ts

@@ -28,7 +28,7 @@ export function configureInstalledEvalBundle(
 ): void {
   if (Object.hasOwn(environment, MAKA_EVAL_BUNDLE_ENV)) return;
   try {
-    if (!statSync(resolve(packageRoot, 'packages/eval')).isDirectory()) return;
+    if (!statSync(resolve(packageRoot, 'node_modules/@maka/eval')).isDirectory()) return;
   } catch {
     return;
   }

packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-deepseek-harness.json

@@ -34,8 +34,8 @@
       ],
       "egressProxy": {
         "composeSourceEnv": "MAKA_EVAL_MAKA_BUNDLE_PATH",
-        "composeRelativePath": "packages/eval/harbor/docker-compose-egress-proxy.yaml",
-        "networkPolicyRelativePath": "packages/eval/harbor/egress-proxy/network-policy",
+        "composeRelativePath": "node_modules/@maka/eval/harbor/docker-compose-egress-proxy.yaml",
+        "networkPolicyRelativePath": "node_modules/@maka/eval/harbor/egress-proxy/network-policy",
         "proxyUrl": "http://maka-eval-mitmproxy:8080",
         "allowedHost": "maka-eval-mitmproxy",
         "containerCaPath": "/opt/maka-egress/mitmproxy-ca-cert.pem"
@@ -73,7 +73,7 @@
           "DEEPSEEK_API_KEY": "DEEPSEEK_API_KEY"
         },
         "args": [
-          "/opt/maka-agent/packages/eval/dist/harbor-external-subject.js",
+          "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js",
           "deepseek-harness",
           "https://api.deepseek.com/v1",
           "/",

packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-eight-arm.json

@@ -31,8 +31,8 @@
       ],
       "egressProxy": {
         "composeSourceEnv": "MAKA_EVAL_MAKA_BUNDLE_PATH",
-        "composeRelativePath": "packages/eval/harbor/docker-compose-egress-proxy.yaml",
-        "networkPolicyRelativePath": "packages/eval/harbor/egress-proxy/network-policy",
+        "composeRelativePath": "node_modules/@maka/eval/harbor/docker-compose-egress-proxy.yaml",
+        "networkPolicyRelativePath": "node_modules/@maka/eval/harbor/egress-proxy/network-policy",
         "proxyUrl": "http://maka-eval-mitmproxy:8080",
         "allowedHost": "maka-eval-mitmproxy",
         "containerCaPath": "/opt/maka-egress/mitmproxy-ca-cert.pem"
@@ -94,7 +94,7 @@
       "credentials": ["DEEPSEEK_API_KEY"],
       "config": {
         "nodePath": "/opt/maka-node-toolchain/bin/node",
-        "shimPath": "/opt/maka-agent/packages/eval/dist/harbor-maka-subject.js",
+        "shimPath": "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-maka-subject.js",
         "runtimeHostsPath": "/tmp/maka-runtime-hosts",
         "hostSettlementTimeoutMs": 120000,
         "connectionSlug": "env-deepseek",
@@ -115,7 +115,7 @@
         "command": "/opt/maka-node-toolchain/bin/node",
         "credentialEnvironment": { "OPENAI_API_KEY": "DEEPSEEK_API_KEY" },
         "args": [
-          "/opt/maka-agent/packages/eval/dist/harbor-external-subject.js",
+          "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js",
           "codex",
           "https://api.deepseek.com",
           "/",
@@ -139,7 +139,7 @@
         "command": "/opt/maka-node-toolchain/bin/node",
         "credentialEnvironment": { "ANTHROPIC_API_KEY": "DEEPSEEK_API_KEY" },
         "args": [
-          "/opt/maka-agent/packages/eval/dist/harbor-external-subject.js",
+          "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js",
           "claude-code",
           "https://api.deepseek.com/anthropic",
           "/",
@@ -170,7 +170,7 @@
         "command": "/opt/maka-node-toolchain/bin/node",
         "credentialEnvironment": { "OPENAI_API_KEY": "DEEPSEEK_API_KEY" },
         "args": [
-          "/opt/maka-agent/packages/eval/dist/harbor-external-subject.js",
+          "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js",
           "reasonix",
           "https://api.deepseek.com",
           "/",
@@ -195,7 +195,7 @@
         "command": "/opt/maka-node-toolchain/bin/node",
         "credentialEnvironment": { "OPENAI_API_KEY": "DEEPSEEK_API_KEY" },
         "args": [
-          "/opt/maka-agent/packages/eval/dist/harbor-external-subject.js",
+          "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js",
           "opencode",
           "https://api.deepseek.com",
           "/",
@@ -223,7 +223,7 @@
         "command": "/opt/maka-node-toolchain/bin/node",
         "credentialEnvironment": { "OPENAI_API_KEY": "DEEPSEEK_API_KEY" },
         "args": [
-          "/opt/maka-agent/packages/eval/dist/harbor-external-subject.js",
+          "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js",
           "kimi-code",
           "https://api.deepseek.com",
           "/",
@@ -244,7 +244,7 @@
         "command": "/opt/maka-node-toolchain/bin/node",
         "credentialEnvironment": { "OPENAI_API_KEY": "DEEPSEEK_API_KEY" },
         "args": [
-          "/opt/maka-agent/packages/eval/dist/harbor-external-subject.js",
+          "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js",
           "zcode",
           "https://api.deepseek.com/v1",
           "/",
@@ -267,7 +267,7 @@
         "command": "/opt/maka-node-toolchain/bin/node",
         "credentialEnvironment": { "OPENAI_API_KEY": "DEEPSEEK_API_KEY" },
         "args": [
-          "/opt/maka-agent/packages/eval/dist/harbor-external-subject.js",
+          "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js",
           "pi",
           "https://api.deepseek.com",
           "/",

packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-four-arm.json

@@ -30,8 +30,8 @@
       ],
       "egressProxy": {
         "composeSourceEnv": "MAKA_EVAL_MAKA_BUNDLE_PATH",
-        "composeRelativePath": "packages/eval/harbor/docker-compose-egress-proxy.yaml",
-        "networkPolicyRelativePath": "packages/eval/harbor/egress-proxy/network-policy",
+        "composeRelativePath": "node_modules/@maka/eval/harbor/docker-compose-egress-proxy.yaml",
+        "networkPolicyRelativePath": "node_modules/@maka/eval/harbor/egress-proxy/network-policy",
         "proxyUrl": "http://maka-eval-mitmproxy:8080",
         "allowedHost": "maka-eval-mitmproxy",
         "containerCaPath": "/opt/maka-egress/mitmproxy-ca-cert.pem"
@@ -73,7 +73,7 @@
       "credentials": ["OPENAI_API_KEY"],
       "config": {
         "nodePath": "/opt/maka-node-toolchain/bin/node",
-        "shimPath": "/opt/maka-agent/packages/eval/dist/harbor-maka-subject.js",
+        "shimPath": "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-maka-subject.js",
         "runtimeHostsPath": "/tmp/maka-runtime-hosts",
         "connectionSlug": "env-openai",
         "baseUrl": "https://api.deepseek.com",
@@ -93,7 +93,7 @@
       "config": {
         "command": "/opt/maka-node-toolchain/bin/node",
         "args": [
-          "/opt/maka-agent/packages/eval/dist/harbor-external-subject.js",
+          "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js",
           "codex",
           "https://api.deepseek.com",
           "/",
@@ -114,7 +114,7 @@
       "config": {
         "command": "/opt/maka-node-toolchain/bin/node",
         "args": [
-          "/opt/maka-agent/packages/eval/dist/harbor-external-subject.js",
+          "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js",
           "claude-code",
           "https://api.deepseek.com/anthropic",
           "/",
@@ -140,7 +140,7 @@
       "config": {
         "command": "/opt/maka-node-toolchain/bin/node",
         "args": [
-          "/opt/maka-agent/packages/eval/dist/harbor-external-subject.js",
+          "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js",
           "reasonix",
           "https://api.deepseek.com",
           "/",

packages/eval/experiments/terminal-bench-2.1-deepseek-v4-flash-maka-vs-deepseek-harness.json

@@ -34,8 +34,8 @@
       ],
       "egressProxy": {
         "composeSourceEnv": "MAKA_EVAL_MAKA_BUNDLE_PATH",
-        "composeRelativePath": "packages/eval/harbor/docker-compose-egress-proxy.yaml",
-        "networkPolicyRelativePath": "packages/eval/harbor/egress-proxy/network-policy",
+        "composeRelativePath": "node_modules/@maka/eval/harbor/docker-compose-egress-proxy.yaml",
+        "networkPolicyRelativePath": "node_modules/@maka/eval/harbor/egress-proxy/network-policy",
         "proxyUrl": "http://maka-eval-mitmproxy:8080",
         "allowedHost": "maka-eval-mitmproxy",
         "containerCaPath": "/opt/maka-egress/mitmproxy-ca-cert.pem"
@@ -69,7 +69,7 @@
       "credentials": ["DEEPSEEK_API_KEY"],
       "config": {
         "nodePath": "/opt/maka-node-toolchain/bin/node",
-        "shimPath": "/opt/maka-agent/packages/eval/dist/harbor-maka-subject.js",
+        "shimPath": "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-maka-subject.js",
         "runtimeHostsPath": "/tmp/maka-runtime-hosts",
         "hostSettlementTimeoutMs": 120000,
         "connectionSlug": "env-deepseek",
@@ -92,7 +92,7 @@
           "DEEPSEEK_API_KEY": "DEEPSEEK_API_KEY"
         },
         "args": [
-          "/opt/maka-agent/packages/eval/dist/harbor-external-subject.js",
+          "/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js",
           "deepseek-harness",
           "https://api.deepseek.com/v1",
           "/",

packages/eval/package.json

@@ -7,9 +7,7 @@
   "releaseFiles": [
     "dist",
     "harbor/deepseek-codex-models.json",
-    "harbor/deepseek-harness-profile/cordis.patch.yml",
-    "harbor/deepseek-harness-profile/cordis.yml",
-    "harbor/deepseek-harness-profile/package.json",
+    "harbor/deepseek-harness-profile",
     "harbor/docker-compose-egress-proxy.yaml",
     "harbor/egress-proxy/Dockerfile",
     "harbor/egress-proxy/entrypoint.sh",

packages/eval/src/__tests__/external-subject.test.ts

@@ -405,7 +405,7 @@ test('refuses a mounted toolchain that is not the pinned tree', async () => {
       ...externalCell({
         command: '/opt/maka-node-toolchain/bin/node',
         args: [
-          '/opt/maka-agent/packages/eval/dist/harbor-external-subject.js',
+          '/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js',
           'codex',
           'https://api.deepseek.com',
           '/',
@@ -469,7 +469,7 @@ test('a verified toolchain reaches the subject and its later attempts', async ()
       ...externalCell({
         command: '/opt/maka-node-toolchain/bin/node',
         args: [
-          '/opt/maka-agent/packages/eval/dist/harbor-external-subject.js',
+          '/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js',
           'codex',
           'https://api.deepseek.com',
           '/',

packages/eval/src/__tests__/lifecycle-boundaries.test.ts

@@ -828,7 +828,10 @@ test('eight-arm spec and wrappers freeze the working provider contracts', async
   };
   // The DeepSeek Harness arm copies its checked-in profile out of the repo
   // mount, so the wrapper needs to find it under the fake system root.
-  const profileSource = join(root, 'opt/maka-agent/packages/eval/harbor/deepseek-harness-profile');
+  const profileSource = join(
+    root,
+    'opt/maka-agent/node_modules/@maka/eval/harbor/deepseek-harness-profile',
+  );
   await mkdir(profileSource, { recursive: true });
   for (const file of ['package.json', 'cordis.yml', 'cordis.patch.yml']) {
     await copyFile(
@@ -998,8 +1001,8 @@ test('eight-arm spec adds Pi with the same pinned DeepSeek execution contract',
   );
   assert.deepEqual(spec.executor.config.egressProxy, {
     composeSourceEnv: 'MAKA_EVAL_MAKA_BUNDLE_PATH',
-    composeRelativePath: 'packages/eval/harbor/docker-compose-egress-proxy.yaml',
-    networkPolicyRelativePath: 'packages/eval/harbor/egress-proxy/network-policy',
+    composeRelativePath: 'node_modules/@maka/eval/harbor/docker-compose-egress-proxy.yaml',
+    networkPolicyRelativePath: 'node_modules/@maka/eval/harbor/egress-proxy/network-policy',
     proxyUrl: 'http://maka-eval-mitmproxy:8080',
     allowedHost: 'maka-eval-mitmproxy',
     containerCaPath: '/opt/maka-egress/mitmproxy-ca-cert.pem',
@@ -1310,8 +1313,8 @@ function experiment(): ExperimentSpec {
 test('pier cannot declare an egress proxy it never enforces', () => {
   const egressProxy = {
     composeSourceEnv: 'MAKA_TEST_BUNDLE',
-    composeRelativePath: 'packages/eval/harbor/docker-compose-egress-proxy.yaml',
-    networkPolicyRelativePath: 'packages/eval/harbor/egress-proxy/network-policy',
+    composeRelativePath: 'node_modules/@maka/eval/harbor/docker-compose-egress-proxy.yaml',
+    networkPolicyRelativePath: 'node_modules/@maka/eval/harbor/egress-proxy/network-policy',
     proxyUrl: 'http://maka-eval-mitmproxy:8080',
     allowedHost: 'maka-eval-mitmproxy',
     containerCaPath: '/opt/maka-egress/mitmproxy-ca-cert.pem',

packages/eval/src/external-subject.ts

@@ -33,7 +33,8 @@ import {
   verifyToolchainDirectory,
 } from './toolchain-verification.js';
 
-const BUNDLED_EXTERNAL_WRAPPER = '/opt/maka-agent/packages/eval/dist/harbor-external-subject.js';
+const BUNDLED_EXTERNAL_WRAPPER =
+  '/opt/maka-agent/node_modules/@maka/eval/dist/harbor-external-subject.js';
 
 export function createExternalSubjectAdapter(): SubjectAdapter {
   const verifiedToolchains = new Map<string, ToolchainIdentity>();

packages/eval/src/harbor-external-subject.ts

@@ -69,7 +69,10 @@ const PROFILE_PREPARERS: Record<Profile, (setup: ProfileSetup) => Promise<string
   codex: async ({ env, home, root, proxyBaseUrl }) => {
     env.OPENAI_API_KEY = 'maka-eval-local';
     env.CODEX_HOME = home;
-    const catalog = rooted(root, '/opt/maka-agent/packages/eval/harbor/deepseek-codex-models.json');
+    const catalog = rooted(
+      root,
+      '/opt/maka-agent/node_modules/@maka/eval/harbor/deepseek-codex-models.json',
+    );
     await writeFile(
       join(home, 'config.toml'),
       [
@@ -261,7 +264,10 @@ const PROFILE_PREPARERS: Record<Profile, (setup: ProfileSetup) => Promise<string
     env.DSH_HOME = join(home, 'dsh');
     const profile = join(env.DSH_HOME, 'profiles', DEEPSEEK_HARNESS_PROFILE);
     await mkdir(profile, { recursive: true, mode: 0o700 });
-    const source = rooted(root, '/opt/maka-agent/packages/eval/harbor/deepseek-harness-profile');
+    const source = rooted(
+      root,
+      '/opt/maka-agent/node_modules/@maka/eval/harbor/deepseek-harness-profile',
+    );
     for (const file of ['package.json', 'cordis.yml', 'cordis.patch.yml']) {
       await copyFile(join(source, file), join(profile, file));
     }

scripts/release-cli-eval-package.mjs

@@ -66,7 +66,10 @@ try {
 
   const maka = join(prefix, 'bin/maka');
   const packageRoot = join(prefix, 'lib/node_modules/maka-agent');
-  if (!existsSync(maka) || !existsSync(join(packageRoot, 'packages/eval/harbor/run_trial.py'))) {
+  if (
+    !existsSync(maka) ||
+    !existsSync(join(packageRoot, 'node_modules/@maka/eval/harbor/run_trial.py'))
+  ) {
     throw new Error('The installed candidate is missing its CLI or bundled Eval runtime');
   }
 

scripts/release-cli-file-policy.mjs

@@ -161,9 +161,9 @@ export function resolveWorkspaceReleaseFiles(directory, manifest) {
           `${manifest.name ?? 'Workspace package'} release dist must be a directory.`,
         );
       }
-    } else if (!entry.isFile()) {
+    } else if (!entry.isFile() && !entry.isDirectory()) {
       throw new Error(
-        `${manifest.name ?? 'Workspace package'} release asset must be a regular file: ${releaseFile}`,
+        `${manifest.name ?? 'Workspace package'} release asset must be a regular file or directory: ${releaseFile}`,
       );
     }
   }

scripts/release-cli-file-policy.test.mjs

@@ -148,18 +148,28 @@ describe('CLI release file policy', () => {
     }
   });
 
-  test('release file declarations reject directories other than dist', () => {
+  test('release file declarations accept a declared directory and reject non-regular entries', () => {
     const workspace = mkdtempSync(join(tmpdir(), 'maka-release-files-'));
     try {
       mkdirSync(join(workspace, 'dist'));
       mkdirSync(join(workspace, 'runtime-assets'));
+      writeFileSync(join(workspace, 'runtime-assets', 'profile.yml'), 'a: 1');
+      // A declared directory is allowed (the whole tree ships) ...
+      assert.deepEqual(
+        resolveWorkspaceReleaseFiles(workspace, {
+          name: '@maka/example',
+          releaseFiles: ['dist', 'runtime-assets'],
+        }),
+        ['dist', 'runtime-assets'],
+      );
+      // ... but a missing entry is still rejected.
       assert.throws(
         () =>
           resolveWorkspaceReleaseFiles(workspace, {
             name: '@maka/example',
-            releaseFiles: ['dist', 'runtime-assets'],
+            releaseFiles: ['dist', 'absent'],
           }),
-        /must be a regular file/u,
+        /release file is missing/u,
       );
     } finally {
       rmSync(workspace, { recursive: true, force: true });