perf: improve CPU throughput and default to configurable INT8

.dockerignore

@@ -1,3 +1,4 @@
 .git
 autogravity
 models/*.onnx
+!models/u2net-int8.onnx

.gitattributes

@@ -0,0 +1,4 @@
+# Keep reproducibility data available without overwhelming code review.
+/tools/quantization/results-*.json linguist-generated=true
+/tools/quantization/dataset-*.json linguist-generated=true
+/tools/quantization/timings-*.json linguist-generated=true

.github/workflows/ci.yml

@@ -37,6 +37,9 @@ jobs:
       - name: Test real image inference
         run: mise run test-integration
 
+      - name: Test FP32 precision override
+        run: make test-integration-fp32
+
   docker:
     name: Docker build (${{ matrix.arch }})
     strategy:

.gitignore

@@ -1,3 +1,3 @@
 /autogravity
 /models/*.onnx
-
+!/models/u2net-int8.onnx

.mise.toml

@@ -3,7 +3,7 @@ go = "1.25.14"
 
 [tasks.fmt]
 description = "Check Go formatting"
-run = "test -z \"$(gofmt -l cmd internal)\""
+run = "test -z \"$(gofmt -l cmd internal tools)\""
 
 [tasks.vet]
 description = "Run static analysis"

CONTRIBUTING.md

@@ -48,6 +48,12 @@ it fails rather than silently skipping when they are missing. GitHub Actions
 installs the pinned runtime and runs this suite on every pull request and push
 to `main`. Default tests need neither the runtime nor network access.
 
+Integration tests follow `MODEL_PRECISION` (default `int8`) and honor explicit
+`MODEL_PATH` overrides. `make test-integration-fp32` tests the FP32 environment
+switch; CI runs both modes. The INT8 binary is intentionally versioned in
+`models/` and checksum-verified, so release builds do not need the experimental
+SSH host or the calibration dataset. See `models/README.md` for provenance.
+
 The gallery also includes three difficult natural scenes with manually annotated
 subject regions. Run `make evaluate` to check a person in a room, a pedestrian
 with a dog, and a bird above tree foliage. Full U²-Net passes the person-in-room
@@ -57,6 +63,17 @@ model predictions.
 
 ## Benchmarks
 
+To measure preprocessing time and Go allocations without loading ONNX Runtime:
+
+```sh
+go test ./internal/imageutil -run '^$' -bench BenchmarkPrepare -benchmem -count=3
+```
+
+`BenchmarkPrepareInto` measures the reusable input-buffer path used by the HTTP
+handler. `BenchmarkPrepare` includes input-buffer allocation. The already-sized
+case isolates normalization from resizing. These are preprocessing measurements,
+not end-to-end inference throughput.
+
 The benchmark uses the included landscape JPEG and portrait PNG. It measures image decoding, orientation handling, resize and normalization, ONNX
 inference, and focal-point calculation. Model startup is excluded.
 
@@ -83,6 +100,12 @@ ONNX Runtime version. The fixtures are synthetic, and the benchmark runs
 analyses sequentially. File reads, HTTP handling, uploads, and model startup
 are excluded.
 
+To tune production throughput, benchmark `MAX_CONCURRENT_ANALYSES` values such
+as 1, 2, 4, and 8 under an HTTP workload while recording requests/second,
+latency percentiles, and peak memory. Test `ONNX_INTRA_OP_THREADS` alongside it:
+the default of 1 is intended for concurrent requests, while a higher value may
+reduce single-request latency when more CPU cores are available.
+
 ## Layout
 
 ```text

Dockerfile

@@ -16,6 +16,9 @@ RUN mkdir -p /opt/onnxruntime \
     && tar -xzf /tmp/onnxruntime.tgz --strip-components=1 -C /opt/onnxruntime
 ADD --checksum=sha256:8d10d2f3bb75ae3b6d527c77944fc5e7dcd94b29809d47a739a7a728a912b491 --chmod=0444 \
     https://github.com/danielgatis/rembg/releases/download/v0.0.0/u2net.onnx /opt/models/u2net.onnx
+COPY models/u2net-int8.onnx /opt/models/u2net-int8.onnx
+COPY models/README.md models/U2NET_LICENSE /opt/models/
+RUN echo "b340186f56660b6665e494aab912e5f8e9adbc2317181c77fd01aa226f06553b  /opt/models/u2net-int8.onnx" | sha256sum -c -
 
 WORKDIR /src
 COPY go.mod go.sum ./
@@ -31,8 +34,9 @@ COPY --from=builder /out/autogravity /usr/local/bin/autogravity
 COPY --chmod=0555 healthcheck.sh /usr/local/bin/healthcheck
 COPY --from=builder /opt/onnxruntime/lib /opt/onnxruntime/lib
 COPY --from=builder /opt/models /opt/models
+WORKDIR /opt
 ENV ADDR=:8080 \
-    MODEL_PATH=/opt/models/u2net.onnx \
+    MODEL_PRECISION=int8 \
     ONNXRUNTIME_LIB=/opt/onnxruntime/lib/libonnxruntime.so.1.23.2
 USER 65532:65532
 EXPOSE 8080

Makefile

@@ -1,8 +1,8 @@
-MODEL_PATH := models/u2net.onnx
-MODEL_URL := https://github.com/danielgatis/rembg/releases/download/v0.0.0/u2net.onnx
-MODEL_SHA256 := 8d10d2f3bb75ae3b6d527c77944fc5e7dcd94b29809d47a739a7a728a912b491
+FP32_MODEL_PATH := models/u2net.onnx
+FP32_MODEL_URL := https://github.com/danielgatis/rembg/releases/download/v0.0.0/u2net.onnx
+FP32_MODEL_SHA256 := 8d10d2f3bb75ae3b6d527c77944fc5e7dcd94b29809d47a739a7a728a912b491
 
-.PHONY: build run test test-integration evaluate model
+.PHONY: build run test test-integration test-integration-fp32 evaluate model model-fp32 model-int8
 
 build:
 	go build -o autogravity ./cmd/autogravity
@@ -16,11 +16,19 @@ test:
 test-integration: model
 	go test -race -tags=integration ./...
 
+test-integration-fp32: model-fp32
+	MODEL_PATH= MODEL_PRECISION=fp32 go test -race -tags=integration ./...
+
 evaluate: model
 	go test -tags=integration,evaluation -run TestEvaluateDifficultScenes -v ./cmd/autogravity
 
-model:
-	@if [ ! -f "$(MODEL_PATH)" ]; then \
-		curl -fL --retry 3 -o "$(MODEL_PATH)" "$(MODEL_URL)"; \
+model: model-fp32 model-int8
+
+model-int8:
+	@echo "b340186f56660b6665e494aab912e5f8e9adbc2317181c77fd01aa226f06553b  models/u2net-int8.onnx" | shasum -a 256 -c
+
+model-fp32:
+	@if [ ! -f "$(FP32_MODEL_PATH)" ]; then \
+		curl -fL --retry 3 -o "$(FP32_MODEL_PATH)" "$(FP32_MODEL_URL)"; \
 	fi
-	@echo "$(MODEL_SHA256)  $(MODEL_PATH)" | shasum -a 256 -c
+	@echo "$(FP32_MODEL_SHA256)  $(FP32_MODEL_PATH)" | shasum -a 256 -c

README.md

@@ -8,7 +8,8 @@ submitted image.
 ## Requirements
 
 - Go 1.25 or newer
-- The full U²-Net ONNX model, approximately 168 MiB (`make model` downloads and verifies it)
+- The included INT8 U²-Net model (approximately 42 MiB). `make model` verifies it
+  and downloads/verifies the FP32 fallback (approximately 168 MiB).
 - An ONNX Runtime shared library. Version 1.23.2 is used by the Docker image and
   matches the pinned Go binding.
 
@@ -34,12 +35,28 @@ The server listens on `:8080`. These environment variables are available:
 | Variable | Default | Purpose |
 | --- | --- | --- |
 | `ADDR` | `:8080` | HTTP listen address |
-| `MODEL_PATH` | `models/u2net.onnx` | U²-Net model path |
+| `MODEL_PRECISION` | `int8` | `int8` or `fp32`, on every architecture |
+| `MODEL_PATH` | unset | Explicit model path; overrides `MODEL_PRECISION` |
 | `ONNXRUNTIME_LIB` | required | Full ONNX Runtime shared-library path |
+| `MAX_CONCURRENT_ANALYSES` | available CPUs | Maximum images decoded and inferred concurrently |
+| `ONNX_INTRA_OP_THREADS` | `1` | CPU threads used within each ONNX operator |
 
 ## Performance
 
-On an Apple M3 Pro, image analysis takes about **290–390 ms per image** with full U²-Net
+INT8 is the default on both amd64 and arm64. Set `MODEL_PRECISION=fp32` to use
+FP32, or `MODEL_PATH` for a custom artifact. Unknown precision values fail
+startup unless an explicit model path is supplied; there is no silent fallback.
+Native runs select `models/u2net-int8.onnx` or `models/u2net.onnx` relative to
+the working directory. Containers use the same selection under `/opt`.
+
+On the tested x86 Xeon, INT8 delivered **38–45% more HTTP throughput** at four
+CPUs and approximately **64–65% lower peak container memory**. Across 200
+held-out public images, median focal-point shift was 0.10%, p95 0.91%, and the
+worst shift 7.4% of an image dimension. These throughput gains are not established
+for ARM64. The calibration and evaluation provenance is documented with the
+model artifact in [models/README.md](models/README.md).
+
+On an Apple M3 Pro, historical **FP32** image analysis takes about **290–390 ms per image** with full U²-Net
 and CPU-only ONNX Runtime:
 
 | Input | Dimensions | Time per image |
@@ -57,15 +74,25 @@ file reads, uploads, and HTTP overhead. Performance varies with hardware and
 input images; see [benchmark instructions](CONTRIBUTING.md#benchmarks) to measure
 your environment.
 
+By default, the server runs one analysis per effective CPU using one shared
+model session. With Go 1.25, this respects Linux container CPU limits through
+the runtime's container-aware `GOMAXPROCS` setting. ONNX Runtime uses one
+intra-op thread per analysis, preventing its internal worker pool from
+multiplying with request concurrency. Set explicit CPU and memory limits for
+predictable resource use, and override `MAX_CONCURRENT_ANALYSES` when memory is
+the tighter constraint.
+
 ## Docker
 
-The image downloads the verified U²-Net model and the CPU-only ONNX Runtime
-library during the build. Docker BuildKit supports both `linux/amd64` and
-`linux/arm64`.
+The image bundles the verified INT8 model and downloads the verified FP32 model
+and CPU-only ONNX Runtime during the build. Both models are included on both
+`linux/amd64` and `linux/arm64`; selection is by environment, not architecture.
 
 ```sh
 docker build -t autogravity .
 docker run --rm -p 8080:8080 autogravity
+# Select FP32 without rebuilding:
+docker run --rm -p 8080:8080 -e MODEL_PRECISION=fp32 autogravity
 ```
 
 ### Container releases

cmd/autogravity/fixtures_test.go

@@ -68,7 +68,7 @@ func TestHandleAnalyzeFixtures(t *testing.T) {
 					result[y*saliency.InputWidth+x] = 0.75
 					return result, nil
 				})
-				app := newApplication(model)
+				app := newApplication(model, 2)
 				response := httptest.NewRecorder()
 				app.handleAnalyze(response, fixtureRequest(t, fixture, multipartBody, testimages.Read(t, fixture.Name)))
 				if response.Code != http.StatusOK {
@@ -104,7 +104,7 @@ func TestHandleAnalyzeTruncatedFixtures(t *testing.T) {
 			}
 			t.Run(name, func(t *testing.T) {
 				model := analyzerFunc(func([]float32) ([]float32, error) { t.Error("inference called for corrupt image"); return nil, nil })
-				app := newApplication(model)
+				app := newApplication(model, 2)
 				data := testimages.Read(t, fixture.Name)
 				response := httptest.NewRecorder()
 				app.handleAnalyze(response, fixtureRequest(t, fixture, multipartBody, data[:len(data)/2]))
@@ -143,7 +143,7 @@ func TestHandleAnalyzeInferenceFailureRecovery(t *testing.T) {
 					return nil, errors.New("private runtime error")
 				}
 				return make([]float32, 320*320), nil
-			}))
+			}), 2)
 			fixture := testimages.All[0]
 			for _, status := range []int{http.StatusInternalServerError, http.StatusOK} {
 				response := httptest.NewRecorder()

cmd/autogravity/integration_test.go

@@ -5,12 +5,14 @@ package main
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"image/png"
 	"math"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"sync"
 	"testing"
 
 	"autogravity/internal/imageutil"
@@ -38,6 +40,84 @@ func TestAnalyzeRealModel(t *testing.T) {
 	})
 }
 
+func TestConcurrentInferenceIsConsistent(t *testing.T) {
+	library := os.Getenv("ONNXRUNTIME_LIB")
+	if library == "" {
+		t.Fatal("integration tests require ONNXRUNTIME_LIB")
+	}
+	modelPath := os.Getenv("MODEL_PATH")
+	if modelPath == "" {
+		selected, err := configuredModelPath()
+		if err != nil {
+			t.Fatal(err)
+		}
+		modelPath = filepath.Join("..", "..", selected)
+	}
+	model, err := saliency.NewWithOptions(library, modelPath, saliency.Options{IntraOpThreads: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := model.Close(); err != nil {
+			t.Error(err)
+		}
+	})
+
+	fixture := testimages.All[0]
+	img, err := imageutil.Decode(testimages.Read(t, fixture.Name))
+	if err != nil {
+		t.Fatal(err)
+	}
+	input, _, err := imageutil.Prepare(img, saliency.InputWidth, saliency.InputHeight)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want, err := model.Infer(input)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	const workers = 4
+	var wait sync.WaitGroup
+	errors := make(chan error, workers)
+	for range workers {
+		wait.Add(1)
+		go func() {
+			defer wait.Done()
+			got, err := model.Infer(input)
+			if err != nil {
+				errors <- err
+				return
+			}
+			for i := range want {
+				if got[i] != want[i] {
+					errors <- fmt.Errorf("output[%d] = %v, want %v", i, got[i], want[i])
+					return
+				}
+			}
+		}()
+	}
+	wait.Wait()
+	close(errors)
+	for err := range errors {
+		t.Error(err)
+	}
+	// Infer returns independently owned Go memory, even after another call
+	// uses different input and the native runtime has been destroyed.
+	snapshot := append([]float32(nil), want...)
+	if _, err := model.Infer(make([]float32, len(input))); err != nil {
+		t.Fatal(err)
+	}
+	if err := model.Close(); err != nil {
+		t.Fatal(err)
+	}
+	for i := range want {
+		if want[i] != snapshot[i] {
+			t.Fatalf("returned output changed at %d after inference/Close", i)
+		}
+	}
+}
+
 // Missing prerequisites fail explicitly so CI cannot silently skip real inference.
 func runSubjectCases(t *testing.T, cases []subjectCase) {
 	t.Helper()
@@ -47,7 +127,11 @@ func runSubjectCases(t *testing.T, cases []subjectCase) {
 	}
 	modelPath := os.Getenv("MODEL_PATH")
 	if modelPath == "" {
-		modelPath = filepath.Join("..", "..", "models", "u2net.onnx")
+		selected, err := configuredModelPath()
+		if err != nil {
+			t.Fatal(err)
+		}
+		modelPath = filepath.Join("..", "..", selected)
 	}
 	model, err := saliency.New(library, modelPath)
 	if err != nil {
@@ -58,7 +142,7 @@ func runSubjectCases(t *testing.T, cases []subjectCase) {
 			t.Error(err)
 		}
 	})
-	app := newApplication(model)
+	app := newApplication(model, 2)
 	analyze := func(t *testing.T, fixture testimages.Fixture, multi bool, data []byte) analyzeResponse {
 		t.Helper()
 		response := httptest.NewRecorder()