A small Go HTTP service that finds the main visual subject in an
image. It runs U²-Net with ONNX Runtime and returns the
- saliency-weighted centroid as normalized X/Y coordinates. It never
+ weighted centroid of the strongest salient region as normalized X/Y coordinates. It never
crops, stores, or modifies the submitted image.
@@ -120,7 +120,7 @@ function DocsPage() {
diff --git a/internal/gravity/gravity.go b/internal/gravity/gravity.go
index c445f9c..57456cc 100644
--- a/internal/gravity/gravity.go
+++ b/internal/gravity/gravity.go
@@ -13,15 +13,20 @@ type Point struct {
Y float64 `json:"y"`
}
-// FromSaliency returns the saliency-weighted centroid and the peak saliency
-// value as confidence. Values that are negative, NaN, or infinite contribute
-// no weight. A map with no usable saliency falls back to the image center.
+// FromSaliency returns the centroid of the strongest connected salient region
+// and the peak saliency value as confidence. Values that are negative, NaN, or
+// infinite contribute no weight. A map with no usable saliency falls back to
+// the image center.
func FromSaliency(saliency []float32, width, height int) (Point, float64, error) {
return FromSaliencyRegion(saliency, width, height, image.Rect(0, 0, width, height))
}
// FromSaliencyRegion calculates a focal point from a rectangular image region
// within a larger saliency map, ignoring any surrounding letterbox padding.
+// Pixels at least half as salient as the peak are grouped into 8-connected
+// components. The component with the greatest total saliency is selected so
+// separated subjects do not produce a focal point in the empty space between
+// them.
func FromSaliencyRegion(saliency []float32, width, height int, region image.Rectangle) (Point, float64, error) {
if width <= 0 || height <= 0 || len(saliency) != width*height {
return Point{}, 0, errors.New("invalid saliency map dimensions")
@@ -31,16 +36,13 @@ func FromSaliencyRegion(saliency []float32, width, height int, region image.Rect
return Point{}, 0, errors.New("invalid saliency map region")
}
- var total, weightedX, weightedY, peak float64
+ var peak float64
for y := region.Min.Y; y < region.Max.Y; y++ {
for x := region.Min.X; x < region.Max.X; x++ {
weight := float64(saliency[y*width+x])
if weight <= 0 || math.IsNaN(weight) || math.IsInf(weight, 0) {
continue
}
- total += weight
- weightedX += float64(x-region.Min.X) * weight
- weightedY += float64(y-region.Min.Y) * weight
if weight > peak {
peak = weight
}
@@ -48,21 +50,74 @@ func FromSaliencyRegion(saliency []float32, width, height int, region image.Rect
}
confidence := clamp01(peak)
- if total == 0 {
+ if peak == 0 {
return Point{X: 0.5, Y: 0.5}, confidence, nil
}
+ type component struct {
+ total float64
+ weightedX, weightedY float64
+ }
+ threshold := peak * 0.5
+ visited := make([]bool, len(saliency))
+ var strongest component
+ directions := [...]image.Point{
+ {X: -1, Y: -1}, {X: 0, Y: -1}, {X: 1, Y: -1},
+ {X: -1, Y: 0}, {X: 1, Y: 0},
+ {X: -1, Y: 1}, {X: 0, Y: 1}, {X: 1, Y: 1},
+ }
+ for y := region.Min.Y; y < region.Max.Y; y++ {
+ for x := region.Min.X; x < region.Max.X; x++ {
+ index := y*width + x
+ if visited[index] || !usableAtThreshold(saliency[index], threshold) {
+ continue
+ }
+ visited[index] = true
+ queue := []image.Point{{X: x, Y: y}}
+ var current component
+ for len(queue) > 0 {
+ point := queue[len(queue)-1]
+ queue = queue[:len(queue)-1]
+ weight := float64(saliency[point.Y*width+point.X])
+ current.total += weight
+ current.weightedX += float64(point.X-region.Min.X) * weight
+ current.weightedY += float64(point.Y-region.Min.Y) * weight
+
+ for _, direction := range directions {
+ neighbor := point.Add(direction)
+ if !neighbor.In(region) {
+ continue
+ }
+ neighborIndex := neighbor.Y*width + neighbor.X
+ if visited[neighborIndex] || !usableAtThreshold(saliency[neighborIndex], threshold) {
+ continue
+ }
+ visited[neighborIndex] = true
+ queue = append(queue, neighbor)
+ }
+ }
+ if current.total > strongest.total {
+ strongest = current
+ }
+ }
+ }
+
x, y := 0.5, 0.5
if region.Dx() > 1 {
- x = weightedX / total / float64(region.Dx()-1)
+ x = strongest.weightedX / strongest.total / float64(region.Dx()-1)
}
if region.Dy() > 1 {
- y = weightedY / total / float64(region.Dy()-1)
+ y = strongest.weightedY / strongest.total / float64(region.Dy()-1)
}
return Point{X: clamp01(x), Y: clamp01(y)}, confidence, nil
}
+func usableAtThreshold(value float32, threshold float64) bool {
+ weight := float64(value)
+ return weight >= threshold && !math.IsNaN(weight) && !math.IsInf(weight, 0)
+}
+
func clamp01(v float64) float64 {
if v < 0 {
return 0
diff --git a/internal/gravity/gravity_test.go b/internal/gravity/gravity_test.go
index 932f418..97e6b25 100644
--- a/internal/gravity/gravity_test.go
+++ b/internal/gravity/gravity_test.go
@@ -24,11 +24,33 @@ func TestFromSaliency(t *testing.T) {
confidence: 1,
},
{
- name: "weighted centroid",
- mapData: []float32{1, 0, 0, 3},
- width: 2,
- height: 2,
- want: Point{X: 0.75, Y: 0.75},
+ name: "strongest disconnected region",
+ mapData: []float32{1, 0, 3},
+ width: 3,
+ height: 1,
+ want: Point{X: 1, Y: 0.5},
+ confidence: 1,
+ },
+ {
+ name: "weighted centroid within strongest region",
+ mapData: []float32{1, 1, 0, 0, 0, 0, 0.75, 0, 0},
+ width: 3,
+ height: 3,
+ want: Point{X: 0.25, Y: 0},
+ confidence: 1,
+ },
+ {
+ name: "diagonally connected pixels form one region",
+ mapData: []float32{
+ 1, 0, 0, 0, 0.75,
+ 0, 1, 0, 0, 0.75,
+ 0, 0, 1, 0, 0,
+ 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0,
+ },
+ width: 5,
+ height: 5,
+ want: Point{X: 0.25, Y: 0.25},
confidence: 1,
},
{
@@ -63,6 +85,35 @@ func TestFromSaliencyRejectsInvalidDimensions(t *testing.T) {
}
}
+// Regression for panoramic images with two distant subjects. Computing one
+// centroid across both regions places gravity in the empty middle; selecting
+// the region with greater integrated saliency keeps it on a subject.
+func TestFromSaliencySelectsStrongestSeparatedSubject(t *testing.T) {
+ const width, height = 11, 5
+ mapData := make([]float32, width*height)
+
+ // Both subjects reach the same peak confidence, but the left subject has
+ // greater saliency mass (6.0 versus 4.0).
+ for _, point := range []image.Point{{1, 1}, {2, 1}, {1, 2}, {2, 2}, {1, 3}, {2, 3}} {
+ mapData[point.Y*width+point.X] = 1
+ }
+ for _, point := range []image.Point{{8, 1}, {9, 1}, {8, 2}, {9, 2}} {
+ mapData[point.Y*width+point.X] = 1
+ }
+
+ got, confidence, err := FromSaliency(mapData, width, height)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := Point{X: 0.15, Y: 0.5}
+ if !closeEnough(got.X, want.X) || !closeEnough(got.Y, want.Y) {
+ t.Fatalf("FromSaliency() = %+v, want strongest left subject at %+v", got, want)
+ }
+ if confidence != 1 {
+ t.Fatalf("confidence = %v, want 1", confidence)
+ }
+}
+
func TestFromSaliencyRegionIgnoresPadding(t *testing.T) {
mapData := make([]float32, 4*4)
mapData[0] = 10 // Padding must not affect the result or confidence.