Optimize clip image indexing from local copy

pkg/clip/clip.go

@@ -262,16 +262,21 @@ func StoreS3(storeS3Opts StoreS3Options) error {
 
 // CreateFromOCIImageOptions configures OCI image indexing
 type CreateFromOCIImageOptions struct {
-	ImageRef      string
-	OutputPath    string
-	CheckpointMiB int64
-	CredProvider  interface{}
-	ProgressChan  chan<- OCIIndexProgress // optional channel for progress updates
+	ImageRef        string                  // Source image to index (can be local)
+	StorageImageRef string                  // Optional: image reference to store in metadata (defaults to ImageRef)
+	OutputPath      string
+	CheckpointMiB   int64
+	CredProvider    interface{}
+	ProgressChan    chan<- OCIIndexProgress // optional channel for progress updates
 }
 
 // CreateFromOCIImage creates a metadata-only index (.clip) file from an OCI image
 func CreateFromOCIImage(ctx context.Context, options CreateFromOCIImageOptions) error {
-	log.Info().Msgf("creating OCI archive index from %s to %s", options.ImageRef, options.OutputPath)
+	if options.StorageImageRef != "" && options.StorageImageRef != options.ImageRef {
+		log.Info().Msgf("creating OCI archive index: indexing from %s, storing reference to %s", options.ImageRef, options.StorageImageRef)
+	} else {
+		log.Info().Msgf("creating OCI archive index from %s to %s", options.ImageRef, options.OutputPath)
+	}
 
 	if options.CheckpointMiB == 0 {
 		options.CheckpointMiB = 2 // default
@@ -287,10 +292,11 @@ func CreateFromOCIImage(ctx context.Context, options CreateFromOCIImageOptions)
 
 	archiver := NewClipArchiver()
 	err := archiver.CreateFromOCI(ctx, IndexOCIImageOptions{
-		ImageRef:      options.ImageRef,
-		CheckpointMiB: options.CheckpointMiB,
-		CredProvider:  credProvider,
-		ProgressChan:  options.ProgressChan,
+		ImageRef:        options.ImageRef,
+		StorageImageRef: options.StorageImageRef,
+		CheckpointMiB:   options.CheckpointMiB,
+		CredProvider:    credProvider,
+		ProgressChan:    options.ProgressChan,
 	}, options.OutputPath)
 
 	if err != nil {

pkg/clip/oci_indexer.go

@@ -36,10 +36,11 @@ type OCIIndexProgress struct {
 
 // IndexOCIImageOptions configures the OCI indexer
 type IndexOCIImageOptions struct {
-	ImageRef      string
-	CheckpointMiB int64                             // Checkpoint every N MiB (default 2)
-	CredProvider  common.RegistryCredentialProvider // optional credential provider for registry authentication
-	ProgressChan  chan<- OCIIndexProgress           // optional channel for progress updates
+	ImageRef        string                            // Source image to index (can be local)
+	StorageImageRef string                            // Optional: image reference to store in metadata (defaults to ImageRef)
+	CheckpointMiB   int64                             // Checkpoint every N MiB (default 2)
+	CredProvider    common.RegistryCredentialProvider // optional credential provider for registry authentication
+	ProgressChan    chan<- OCIIndexProgress           // optional channel for progress updates
 }
 
 // countingReader tracks bytes read from an io.Reader
@@ -70,16 +71,34 @@ func (ca *ClipArchiver) IndexOCIImage(ctx context.Context, opts IndexOCIImageOpt
 		opts.CheckpointMiB = 2 // default
 	}
 
-	// Parse image reference
+	// Parse image reference for fetching
 	ref, err := name.ParseReference(opts.ImageRef)
 	if err != nil {
 		return nil, nil, nil, nil, "", "", "", nil, fmt.Errorf("failed to parse image reference: %w", err)
 	}
 
-	// Extract registry and repository info
-	registryURL = ref.Context().RegistryStr()
-	repository = ref.Context().RepositoryStr()
-	reference = ref.Identifier()
+	// Determine which image reference to store in metadata
+	// If StorageImageRef is provided, use it; otherwise use ImageRef
+	storageRef := opts.ImageRef
+	if opts.StorageImageRef != "" {
+		storageRef = opts.StorageImageRef
+	}
+
+	// Parse storage reference for metadata
+	storageRefParsed, err := name.ParseReference(storageRef)
+	if err != nil {
+		return nil, nil, nil, nil, "", "", "", nil, fmt.Errorf("failed to parse storage image reference: %w", err)
+	}
+
+	// Extract registry and repository info from storage reference
+	registryURL = storageRefParsed.Context().RegistryStr()
+	repository = storageRefParsed.Context().RepositoryStr()
+	reference = storageRefParsed.Identifier()
+
+	// Log the indexing strategy
+	if storageRef != opts.ImageRef {
+		log.Info().Msgf("Indexing from local: %s, will store reference to: %s", opts.ImageRef, storageRef)
+	}
 
 	// Determine which credential provider to use
 	credProvider := opts.CredProvider

pkg/clip/oci_test.go

@@ -392,6 +392,129 @@ func TestOCIStorageReadFile(t *testing.T) {
 	}
 }
 
+// TestOCISeparateSourceAndStorageRefs tests indexing from one ref while storing another
+func TestOCISeparateSourceAndStorageRefs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	ctx := context.Background()
+
+	// Simulate the real-world scenario:
+	// - We have a local image (e.g., after build)
+	// - We want to index from local (fast, no network)
+	// - But store the remote registry reference in metadata (for later use)
+
+	// For testing, we'll use the same image but simulate different references
+	sourceRef := "docker.io/library/alpine:3.18"  // Source to index from
+	storageRef := "myregistry.example.com:5000/myapp/alpine:production-v1"  // Reference to store
+
+	tempDir := t.TempDir()
+	outputFile := filepath.Join(tempDir, "alpine-split-ref.clip")
+
+	// Create index using separate source and storage references
+	archiver := NewClipArchiver()
+	err := archiver.CreateFromOCI(ctx, IndexOCIImageOptions{
+		ImageRef:        sourceRef,
+		StorageImageRef: storageRef,
+		CheckpointMiB:   2,
+	}, outputFile)
+
+	require.NoError(t, err, "Failed to index OCI image with separate references")
+
+	// Verify output file exists
+	info, err := os.Stat(outputFile)
+	require.NoError(t, err, "Output file should exist")
+	assert.Greater(t, info.Size(), int64(0), "Output file should not be empty")
+
+	t.Logf("Created index file: %s (size: %d bytes)", outputFile, info.Size())
+
+	// Load and verify metadata
+	metadata, err := archiver.ExtractMetadata(outputFile)
+	require.NoError(t, err, "Should be able to extract metadata")
+
+	assert.NotNil(t, metadata.Index, "Index should not be nil")
+	assert.Greater(t, metadata.Index.Len(), 0, "Index should contain nodes")
+
+	// Verify storage info contains the STORAGE reference, not the source
+	require.NotNil(t, metadata.StorageInfo, "Storage info should not be nil")
+
+	ociInfo, ok := metadata.StorageInfo.(common.OCIStorageInfo)
+	if !ok {
+		ociInfoPtr, ok := metadata.StorageInfo.(*common.OCIStorageInfo)
+		require.True(t, ok, "Storage info should be OCIStorageInfo")
+		ociInfo = *ociInfoPtr
+	}
+
+	// Parse expected storage reference parts
+	expectedRegistry := "myregistry.example.com:5000"
+	expectedRepo := "myapp/alpine"
+	expectedRef := "production-v1"
+
+	// Verify metadata contains the storage reference, NOT the source reference
+	assert.Equal(t, expectedRegistry, ociInfo.RegistryURL, 
+		"Registry URL should be from storage ref, not source ref")
+	assert.Equal(t, expectedRepo, ociInfo.Repository, 
+		"Repository should be from storage ref, not source ref")
+	assert.Equal(t, expectedRef, ociInfo.Reference, 
+		"Reference should be from storage ref, not source ref")
+
+	// Verify the index was actually populated (meaning we successfully indexed from source)
+	assert.Greater(t, len(ociInfo.Layers), 0, "Should have indexed layers from source image")
+	assert.NotNil(t, ociInfo.GzipIdxByLayer, "Should have gzip indexes from source image")
+
+	t.Logf("✓ Successfully indexed from: %s", sourceRef)
+	t.Logf("✓ Metadata correctly stores: %s/%s:%s", expectedRegistry, expectedRepo, expectedRef)
+	t.Logf("✓ Index contains %d files across %d layers", metadata.Index.Len(), len(ociInfo.Layers))
+}
+
+// TestOCIDefaultStorageRef tests that StorageImageRef defaults to ImageRef when not provided
+func TestOCIDefaultStorageRef(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	ctx := context.Background()
+
+	// Test default behavior (backward compatibility)
+	imageRef := "docker.io/library/alpine:3.18"
+
+	tempDir := t.TempDir()
+	outputFile := filepath.Join(tempDir, "alpine-default.clip")
+
+	// Create index WITHOUT specifying StorageImageRef (should default to ImageRef)
+	archiver := NewClipArchiver()
+	err := archiver.CreateFromOCI(ctx, IndexOCIImageOptions{
+		ImageRef:      imageRef,
+		// StorageImageRef not specified - should default to ImageRef
+		CheckpointMiB: 2,
+	}, outputFile)
+
+	require.NoError(t, err, "Failed to index OCI image")
+
+	// Load metadata
+	metadata, err := archiver.ExtractMetadata(outputFile)
+	require.NoError(t, err, "Should be able to extract metadata")
+
+	ociInfo, ok := metadata.StorageInfo.(common.OCIStorageInfo)
+	if !ok {
+		ociInfoPtr, ok := metadata.StorageInfo.(*common.OCIStorageInfo)
+		require.True(t, ok, "Storage info should be OCIStorageInfo")
+		ociInfo = *ociInfoPtr
+	}
+
+	// Verify metadata contains the same reference as source
+	// Note: Docker normalizes "docker.io" to "index.docker.io"
+	assert.Equal(t, "index.docker.io", ociInfo.RegistryURL, 
+		"Registry should default to source ref")
+	assert.Equal(t, "library/alpine", ociInfo.Repository, 
+		"Repository should default to source ref")
+	assert.Equal(t, "3.18", ociInfo.Reference, 
+		"Reference should default to source ref")
+
+	t.Logf("✓ Default behavior works: metadata matches source reference")
+}
+
 // TestLayerCaching verifies that layers are properly cached after first read
 func TestLayerCaching(t *testing.T) {
 	if testing.Short() {