CP-32687: detect missing tables during update/insert and treat as fatal (#452)

We have seen a couple of instances where the SQLite tables don't seem
to be created during initialization, leading to an infinite stream of
errors when we attempt to write data to the table.

This patch will simply detect these errors, and change the log event
from a warning to a fatal, causing the process to exit, which will
give Kubernetes a chance to restart the pod and initialize a new
database, this time hopefully *with* the missing table(s).

Author: evan-cz

app/storage/core/errors.go

@@ -5,13 +5,15 @@ package core
 
 import (
 	"errors"
+	"strings"
 
+	"github.com/mattn/go-sqlite3"
 	"gorm.io/gorm"
 
 	"github.com/cloudzero/cloudzero-agent/app/types"
 )
 
-// TranslateError maps GORM errors to application-specific errors.
+// TranslateError maps GORM / SQLite errors to application-specific errors.
 // If the error does not match any known GORM errors, it returns the original error.
 func TranslateError(err error) error {
 	if errors.Is(err, gorm.ErrRecordNotFound) {
@@ -62,5 +64,18 @@ func TranslateError(err error) error {
 	case errors.Is(err, gorm.ErrCheckConstraintViolated):
 		return types.ErrCheckConstraintViolated
 	}
+
+	// Check for SQLite-specific "no such table" errors
+	// This is a fatal error that should cause the application to exit
+	var sqliteErr sqlite3.Error
+	if errors.As(err, &sqliteErr) {
+		// SQLITE_ERROR (1) with "no such table" message indicates missing table
+		if sqliteErr.Code == sqlite3.ErrError {
+			if strings.HasPrefix(sqliteErr.Error(), "no such table: ") {
+				return types.ErrTableMissing
+			}
+		}
+	}
+
 	return err
 }

app/storage/core/errors_test.go

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) 2016-2025, CloudZero, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package core_test
+
+import (
+	"testing"
+
+	"github.com/cloudzero/cloudzero-agent/app/storage/core"
+	"github.com/cloudzero/cloudzero-agent/app/storage/sqlite"
+	"github.com/cloudzero/cloudzero-agent/app/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTranslateError_SQLiteTableMissing(t *testing.T) {
+	// Create an in-memory SQLite database
+	db, err := sqlite.NewSQLiteDriver(sqlite.MemorySharedCached)
+	if err != nil {
+		t.Fatalf("Failed to create database: %v", err)
+	}
+
+	// Try to query a non-existent table to get the real SQLite error
+	var results []struct {
+		ID   uint `gorm:"primaryKey"`
+		Name string
+	}
+	err = db.Table("resource_tags").Find(&results).Error
+
+	// Verify we got the expected SQLite error
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "no such table: resource_tags")
+
+	// Test our TranslateError function
+	translatedErr := core.TranslateError(err)
+	assert.Error(t, translatedErr)
+	assert.Equal(t, types.ErrTableMissing, translatedErr)
+}
+
+func TestTranslateError_UnknownError(t *testing.T) {
+	// Test that unknown errors are passed through unchanged
+	unknownErr := assert.AnError
+	result := core.TranslateError(unknownErr)
+	assert.Equal(t, unknownErr, result)
+}

app/storage/repo/resource_store_impl.go

@@ -10,6 +10,7 @@ package repo
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"sync"
 
@@ -20,6 +21,7 @@ import (
 	"github.com/cloudzero/cloudzero-agent/app/storage/sqlite"
 	"github.com/cloudzero/cloudzero-agent/app/types"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 )
 
@@ -37,6 +39,15 @@ var (
 	)
 )
 
+// createDBErrorEvent returns a logger event with the appropriate level based on the error type.
+// Fatal errors (like missing tables) will cause the application to exit.
+func createDBErrorEvent(logger *zerolog.Logger, translatedErr error) *zerolog.Event {
+	if errors.Is(translatedErr, types.ErrTableMissing) {
+		return logger.Fatal() //nolint:zerologlint // Caller will dispatch the event
+	}
+	return logger.Warn() //nolint:zerologlint // Caller will dispatch the event
+}
+
 // NewInMemoryResourceRepository creates a new in-memory resource repository.
 func NewInMemoryResourceRepository(clock types.TimeProvider) (types.ResourceStore, error) {
 	remoteWriteStatsOnce.Do(func() {
@@ -100,14 +111,18 @@ func (r *resourceRepoImpl) Create(ctx context.Context, it *types.ResourceTags) e
 			DoNothing: true,
 		}).Create(it).Error
 	if err != nil {
-		log.Ctx(ctx).Warn().Err(err).Msg("storage write create failure")
+		translatedErr := core.TranslateError(err)
+
+		// Create log entry with appropriate level
+		createDBErrorEvent(log.Ctx(ctx), translatedErr).Msg("storage write create failure")
+
 		StorageWriteFailures.With(prometheus.Labels{
 			"action":        "create",
 			"resource_type": fmt.Sprintf("%d", it.Type),
 			"namespace":     *it.Namespace,
 			"resource_name": it.Name,
 		}).Inc()
-		return core.TranslateError(err)
+		return translatedErr
 	}
 	return nil
 }
@@ -179,14 +194,18 @@ func (r *resourceRepoImpl) Update(ctx context.Context, it *types.ResourceTags) e
 	if err := r.DB(ctx).Model(it).
 		Where("id = ?", it.ID).
 		Updates(updates).Error; err != nil {
-		log.Ctx(ctx).Warn().Err(err).Msg("storage write update failure")
+		translatedErr := core.TranslateError(err)
+
+		// Create log entry with appropriate level
+		createDBErrorEvent(log.Ctx(ctx), translatedErr).Msg("storage write update failure")
+
 		StorageWriteFailures.With(prometheus.Labels{
 			"action":        "update",
 			"resource_type": fmt.Sprintf("%d", it.Type),
 			"namespace":     *it.Namespace,
 			"resource_name": it.Name,
 		}).Inc()
-		return core.TranslateError(err)
+		return translatedErr
 	}
 	return nil
 }

app/types/errors.go

@@ -108,4 +108,7 @@ var (
 
 	// ErrCheckConstraintViolated is returned when a check constraint is violated.
 	ErrCheckConstraintViolated = errors.New("check constraint violated")
+
+	// ErrTableMissing is returned when a required database table does not exist.
+	ErrTableMissing = errors.New("required database table missing")
 )

go.mod

@@ -208,7 +208,7 @@ require (
 	github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
-	github.com/mattn/go-sqlite3 v1.14.22 // indirect
+	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/microcosm-cc/bluemonday v1.0.27
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.66.1