feat:func to return Count of entities in registry table w/signoff

logger/logger.go

@@ -12,6 +12,7 @@ import (
 
 type Handler interface {
 	Info(description ...interface{})
+
 	Debug(description ...interface{})
 	Warn(err error)
 	Error(err error)
@@ -84,7 +85,6 @@ func (l *Logger) Info(description ...interface{}) {
 		description...,
 	)
 }
-
 func (l *Logger) Debug(description ...interface{}) {
 	l.handler.Log(logrus.DebugLevel,
 		description...,

models/meshmodel/registry/error.go

@@ -1,13 +1,141 @@
 package registry
 
 import (
+	"fmt"
+
+	"github.com/google/uuid"
 	"github.com/layer5io/meshkit/errors"
+	"github.com/layer5io/meshkit/models/meshmodel/core/v1alpha1"
 )
 
+type EntityErrorCount struct {
+	Attempt int
+	Error   error
+}
+
+type EntityCountWithErrors struct {
+	Model        map[string]EntityErrorCount
+	Registry     map[string]EntityErrorCount
+	Component    map[string]EntityErrorCount
+	Relationship map[uuid.UUID]EntityErrorCount
+	Policy       map[uuid.UUID]EntityErrorCount
+}
+
+var RegisterAttempts EntityCountWithErrors
+
+var NonImportModel map[string]v1alpha1.EntitySummary
+
+func init() {
+	RegisterAttempts = EntityCountWithErrors{
+		Model:        make(map[string]EntityErrorCount),
+		Registry:     make(map[string]EntityErrorCount),
+		Component:    make(map[string]EntityErrorCount),
+		Relationship: make(map[uuid.UUID]EntityErrorCount),
+		Policy:       make(map[uuid.UUID]EntityErrorCount),
+	}
+	NonImportModel = make(map[string]v1alpha1.EntitySummary)
+}
+
 var (
-	ErrUnknownHostCode = "11097"
+	ErrUnknownHostCode                 = "11097"
+	ErrEmptySchemaCode                 = "11098"
+	ErrMarshalingRegisteryAttemptsCode = "11099"
+	ErrWritingRegisteryAttemptsCode    = "11100"
+	ErrRegisteringEntityCode           = "11101"
+	ErrUnknownHostInMapCode            = "11102"
 )
 
+func ErrUnknownHostInMap() error {
+	return errors.New(ErrUnknownHostCode, errors.Alert, []string{"Registrant has no error or it is not supported or unknown."}, nil, []string{"The host registering the entites has no errors with it's entites or is unknown."}, []string{"Validate the registrant name again or check /server/cmd/registery_attempts.json for futher details"})
+}
 func ErrUnknownHost(err error) error {
-	return errors.New(ErrUnknownHostCode, errors.Alert, []string{"host is not supported"}, []string{err.Error()}, []string{"The component's host is not supported by the version of server you are running"}, []string{"Try upgrading to latest available version"})
+	return errors.New(ErrUnknownHostCode, errors.Alert, []string{"Registrant type is not supported or unknown."}, []string{err.Error()}, []string{"The host registering a Model and it's components is not recognized by Meshery Server (or by the version currently running)."}, []string{"Validate the name and location of the model registrant. Try upgrading to latest available Meshery version."})
+}
+func ErrEmptySchema() error {
+	return errors.New(ErrEmptySchemaCode, errors.Alert, []string{"Empty schema for the component"}, nil, []string{"The schema is empty for the component."}, []string{"For the particular component the schema is empty. Use the docs or discussion forum for more details  "})
+}
+func ErrMarshalingRegisteryAttempts(err error) error {
+	return errors.New(ErrMarshalingRegisteryAttemptsCode, errors.Alert, []string{"Error marshaling RegisterAttempts to JSON"}, []string{"Error marshaling RegisterAttempts to JSON: ", err.Error()}, []string{}, []string{})
+}
+func ErrWritingRegisteryAttempts(err error) error {
+	return errors.New(ErrWritingRegisteryAttemptsCode, errors.Alert, []string{"Error writing RegisteryAttempts JSON data to file"}, []string{"Error writing RegisteryAttempts JSON data to file:", err.Error()}, []string{}, []string{})
+}
+func ErrRegisteringEntity(failedMsg string, hostName string) error {
+	return errors.New(ErrRegisteringEntityCode, errors.Alert, []string{fmt.Sprintf("The import process for a registrant %s encountered difficulties,due to which %s. Specific issues during the import process resulted in certain entities not being successfully registered in the table.", hostName, failedMsg)}, []string{fmt.Sprintf("For registrant %s %s", hostName, failedMsg)}, []string{"Could be because of empty schema or some issue with the json or yaml file"}, []string{"Check /server/cmd/registery_attempts.json for futher details"})
+}
+
+func onModelError(reg Registry, modelName string, h Host, err error) {
+	if entityCount, ok := RegisterAttempts.Model[modelName]; ok {
+		entityCount.Attempt++
+		RegisterAttempts.Model[modelName] = entityCount
+	} else {
+		RegisterAttempts.Model[modelName] = EntityErrorCount{Attempt: 1, Error: err}
+	}
+
+	if RegisterAttempts.Model[modelName].Attempt == 1 {
+		currentValue := NonImportModel[HostnameToPascalCase(h.Hostname)]
+		currentValue.Models++
+		NonImportModel[HostnameToPascalCase(h.Hostname)] = currentValue
+	}
+}
+
+func onRegistrantError(h Host) {
+	if entityCount, ok := RegisterAttempts.Registry[HostnameToPascalCase(h.Hostname)]; ok {
+		entityCount.Attempt++
+		RegisterAttempts.Registry[HostnameToPascalCase(h.Hostname)] = entityCount
+	} else {
+		RegisterAttempts.Registry[HostnameToPascalCase(h.Hostname)] = EntityErrorCount{Attempt: 1}
+	}
+}
+
+func onEntityError(en Entity, h Host, err error) {
+	switch entity := en.(type) {
+	case v1alpha1.ComponentDefinition:
+		entityName := entity.DisplayName
+		if err == nil {
+			err = ErrEmptySchema()
+		}
+		if entityCount, ok := RegisterAttempts.Component[entityName]; ok {
+			entityCount.Attempt++
+			RegisterAttempts.Component[entityName] = entityCount
+		} else {
+			RegisterAttempts.Component[entityName] = EntityErrorCount{Attempt: 1, Error: err}
+		}
+
+		if RegisterAttempts.Component[entityName].Attempt == 1 {
+			currentValue := NonImportModel[HostnameToPascalCase(h.Hostname)]
+			currentValue.Components++
+			NonImportModel[HostnameToPascalCase(h.Hostname)] = currentValue
+		}
+
+	case v1alpha1.RelationshipDefinition:
+		entityID := entity.ID
+		if entityCount, ok := RegisterAttempts.Relationship[entityID]; ok {
+			entityCount.Attempt++
+			RegisterAttempts.Relationship[entityID] = entityCount
+		} else {
+			RegisterAttempts.Relationship[entityID] = EntityErrorCount{Attempt: 1, Error: err}
+		}
+
+		if RegisterAttempts.Relationship[entityID].Attempt == 1 {
+			currentValue := NonImportModel[HostnameToPascalCase(h.Hostname)]
+			currentValue.Relationships++
+			NonImportModel[HostnameToPascalCase(h.Hostname)] = currentValue
+		}
+
+	case v1alpha1.PolicyDefinition:
+		entityID := entity.ID
+		if entityCount, ok := RegisterAttempts.Policy[entityID]; ok {
+			entityCount.Attempt++
+			RegisterAttempts.Policy[entityID] = entityCount
+		} else {
+			RegisterAttempts.Policy[entityID] = EntityErrorCount{Attempt: 1, Error: err}
+		}
+
+		if RegisterAttempts.Policy[entityID].Attempt == 1 {
+			currentValue := NonImportModel[HostnameToPascalCase(h.Hostname)]
+			currentValue.Policies++
+			NonImportModel[HostnameToPascalCase(h.Hostname)] = currentValue
+		}
+	}
 }

models/meshmodel/registry/host.go

@@ -28,6 +28,7 @@ type Host struct {
 func createHost(db *database.Handler, h Host) (uuid.UUID, error) {
 	byt, err := json.Marshal(h)
 	if err != nil {
+		onRegistrantError(h)
 		return uuid.UUID{}, err
 	}
 	hID := uuid.NewSHA1(uuid.UUID{}, byt)
@@ -36,6 +37,7 @@ func createHost(db *database.Handler, h Host) (uuid.UUID, error) {
 	defer hostCreationLock.Unlock()
 	err = db.First(&host, "id = ?", hID).Error // check if the host already exists
 	if err != nil && err != gorm.ErrRecordNotFound {
+		onRegistrantError(h)
 		return uuid.UUID{}, err
 	}
 
@@ -44,6 +46,7 @@ func createHost(db *database.Handler, h Host) (uuid.UUID, error) {
 		h.ID = hID
 		err = db.Create(&h).Error
 		if err != nil {
+			onRegistrantError(h)
 			return uuid.UUID{}, err
 		}
 		return h.ID, nil

models/meshmodel/registry/registry.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/google/uuid"
@@ -37,6 +38,8 @@ type Registry struct {
 	UpdatedAt    time.Time
 }
 
+var Mutex sync.Mutex
+
 // Entity is referred as any type of schema managed by the registry
 // ComponentDefinitions and PolicyDefinitions are examples of entities
 type Entity interface {
@@ -50,7 +53,7 @@ type RegistryManager struct {
 }
 
 // Registers models into registries table.
-func registerModel(db *database.Handler, regID, modelID uuid.UUID) error {
+func registerModel(db *database.Handler, regID, modelID uuid.UUID, modelName string, h Host) error {
 	entity := Registry{
 		RegistrantID: regID,
 		Entity:       modelID,
@@ -59,20 +62,23 @@ func registerModel(db *database.Handler, regID, modelID uuid.UUID) error {
 
 	byt, err := json.Marshal(entity)
 	if err != nil {
+		onModelError(entity, modelName, h, err)
 		return err
 	}
 
 	entityID := uuid.NewSHA1(uuid.UUID{}, byt)
 	var reg Registry
 	err = db.First(&reg, "id = ?", entityID).Error
 	if err != nil && err != gorm.ErrRecordNotFound {
+		onModelError(entity, modelName, h, err)
 		return err
 	}
 
 	if err == gorm.ErrRecordNotFound {
 		entity.ID = entityID
 		err = db.Create(&entity).Error
 		if err != nil {
+			onModelError(entity, modelName, h, err)
 			return err
 		}
 	}
@@ -113,25 +119,30 @@ func (rm *RegistryManager) Cleanup() {
 		&v1alpha1.RelationshipDefinitionDB{},
 	)
 }
+
 func (rm *RegistryManager) RegisterEntity(h Host, en Entity) error {
 	switch entity := en.(type) {
 	case v1alpha1.ComponentDefinition:
 		if entity.Schema == "" { //For components with an empty schema, exit quietly
+			onEntityError(entity, h, nil)
 			return nil
 		}
 
 		registrantID, err := createHost(rm.db, h)
 		if err != nil {
+			onEntityError(entity, h, err)
 			return err
 		}
 
 		componentID, modelID, err := v1alpha1.CreateComponent(rm.db, entity)
 		if err != nil {
+			onEntityError(entity, h, err)
 			return err
 		}
 
-		err = registerModel(rm.db, registrantID, modelID)
+		err = registerModel(rm.db, registrantID, modelID, entity.Model.DisplayHostName, h)
 		if err != nil {
+			onEntityError(entity, h, err)
 			return err
 		}
 
@@ -143,21 +154,28 @@ func (rm *RegistryManager) RegisterEntity(h Host, en Entity) error {
 			CreatedAt:    time.Now(),
 			UpdatedAt:    time.Now(),
 		}
-		return rm.db.Create(&entry).Error
+		err = rm.db.Create(&entry).Error
+		if err != nil {
+			onEntityError(entity, h, err)
+		}
+		return err
 	case v1alpha1.RelationshipDefinition:
 
 		registrantID, err := createHost(rm.db, h)
 		if err != nil {
+			onEntityError(entity, h, err)
 			return err
 		}
 
 		relationshipID, modelID, err := v1alpha1.CreateRelationship(rm.db, entity)
 		if err != nil {
+			onEntityError(entity, h, err)
 			return err
 		}
 
-		err = registerModel(rm.db, registrantID, modelID)
+		err = registerModel(rm.db, registrantID, modelID, entity.Model.Name, h)
 		if err != nil {
+			onEntityError(entity, h, err)
 			return err
 		}
 
@@ -169,21 +187,28 @@ func (rm *RegistryManager) RegisterEntity(h Host, en Entity) error {
 			CreatedAt:    time.Now(),
 			UpdatedAt:    time.Now(),
 		}
-		return rm.db.Create(&entry).Error
+		err = rm.db.Create(&entry).Error
+		if err != nil {
+			onEntityError(entity, h, err)
+		}
+		return err
 	//Add logic for Policies and other entities below
 	case v1alpha1.PolicyDefinition:
 		registrantID, err := createHost(rm.db, h)
 		if err != nil {
+			onEntityError(entity, h, err)
 			return err
 		}
 
 		policyID, modelID, err := v1alpha1.CreatePolicy(rm.db, entity)
 		if err != nil {
+			onEntityError(entity, h, err)
 			return err
 		}
 
-		err = registerModel(rm.db, registrantID, modelID)
+		err = registerModel(rm.db, registrantID, modelID, entity.Model.DisplayName, h)
 		if err != nil {
+			onEntityError(entity, h, err)
 			return err
 		}
 
@@ -195,13 +220,37 @@ func (rm *RegistryManager) RegisterEntity(h Host, en Entity) error {
 			CreatedAt:    time.Now(),
 			UpdatedAt:    time.Now(),
 		}
-		return rm.db.Create(&entry).Error
-
+		err = rm.db.Create(&entry).Error
+		if err != nil {
+			onEntityError(entity, h, err)
+		}
+		return err
 	default:
 		return nil
 	}
 }
+func FailedMsgCompute(failedMsg string, hostName string) (string, error) {
+	nonImportModel, exists := NonImportModel[hostName]
+	if !exists {
+		return "", ErrUnknownHostInMap()
+	}
 
+	if nonImportModel.Models > 0 || nonImportModel.Components > 0 || nonImportModel.Relationships > 0 || nonImportModel.Policies > 0 {
+		failedMsg = "failed to import"
+		appendIfNonZero := func(msg string, count int64, entityName string) string {
+			if count > 0 {
+				return fmt.Sprintf("%s %d %s", msg, count, entityName)
+			}
+			return msg
+		}
+
+		failedMsg = appendIfNonZero(failedMsg, nonImportModel.Models, "models")
+		failedMsg = appendIfNonZero(failedMsg, nonImportModel.Components, "components")
+		failedMsg = appendIfNonZero(failedMsg, nonImportModel.Relationships, "relationships")
+		failedMsg = appendIfNonZero(failedMsg, nonImportModel.Policies, "policies")
+	}
+	return failedMsg, nil
+}
 func (rm *RegistryManager) GetRegistrants(f *v1alpha1.HostFilter) ([]v1alpha1.MeshModelHostsWithEntitySummary, int64, error) {
 	var result []v1alpha1.MesheryHostSummaryDB
 	var totalcount int64
@@ -467,7 +516,6 @@ func (rm *RegistryManager) GetRegistrant(e Entity) Host {
 	_ = rm.db.Where("id = ?", reg.RegistrantID).Find(&h).Error
 	return h
 }
-
 func HostnameToPascalCase(input string) string {
 	parts := strings.Split(input, ".")
 	caser := cases.Title(language.English)