From f42268718e4946b82e6739ca597dd0f5eb77816e Mon Sep 17 00:00:00 2001 From: Eric Solender Date: Fri, 6 Nov 2020 11:19:56 -0500 Subject: [PATCH] updated to support neo4j v4 --- ...er-compose.yaml => docker-compose-v3.yaml} | 4 +- .github/docker-compose-v4.yaml | 24 + .github/workflows/go.yml | 24 +- config.go | 47 ++ index.go | 292 +----------- index_v3.go | 306 +++++++++++++ index_v4.go | 414 ++++++++++++++++++ load_strategy.go | 6 +- save.go | 16 +- 9 files changed, 833 insertions(+), 300 deletions(-) rename .github/{docker-compose.yaml => docker-compose-v3.yaml} (83%) create mode 100644 .github/docker-compose-v4.yaml create mode 100644 index_v3.go create mode 100644 index_v4.go diff --git a/.github/docker-compose.yaml b/.github/docker-compose-v3.yaml similarity index 83% rename from .github/docker-compose.yaml rename to .github/docker-compose-v3.yaml index 0e41be7..c4bf5e2 100644 --- a/.github/docker-compose.yaml +++ b/.github/docker-compose-v3.yaml @@ -2,8 +2,8 @@ version: '3' networks: lan: services: - core1: - image: neo4j:3.5.17-enterprise + core: + image: neo4j:3.5-enterprise networks: - lan ports: diff --git a/.github/docker-compose-v4.yaml b/.github/docker-compose-v4.yaml new file mode 100644 index 0000000..7ada182 --- /dev/null +++ b/.github/docker-compose-v4.yaml @@ -0,0 +1,24 @@ +version: '3' +networks: + lan: +services: + core1: + image: neo4j:4.1-enterprise + networks: + - lan + ports: + - 7474:7474 + - 6477:6477 + - 7687:7687 + environment: + - NEO4J_apoc_export_file_enabled=true + - NEO4J_apoc_import_file_enabled=true + - NEO4J_dbms_security_procedures_unrestricted=apoc.*,algo.* + - NEO4J_dbms_memory_heap_initial__size=512m + - NEO4J_dbms_memory_heap_max__size=2G + - NEO4J_apoc_uuid_enabled=true + - NEO4J_dbms_default__listen__address=0.0.0.0 + - NEO4J_dbms_allow__upgrade=true + - NEO4J_dbms_default__database=neo4j + - NEO4J_AUTH=neo4j/changeme + - NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \ No newline at end of file diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index b7df4d6..a8ee061 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -53,15 +53,27 @@ jobs: run: go build -v . - name: Run Unit Tests run: go test ./... -cover -short - - name: Start Neo4j Docker + + - name: Start Neo4j V3 Docker run: | - docker-compose -f .github/docker-compose.yaml up -d - - name: Wait for neo4j to be ready + docker-compose -f .github/docker-compose-v3.yaml up -d + - name: Wait for neo4j v3 to be ready run: | sleep 45 - - name: Run Integration Test + - name: Run Integration Test on V3 run: go test ./... -cover -run Integration - - name: Stop Neo4j Docker + - name: Stop Neo4j Docker on v3 run: | - docker-compose -f .github/docker-compose.yaml down --remove-orphans + docker-compose -f .github/docker-compose-v3.yaml down --remove-orphans + - name: Start Neo4j V4 Docker + run: | + docker-compose -f .github/docker-compose-v4.yaml up -d + - name: Wait for neo4j v4 to be ready + run: | + sleep 45 + - name: Run Integration Test on V4 + run: go test ./... -cover -run Integration + - name: Stop Neo4j Docker on v4 + run: | + docker-compose -f .github/docker-compose-v4.yaml down --remove-orphans diff --git a/config.go b/config.go index 701f732..0fea954 100644 --- a/config.go +++ b/config.go @@ -26,9 +26,12 @@ import ( "github.com/neo4j/neo4j-go-driver/neo4j" "github.com/sirupsen/logrus" "reflect" + "strconv" + "strings" ) var externalLog *logrus.Entry +var neoVersion float64 var log = getLogger() @@ -52,6 +55,10 @@ func SetLogger(logger *logrus.Entry) error { return nil } +func getIsV4() bool { + return true +} + // Config Defined GoGM config type Config struct { // Host is the neo4j host @@ -76,6 +83,7 @@ type Config struct { // Index Strategy defines the index strategy for GoGM IndexStrategy IndexStrategy `yaml:"index_strategy" json:"index_strategy" mapstructure:"index_strategy"` + TargetDbs []string `yaml:"target_dbs" json:"target_dbs" mapstructure:"target_dbs"` } // ConnectionString builds the neo4j bolt/bolt+routing connection string @@ -132,6 +140,8 @@ func Reset() { isSetup = false } +var internalConfig *Config + // internal setup logic for gogm func setupInit(isTest bool, conf *Config, mapTypes ...interface{}) error { if isSetup && !isTest { @@ -146,6 +156,18 @@ func setupInit(isTest bool, conf *Config, mapTypes ...interface{}) error { } } + if conf != nil { + if conf.TargetDbs == nil || len(conf.TargetDbs) == 0 { + conf.TargetDbs = []string{"neo4j"} + } + + internalConfig = conf + } else { + internalConfig = &Config{ + TargetDbs: []string{"neo4j"}, + } + } + log.Debug("mapping types") for _, t := range mapTypes { name := reflect.TypeOf(t).Elem().Name() @@ -176,6 +198,31 @@ func setupInit(isTest bool, conf *Config, mapTypes ...interface{}) error { if err != nil { return err } + + // get neoversion + sess, err := driver.Session(neo4j.AccessModeRead) + if err != nil { + return err + } + + res, err := sess.Run("return 1", nil) + if err != nil { + return err + } else if err = res.Err(); err != nil { + return err + } + + sum, err := res.Summary() + if err != nil { + return err + } + + // grab version + version := strings.Split(strings.Replace(strings.ToLower(sum.Server().Version()), "neo4j/", "", -1), ".") + neoVersion, err = strconv.ParseFloat(version[0], 64) + if err != nil { + return err + } } log.Debug("starting index verification step") diff --git a/index.go b/index.go index 19011de..80cbcad 100644 --- a/index.go +++ b/index.go @@ -20,306 +20,32 @@ package gogm import ( - "errors" - "fmt" - "github.com/adam-hanna/arrayOperations" "github.com/cornelk/hashmap" - dsl "github.com/mindstand/go-cypherdsl" - "github.com/neo4j/neo4j-go-driver/neo4j" ) -func resultToStringArr(res neo4j.Result) ([]string, error) { - if res == nil { - return nil, errors.New("result is nil") - } - - var result []string - - for res.Next() { - val := res.Record().Values() - // nothing to parse - if val == nil || len(val) == 0 { - continue - } - - str, ok := val[0].(string) - if !ok { - return nil, fmt.Errorf("unable to parse [%T] to string, %w", val[0], ErrInternal) - } - - result = append(result, str) - } - - return result, nil -} - //drops all known indexes func dropAllIndexesAndConstraints() error { - sess, err := driver.Session(neo4j.AccessModeWrite) - if err != nil { - return err - } - defer sess.Close() - - res, err := sess.Run("CALL db.constraints", nil) - if err != nil { - return err - } - - constraints, err := resultToStringArr(res) - if err != nil { - return err - } - - //if there is anything, get rid of it - if len(constraints) != 0 { - tx, err := sess.BeginTransaction() - if err != nil { - return err - } - - for _, constraint := range constraints { - log.Debugf("dropping constraint '%s'", constraint) - _, err := tx.Run(fmt.Sprintf("DROP %s", constraint), nil) - if err != nil { - oerr := err - err = tx.Rollback() - if err != nil { - return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) - } - - return oerr - } - } - - err = tx.Commit() - if err != nil { - return err - } - } - - res, err = sess.Run("CALL db.indexes()", nil) - if err != nil { - return err - } - - indexes, err := resultToStringArr(res) - if err != nil { - return err + if neoVersion >= 4 { + return dropAllIndexesAndConstraintsV4() } - //if there is anything, get rid of it - if len(indexes) != 0 { - tx, err := sess.BeginTransaction() - if err != nil { - return err - } - - for _, index := range indexes { - if len(index) == 0 { - return errors.New("invalid index config") - } - - _, err := tx.Run(fmt.Sprintf("DROP %s", index), nil) - if err != nil { - oerr := err - err = tx.Rollback() - if err != nil { - return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) - } - - return oerr - } - } - - return tx.Commit() - } else { - return nil - } + return dropAllIndexesAndConstraintsV3() } //creates all indexes func createAllIndexesAndConstraints(mappedTypes *hashmap.HashMap) error { - sess, err := driver.Session(neo4j.AccessModeWrite) - if err != nil { - return err - } - defer sess.Close() - - //validate that we have to do anything - if mappedTypes == nil || mappedTypes.Len() == 0 { - return errors.New("must have types to map") + if neoVersion >= 4 { + return createAllIndexesAndConstraintsV4(mappedTypes) } - numIndexCreated := 0 - - tx, err := sess.BeginTransaction() - if err != nil { - return err - } - - //index and/or create unique constraints wherever necessary - //for node, structConfig := range mappedTypes{ - for nodes := range mappedTypes.Iter() { - node := nodes.Key.(string) - structConfig := nodes.Value.(structDecoratorConfig) - if structConfig.Fields == nil || len(structConfig.Fields) == 0 { - continue - } - - var indexFields []string - - for _, config := range structConfig.Fields { - //pk is a special unique key - if config.PrimaryKey || config.Unique { - numIndexCreated++ - - cyp, err := dsl.QB().Create(dsl.NewConstraint(&dsl.ConstraintConfig{ - Unique: true, - Name: node, - Type: structConfig.Label, - Field: config.Name, - })).ToCypher() - if err != nil { - return err - } - - _, err = tx.Run(cyp, nil) - if err != nil { - oerr := err - err = tx.Rollback() - if err != nil { - return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) - } - - return oerr - } - } else if config.Index { - indexFields = append(indexFields, config.Name) - } - } - - //create composite index - if len(indexFields) > 0 { - numIndexCreated++ - cyp, err := dsl.QB().Create(dsl.NewIndex(&dsl.IndexConfig{ - Type: structConfig.Label, - Fields: indexFields, - })).ToCypher() - if err != nil { - return err - } - - _, err = tx.Run(cyp, nil) - if err != nil { - oerr := err - err = tx.Rollback() - if err != nil { - return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) - } - - return oerr - } - } - } - - log.Debugf("created (%v) indexes", numIndexCreated) - - return tx.Commit() + return createAllIndexesAndConstraintsV3(mappedTypes) } //verifies all indexes func verifyAllIndexesAndConstraints(mappedTypes *hashmap.HashMap) error { - sess, err := driver.Session(neo4j.AccessModeWrite) - if err != nil { - return err + if neoVersion >= 4 { + return verifyAllIndexesAndConstraintsV4(mappedTypes) } - defer sess.Close() - - //validate that we have to do anything - if mappedTypes == nil || mappedTypes.Len() == 0 { - return errors.New("must have types to map") - } - - var constraints []string - var indexes []string - - //build constraint strings - for nodes := range mappedTypes.Iter() { - node := nodes.Key.(string) - structConfig := nodes.Value.(structDecoratorConfig) - - if structConfig.Fields == nil || len(structConfig.Fields) == 0 { - continue - } - - fields := []string{} - - for _, config := range structConfig.Fields { - - if config.PrimaryKey || config.Unique { - t := fmt.Sprintf("CONSTRAINT ON (%s:%s) ASSERT %s.%s IS UNIQUE", node, structConfig.Label, node, config.Name) - constraints = append(constraints, t) - - indexes = append(indexes, fmt.Sprintf("INDEX ON :%s(%s)", structConfig.Label, config.Name)) - - } else if config.Index { - fields = append(fields, config.Name) - } - } - - f := "(" - for _, field := range fields { - f += field - } - - f += ")" - - indexes = append(indexes, fmt.Sprintf("INDEX ON :%s%s", structConfig.Label, f)) - - } - - //get whats there now - foundResult, err := sess.Run("CALL db.constraints", nil) - if err != nil { - return err - } - - foundConstraints, err := resultToStringArr(foundResult) - if err != nil { - return err - } - - foundInxdexResult, err := sess.Run("CALL db.indexes()", nil) - if err != nil { - return err - } - - foundIndexes, err := resultToStringArr(foundInxdexResult) - if err != nil { - return err - } - - //verify from there - delta, found := arrayOperations.Difference(foundIndexes, indexes) - if !found { - return fmt.Errorf("found differences in remote vs ogm for found indexes, %v", delta) - } - - log.Debug(delta) - - var founds []string - - for _, constraint := range foundConstraints { - founds = append(founds, constraint) - } - - delta, found = arrayOperations.Difference(founds, constraints) - if !found { - return fmt.Errorf("found differences in remote vs ogm for found constraints, %v", delta) - } - - log.Debug(delta) - return nil + return verifyAllIndexesAndConstraintsV3(mappedTypes) } diff --git a/index_v3.go b/index_v3.go new file mode 100644 index 0000000..cee44be --- /dev/null +++ b/index_v3.go @@ -0,0 +1,306 @@ +package gogm + +import ( + "errors" + "fmt" + "github.com/adam-hanna/arrayOperations" + "github.com/cornelk/hashmap" + dsl "github.com/mindstand/go-cypherdsl" + "github.com/neo4j/neo4j-go-driver/neo4j" +) + +func resultToStringArrV3(res neo4j.Result) ([]string, error) { + if res == nil { + return nil, errors.New("result is nil") + } + + var result []string + + for res.Next() { + val := res.Record().Values() + // nothing to parse + if val == nil || len(val) == 0 { + continue + } + + str, ok := val[0].(string) + if !ok { + return nil, fmt.Errorf("unable to parse [%T] to string. Value is %v: %w", val[0], val[0], ErrInternal) + } + + result = append(result, str) + } + + return result, nil +} + +//drops all known indexes +func dropAllIndexesAndConstraintsV3() error { + sess, err := driver.Session(neo4j.AccessModeWrite) + if err != nil { + return err + } + defer sess.Close() + + res, err := sess.Run("CALL db.constraints", nil) + if err != nil { + return err + } + + constraints, err := resultToStringArrV3(res) + if err != nil { + return err + } + + //if there is anything, get rid of it + if len(constraints) != 0 { + tx, err := sess.BeginTransaction() + if err != nil { + return err + } + + for _, constraint := range constraints { + log.Debugf("dropping constraint '%s'", constraint) + _, err := tx.Run(fmt.Sprintf("DROP %s", constraint), nil) + if err != nil { + oerr := err + err = tx.Rollback() + if err != nil { + return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) + } + + return oerr + } + } + + err = tx.Commit() + if err != nil { + return err + } + } + + res, err = sess.Run("CALL db.indexes()", nil) + if err != nil { + return err + } + + indexes, err := resultToStringArrV3(res) + if err != nil { + return err + } + + //if there is anything, get rid of it + if len(indexes) != 0 { + tx, err := sess.BeginTransaction() + if err != nil { + return err + } + + for _, index := range indexes { + if len(index) == 0 { + return errors.New("invalid index config") + } + + _, err := tx.Run(fmt.Sprintf("DROP %s", index), nil) + if err != nil { + oerr := err + err = tx.Rollback() + if err != nil { + return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) + } + + return oerr + } + } + + return tx.Commit() + } else { + return nil + } +} + +//creates all indexes +func createAllIndexesAndConstraintsV3(mappedTypes *hashmap.HashMap) error { + sess, err := driver.Session(neo4j.AccessModeWrite) + if err != nil { + return err + } + defer sess.Close() + + //validate that we have to do anything + if mappedTypes == nil || mappedTypes.Len() == 0 { + return errors.New("must have types to map") + } + + numIndexCreated := 0 + + tx, err := sess.BeginTransaction() + if err != nil { + return err + } + + //index and/or create unique constraints wherever necessary + //for node, structConfig := range mappedTypes{ + for nodes := range mappedTypes.Iter() { + node := nodes.Key.(string) + structConfig := nodes.Value.(structDecoratorConfig) + if structConfig.Fields == nil || len(structConfig.Fields) == 0 { + continue + } + + var indexFields []string + + for _, config := range structConfig.Fields { + //pk is a special unique key + if config.PrimaryKey || config.Unique { + numIndexCreated++ + + cyp, err := dsl.QB().Create(dsl.NewConstraint(&dsl.ConstraintConfig{ + Unique: true, + Name: node, + Type: structConfig.Label, + Field: config.Name, + })).ToCypher() + if err != nil { + return err + } + + _, err = tx.Run(cyp, nil) + if err != nil { + oerr := err + err = tx.Rollback() + if err != nil { + return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) + } + + return oerr + } + } else if config.Index { + indexFields = append(indexFields, config.Name) + } + } + + //create composite index + if len(indexFields) > 0 { + numIndexCreated++ + cyp, err := dsl.QB().Create(dsl.NewIndex(&dsl.IndexConfig{ + Type: structConfig.Label, + Fields: indexFields, + })).ToCypher() + if err != nil { + return err + } + + _, err = tx.Run(cyp, nil) + if err != nil { + oerr := err + err = tx.Rollback() + if err != nil { + return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) + } + + return oerr + } + } + } + + log.Debugf("created (%v) indexes", numIndexCreated) + + return tx.Commit() +} + +//verifies all indexes +func verifyAllIndexesAndConstraintsV3(mappedTypes *hashmap.HashMap) error { + sess, err := driver.Session(neo4j.AccessModeWrite) + if err != nil { + return err + } + defer sess.Close() + + //validate that we have to do anything + if mappedTypes == nil || mappedTypes.Len() == 0 { + return errors.New("must have types to map") + } + + var constraints []string + var indexes []string + + //build constraint strings + for nodes := range mappedTypes.Iter() { + node := nodes.Key.(string) + structConfig := nodes.Value.(structDecoratorConfig) + + if structConfig.Fields == nil || len(structConfig.Fields) == 0 { + continue + } + + fields := []string{} + + for _, config := range structConfig.Fields { + + if config.PrimaryKey || config.Unique { + t := fmt.Sprintf("CONSTRAINT ON (%s:%s) ASSERT %s.%s IS UNIQUE", node, structConfig.Label, node, config.Name) + constraints = append(constraints, t) + + indexes = append(indexes, fmt.Sprintf("INDEX ON :%s(%s)", structConfig.Label, config.Name)) + + } else if config.Index { + fields = append(fields, config.Name) + } + } + + f := "(" + for _, field := range fields { + f += field + } + + f += ")" + + indexes = append(indexes, fmt.Sprintf("INDEX ON :%s%s", structConfig.Label, f)) + + } + + //get whats there now + foundResult, err := sess.Run("CALL db.constraints", nil) + if err != nil { + return err + } + + foundConstraints, err := resultToStringArrV3(foundResult) + if err != nil { + return err + } + + foundInxdexResult, err := sess.Run("CALL db.indexes()", nil) + if err != nil { + return err + } + + foundIndexes, err := resultToStringArrV3(foundInxdexResult) + if err != nil { + return err + } + + //verify from there + delta, found := arrayOperations.Difference(foundIndexes, indexes) + if !found { + return fmt.Errorf("found differences in remote vs ogm for found indexes, %v", delta) + } + + log.Debug(delta) + + var founds []string + + for _, constraint := range foundConstraints { + founds = append(founds, constraint) + } + + delta, found = arrayOperations.Difference(founds, constraints) + if !found { + return fmt.Errorf("found differences in remote vs ogm for found constraints, %v", delta) + } + + log.Debug(delta) + + return nil +} diff --git a/index_v4.go b/index_v4.go new file mode 100644 index 0000000..332f667 --- /dev/null +++ b/index_v4.go @@ -0,0 +1,414 @@ +// Copyright (c) 2020 MindStand Technologies, Inc +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of +// this software and associated documentation files (the "Software"), to deal in +// the Software without restriction, including without limitation the rights to +// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software is furnished to do so, +// subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +package gogm + +import ( + "errors" + "fmt" + "github.com/adam-hanna/arrayOperations" + "github.com/cornelk/hashmap" + dsl "github.com/mindstand/go-cypherdsl" + "github.com/neo4j/neo4j-go-driver/neo4j" +) + +func resultToStringArrV4(isConstraint bool, res neo4j.Result) ([]string, error) { + if res == nil { + return nil, errors.New("result is nil") + } + + var result []string + + var i int + if isConstraint { + i = 0 + } else { + i = 1 + } + + for res.Next() { + val := res.Record().Values() + // nothing to parse + if val == nil || len(val) == 0 { + continue + } + + str, ok := val[i].(string) + if !ok { + return nil, fmt.Errorf("unable to parse [%T] to string. Value is %v: %w", val[i], val[i], ErrInternal) + } + + result = append(result, str) + } + + return result, nil +} + +//drops all known indexes +func dropAllIndexesAndConstraintsV4() error { + for _, db := range internalConfig.TargetDbs { + sess, err := driver.NewSession(neo4j.SessionConfig{ + AccessMode: neo4j.AccessModeWrite, + Bookmarks: nil, + DatabaseName: db, + }) + if err != nil { + return err + } + defer sess.Close() + + res, err := sess.Run("CALL db.constraints()", nil) + if err != nil { + return err + } + + constraints, err := resultToStringArrV4(true, res) + if err != nil { + return err + } + + //if there is anything, get rid of it + if len(constraints) != 0 { + tx, err := sess.BeginTransaction() + if err != nil { + return err + } + + for _, constraint := range constraints { + log.Debugf("dropping constraint '%s'", constraint) + res, err := tx.Run(fmt.Sprintf("DROP CONSTRAINT %s IF EXISTS", constraint), nil) + if err != nil { + oerr := err + err = tx.Rollback() + if err != nil { + return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) + } + + return oerr + } else if err = res.Err(); err != nil { + oerr := err + err = tx.Rollback() + if err != nil { + return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) + } + + return oerr + } + } + + err = tx.Commit() + if err != nil { + oerr := err + err = tx.Rollback() + if err != nil { + return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) + } + + return oerr + } + } + + res, err = sess.Run("CALL db.indexes()", nil) + if err != nil { + return err + } else if err = res.Err(); err != nil { + return err + } + + indexes, err := resultToStringArrV4(false, res) + if err != nil { + return err + } + + //if there is anything, get rid of it + if len(indexes) != 0 { + tx, err := sess.BeginTransaction() + if err != nil { + return err + } + + for _, index := range indexes { + if len(index) == 0 { + return errors.New("invalid index config") + } + + res, err := tx.Run(fmt.Sprintf("DROP INDEX %s IF EXISTS", index), nil) + if err != nil { + oerr := err + err = tx.Rollback() + if err != nil { + return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) + } + + return oerr + } else if err = res.Err(); err != nil { + oerr := err + err = tx.Rollback() + if err != nil { + return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) + } + + return oerr + } + } + + err = tx.Commit() + if err != nil { + oerr := err + err = tx.Rollback() + if err != nil { + return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) + } + + return oerr + } + } else { + continue + } + } + return nil +} + +//creates all indexes +func createAllIndexesAndConstraintsV4(mappedTypes *hashmap.HashMap) error { + for _, db := range internalConfig.TargetDbs { + sess, err := driver.NewSession(neo4j.SessionConfig{ + AccessMode: neo4j.AccessModeWrite, + Bookmarks: nil, + DatabaseName: db, + }) + if err != nil { + return err + } + defer sess.Close() + + //validate that we have to do anything + if mappedTypes == nil || mappedTypes.Len() == 0 { + return errors.New("must have types to map") + } + + numIndexCreated := 0 + + tx, err := sess.BeginTransaction() + if err != nil { + return err + } + + //index and/or create unique constraints wherever necessary + //for node, structConfig := range mappedTypes{ + for nodes := range mappedTypes.Iter() { + node := nodes.Key.(string) + structConfig := nodes.Value.(structDecoratorConfig) + if structConfig.Fields == nil || len(structConfig.Fields) == 0 { + continue + } + + var indexFields []string + + for _, config := range structConfig.Fields { + //pk is a special unique key + if config.PrimaryKey || config.Unique { + numIndexCreated++ + + cyp, err := dsl.QB().Create(dsl.NewConstraint(&dsl.ConstraintConfig{ + Unique: true, + Name: node, + Type: structConfig.Label, + Field: config.Name, + })).ToCypher() + if err != nil { + return err + } + + res, err := tx.Run(cyp, nil) + if err != nil { + oerr := err + err = tx.Rollback() + if err != nil { + return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) + } + + return oerr + } else if err = res.Err(); err != nil { + oerr := err + err = tx.Rollback() + if err != nil { + return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) + } + + return oerr + } + } else if config.Index { + indexFields = append(indexFields, config.Name) + } + } + + //create composite index + if len(indexFields) > 0 { + numIndexCreated++ + cyp, err := dsl.QB().Create(dsl.NewIndex(&dsl.IndexConfig{ + Type: structConfig.Label, + Fields: indexFields, + })).ToCypher() + if err != nil { + return err + } + + res, err := tx.Run(cyp, nil) + if err != nil { + oerr := err + err = tx.Rollback() + if err != nil { + return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) + } + + return oerr + } else if err = res.Err(); err != nil { + oerr := err + err = tx.Rollback() + if err != nil { + return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) + } + + return oerr + } + } + } + + log.Debugf("created (%v) indexes", numIndexCreated) + + err = tx.Commit() + if err != nil { + oerr := err + err = tx.Rollback() + if err != nil { + return fmt.Errorf("failed to rollback, original error was %s", oerr.Error()) + } + + return oerr + } + } + return nil +} + +//verifies all indexes +func verifyAllIndexesAndConstraintsV4(mappedTypes *hashmap.HashMap) error { + for _, db := range internalConfig.TargetDbs { + sess, err := driver.NewSession(neo4j.SessionConfig{ + AccessMode: neo4j.AccessModeWrite, + Bookmarks: nil, + DatabaseName: db, + }) + if err != nil { + return err + } + defer sess.Close() + + //validate that we have to do anything + if mappedTypes == nil || mappedTypes.Len() == 0 { + return errors.New("must have types to map") + } + + var constraints []string + var indexes []string + + //build constraint strings + for nodes := range mappedTypes.Iter() { + node := nodes.Key.(string) + structConfig := nodes.Value.(structDecoratorConfig) + + if structConfig.Fields == nil || len(structConfig.Fields) == 0 { + continue + } + + fields := []string{} + + for _, config := range structConfig.Fields { + + if config.PrimaryKey || config.Unique { + t := fmt.Sprintf("CONSTRAINT ON (%s:%s) ASSERT %s.%s IS UNIQUE", node, structConfig.Label, node, config.Name) + constraints = append(constraints, t) + + indexes = append(indexes, fmt.Sprintf("INDEX ON :%s(%s)", structConfig.Label, config.Name)) + + } else if config.Index { + fields = append(fields, config.Name) + } + } + + f := "(" + for _, field := range fields { + f += field + } + + f += ")" + + indexes = append(indexes, fmt.Sprintf("INDEX ON :%s%s", structConfig.Label, f)) + + } + + //get whats there now + foundResult, err := sess.Run("CALL db.constraints", nil) + if err != nil { + return err + } else if err = foundResult.Err(); err != nil { + return err + } + + foundConstraints, err := resultToStringArrV4(true, foundResult) + if err != nil { + return err + } + + foundInxdexResult, err := sess.Run("CALL db.indexes()", nil) + if err != nil { + return err + } else if err = foundInxdexResult.Err(); err != nil { + return err + } + + foundIndexes, err := resultToStringArrV4(false, foundInxdexResult) + if err != nil { + return err + } + + //verify from there + delta, found := arrayOperations.Difference(foundIndexes, indexes) + if !found { + return fmt.Errorf("found differences in remote vs ogm for found indexes, %v", delta) + } + + log.Debug(delta) + + var founds []string + + for _, constraint := range foundConstraints { + founds = append(founds, constraint) + } + + delta, found = arrayOperations.Difference(founds, constraints) + if !found { + return fmt.Errorf("found differences in remote vs ogm for found constraints, %v", delta) + } + + log.Debug(delta) + } + + return nil +} diff --git a/load_strategy.go b/load_strategy.go index 612314c..9ba8a3d 100644 --- a/load_strategy.go +++ b/load_strategy.go @@ -101,14 +101,14 @@ func PathLoadStrategyOne(variable, label string, depth int, additionalConstraint Name: variable, Field: "uuid", ConditionOperator: dsl.EqualToOperator, - Check: dsl.ParamString("{uuid}"), + Check: dsl.ParamString("$uuid"), })) } else { builder = builder.Where(dsl.C(&dsl.ConditionConfig{ Name: variable, Field: "uuid", ConditionOperator: dsl.EqualToOperator, - Check: dsl.ParamString("{uuid}"), + Check: dsl.ParamString("$uuid"), })) } @@ -130,7 +130,7 @@ func PathLoadStrategyEdgeConstraint(startVariable, startLabel, endLabel, endTarg } qp, err := dsl.ParamsFromMap(map[string]interface{}{ - endTargetField: dsl.ParamString(fmt.Sprintf("{%s}", endTargetField)), + endTargetField: dsl.ParamString(fmt.Sprintf("$%s", endTargetField)), }) if err != nil { return nil, err diff --git a/save.go b/save.go index 92c75b3..e4f5fb8 100644 --- a/save.go +++ b/save.go @@ -243,7 +243,7 @@ func removeRelations(runFunc neoRunFunc, dels map[string][]int64) error { } cyq, err := dsl.QB(). - Cypher("UNWIND {rows} as row"). + Cypher("UNWIND $rows as row"). Match(dsl.Path(). V(dsl.V{ Name: "start", @@ -260,11 +260,13 @@ func removeRelations(runFunc neoRunFunc, dels map[string][]int64) error { return err } - _, err = runFunc(cyq, map[string]interface{}{ + res, err := runFunc(cyq, map[string]interface{}{ "rows": params, }) if err != nil { - return fmt.Errorf("%s, %w", err.Error(), ErrInternal) + return fmt.Errorf("%s: %w", err.Error(), ErrInternal) + } else if err = res.Err(); err != nil { + return fmt.Errorf("%s: %w", err.Error(), ErrInternal) } //todo sanity check to make sure the affects worked @@ -300,7 +302,7 @@ func createNodes(runFunc neoRunFunc, crNodes map[string]map[string]nodeCreateCon //todo replace once unwind is fixed and path cyp, err := dsl.QB(). - Cypher("UNWIND {rows} as row"). + Cypher("UNWIND $rows as row"). Merge(&dsl.MergeConfig{ Path: path, }). @@ -416,7 +418,7 @@ func relateNodes(runFunc neoRunFunc, relations map[string][]relCreateConf, ids m } cyp, err := dsl.QB(). - Cypher("UNWIND {rows} as row"). + Cypher("UNWIND $rows as row"). Match(dsl.Path().V(dsl.V{Name: "startNode"}).Build()). Where(dsl.C(&dsl.ConditionConfig{ FieldManipulationFunction: "ID", @@ -447,11 +449,13 @@ func relateNodes(runFunc neoRunFunc, relations map[string][]relCreateConf, ids m Cypher("SET rel += row.props"). ToCypher() - _, err = runFunc(cyp, map[string]interface{}{ + res, err := runFunc(cyp, map[string]interface{}{ "rows": params, }) if err != nil { return err + } else if err = res.Err(); err != nil { + return err } }