From e34fb468ea6297a920b902ab4ec9b5dfd8c0aaf9 Mon Sep 17 00:00:00 2001 From: Duc Nguyen Date: Sun, 30 Jun 2024 23:09:23 +0700 Subject: [PATCH] Refactor code --- batch.go => batch/batch.go | 80 +++++- client/client.go | 11 + config.go => client/config.go | 22 +- elasticsearch.go | 242 +++--------------- elasticsearch_loader.go | 76 ------ generic_writer.go | 66 ----- health_checker.go => health/health_checker.go | 2 +- loader.go | 79 ------ field_loader.go => loader/field_loader.go | 2 +- .../passcode_repository.go | 51 +++- query.go | 12 +- query/{ => builder}/query_builder.go | 2 +- repository.go | 20 -- search_builder.go | 18 +- search_load.go | 19 -- search_write.go | 37 --- searcher.go | 24 -- writer.go | 63 ----- .../elasticsearch_writer.go | 2 +- inserter.go => writer/inserter.go | 2 +- updater.go => writer/updater.go | 2 +- 21 files changed, 194 insertions(+), 638 deletions(-) rename batch.go => batch/batch.go (74%) create mode 100644 client/client.go rename config.go => client/config.go (77%) delete mode 100644 elasticsearch_loader.go delete mode 100644 generic_writer.go rename health_checker.go => health/health_checker.go (97%) delete mode 100644 loader.go rename field_loader.go => loader/field_loader.go (98%) rename passcode_repository.go => passcode/passcode_repository.go (61%) rename query/{ => builder}/query_builder.go (99%) delete mode 100644 repository.go delete mode 100644 search_load.go delete mode 100644 search_write.go delete mode 100644 searcher.go delete mode 100644 writer.go rename elasticsearch_writer.go => writer/elasticsearch_writer.go (98%) rename inserter.go => writer/inserter.go (98%) rename updater.go => writer/updater.go (98%) diff --git a/batch.go b/batch/batch.go similarity index 74% rename from batch.go rename to batch/batch.go index 2ede262..b494723 100644 --- a/batch.go +++ b/batch/batch.go @@ -1,4 +1,4 @@ -package elasticsearch +package batch import ( "context" @@ -74,6 +74,69 @@ func (w *BatchInserter) Write(ctx context.Context, model interface{}) ([]int, [] } return successIndices, failureIndices, errors.New("invalid input") } +func FindIdField(modelType reflect.Type) (int, string, string) { + return FindBsonField(modelType, "_id") +} +func FindBsonField(modelType reflect.Type, bsonName string) (int, string, string) { + numField := modelType.NumField() + for i := 0; i < numField; i++ { + field := modelType.Field(i) + bsonTag := field.Tag.Get("bson") + tags := strings.Split(bsonTag, ",") + json := field.Name + if tag1, ok1 := field.Tag.Lookup("json"); ok1 { + json = strings.Split(tag1, ",")[0] + } + for _, tag := range tags { + if strings.TrimSpace(tag) == bsonName { + return i, field.Name, json + } + } + } + return -1, "", "" +} + +func BuildQueryWithoutIdFromObject(object interface{}) map[string]interface{} { + valueOf := reflect.Indirect(reflect.ValueOf(object)) + idIndex, _, _ := FindIdField(valueOf.Type()) + result := map[string]interface{}{} + for i := 0; i < valueOf.NumField(); i++ { + if i != idIndex { + _, jsonName := FindFieldByIndex(valueOf.Type(), i) + result[jsonName] = valueOf.Field(i).Interface() + } + } + return result +} +func FindFieldByIndex(modelType reflect.Type, fieldIndex int) (fieldName, jsonTagName string) { + if fieldIndex < modelType.NumField() { + field := modelType.Field(fieldIndex) + jsonTagName := "" + if jsonTag, ok := field.Tag.Lookup("json"); ok { + jsonTagName = strings.Split(jsonTag, ",")[0] + } + return field.Name, jsonTagName + } + return "", "" +} + +func BuildIndicesResult(listIds, successIds, failIds []interface{}) (successIndices, failureIndices []int) { + if len(listIds) > 0 { + for _, idValue := range listIds { + for index, id := range successIds { + if id == idValue { + successIndices = append(successIndices, int(index)) + } + } + for index, id := range failIds { + if id == idValue { + failureIndices = append(failureIndices, int(index)) + } + } + } + } + return +} func InsertMany(ctx context.Context, es *elasticsearch.Client, indexName string, modelType reflect.Type, model interface{}) ([]int, []int, error) { value := reflect.Indirect(reflect.ValueOf(model)) @@ -184,3 +247,18 @@ func UpsertMany(ctx context.Context, es *elasticsearch.Client, indexName string, } return successIndices, failureIndices, errors.New("invalid input") } +func FindListIdField(modelType reflect.Type, model interface{}) (listIdS []interface{}) { + value := reflect.Indirect(reflect.ValueOf(model)) + + if value.Kind() == reflect.Slice { + for i := 0; i < value.Len(); i++ { + sliceValue := value.Index(i).Interface() + if idIndex, _, _ := FindIdField(modelType); idIndex >= 0 { + modelValue := reflect.Indirect(reflect.ValueOf(sliceValue)) + idValue := modelValue.Field(idIndex).String() + listIdS = append(listIdS, idValue) + } + } + } + return +} diff --git a/client/client.go b/client/client.go new file mode 100644 index 0000000..4215703 --- /dev/null +++ b/client/client.go @@ -0,0 +1,11 @@ +package client + +import ( + "github.com/elastic/go-elasticsearch/v8" + "time" +) + +func NewClient(config Config, timeouts ...time.Duration) (*elasticsearch.Client, error) { + c := GetConfig(config, timeouts...) + return elasticsearch.NewClient(c) +} diff --git a/config.go b/client/config.go similarity index 77% rename from config.go rename to client/config.go index d981366..a139748 100644 --- a/config.go +++ b/client/config.go @@ -1,4 +1,4 @@ -package elasticsearch +package client import ( "crypto/tls" @@ -14,12 +14,12 @@ type TransportConfig struct { Timeout *int64 `yaml:"timeout" mapstructure:"timeout" json:"timeout,omitempty" gorm:"column:timeout" bson:"timeout,omitempty" dynamodbav:"timeout,omitempty" firestore:"timeout,omitempty"` } type Config struct { - Addresses []string `yaml:"addresses" mapstructure:"addresses" json:"addresses,omitempty" gorm:"column:addresses" bson:"addresses,omitempty" dynamodbav:"addresses,omitempty" firestore:"addresses,omitempty"` - Username *string `yaml:"username" mapstructure:"username" json:"username,omitempty" gorm:"column:username" bson:"username,omitempty" dynamodbav:"username,omitempty" firestore:"username,omitempty"` - Password *string `yaml:"password" mapstructure:"password" json:"password,omitempty" gorm:"column:password" bson:"password,omitempty" dynamodbav:"password,omitempty" firestore:"password,omitempty"` - CloudID *string `yaml:"cloud_id" mapstructure:"cloud_id" json:"cloudID,omitempty" gorm:"column:cloudid" bson:"cloudID,omitempty" dynamodbav:"cloudID,omitempty" firestore:"cloudID,omitempty"` - APIKey *string `yaml:"api_key" mapstructure:"api_key" json:"apiKey,omitempty" gorm:"column:apikey" bson:"apiKey,omitempty" dynamodbav:"apiKey,omitempty" firestore:"apiKey,omitempty"` - DisableRetry *bool `yaml:"disable_retry" mapstructure:"disable_retry" json:"disableRetry,omitempty" gorm:"column:disableretry" bson:"disableRetry,omitempty" dynamodbav:"disableRetry,omitempty" firestore:"disableRetry,omitempty"` + Addresses []string `yaml:"addresses" mapstructure:"addresses" json:"addresses,omitempty" gorm:"column:addresses" bson:"addresses,omitempty" dynamodbav:"addresses,omitempty" firestore:"addresses,omitempty"` + Username *string `yaml:"username" mapstructure:"username" json:"username,omitempty" gorm:"column:username" bson:"username,omitempty" dynamodbav:"username,omitempty" firestore:"username,omitempty"` + Password *string `yaml:"password" mapstructure:"password" json:"password,omitempty" gorm:"column:password" bson:"password,omitempty" dynamodbav:"password,omitempty" firestore:"password,omitempty"` + CloudID *string `yaml:"cloud_id" mapstructure:"cloud_id" json:"cloudID,omitempty" gorm:"column:cloudid" bson:"cloudID,omitempty" dynamodbav:"cloudID,omitempty" firestore:"cloudID,omitempty"` + APIKey *string `yaml:"api_key" mapstructure:"api_key" json:"apiKey,omitempty" gorm:"column:apikey" bson:"apiKey,omitempty" dynamodbav:"apiKey,omitempty" firestore:"apiKey,omitempty"` + DisableRetry *bool `yaml:"disable_retry" mapstructure:"disable_retry" json:"disableRetry,omitempty" gorm:"column:disableretry" bson:"disableRetry,omitempty" dynamodbav:"disableRetry,omitempty" firestore:"disableRetry,omitempty"` // EnableRetryOnTimeout *bool `yaml:"enable_retry_on_timeout" mapstructure:"enable_retry_on_timeout" json:"enableRetryOnTimeout,omitempty" gorm:"column:enableretryontimeout" bson:"enableRetryOnTimeout,omitempty" dynamodbav:"enableRetryOnTimeout,omitempty" firestore:"enableRetryOnTimeout,omitempty"` MaxRetries *int `yaml:"max_retries" mapstructure:"max_retries" json:"maxRetries,omitempty" gorm:"column:maxretries" bson:"maxRetries,omitempty" dynamodbav:"maxRetries,omitempty" firestore:"maxRetries,omitempty"` DiscoverNodesOnStart *bool `yaml:"discover_nodes_on_start" mapstructure:"discover_nodes_on_start" json:"discoverNodesOnStart,omitempty" gorm:"column:discovernodesonstart" bson:"discoverNodesOnStart,omitempty" dynamodbav:"discoverNodesOnStart,omitempty" firestore:"discoverNodesOnStart,omitempty"` @@ -69,10 +69,10 @@ func GetConfig(conf Config, timeouts ...time.Duration) elasticsearch.Config { c.DisableRetry = *conf.DisableRetry } /* - if conf.EnableRetryOnTimeout != nil { - c.EnableRetryOnTimeout = *conf.EnableRetryOnTimeout - } - */ + if conf.EnableRetryOnTimeout != nil { + c.EnableRetryOnTimeout = *conf.EnableRetryOnTimeout + } + */ if conf.MaxRetries != nil { c.MaxRetries = *conf.MaxRetries } diff --git a/elasticsearch.go b/elasticsearch.go index 4b03093..0c9b006 100644 --- a/elasticsearch.go +++ b/elasticsearch.go @@ -4,24 +4,19 @@ import ( "context" "encoding/json" "errors" + "fmt" "github.com/elastic/go-elasticsearch/v8" "github.com/elastic/go-elasticsearch/v8/esapi" "github.com/elastic/go-elasticsearch/v8/esutil" "log" "reflect" "strings" - "time" ) -func Connect(config Config, timeouts ...time.Duration) (*elasticsearch.Client, error) { - c := GetConfig(config, timeouts...) - return elasticsearch.NewClient(c) -} - func FindIdField(modelType reflect.Type) (int, string, string) { - return FindBsonField(modelType, "_id") + return findBsonField(modelType, "_id") } -func FindBsonField(modelType reflect.Type, bsonName string) (int, string, string) { +func findBsonField(modelType reflect.Type, bsonName string) (int, string, string) { numField := modelType.NumField() for i := 0; i < numField; i++ { field := modelType.Field(i) @@ -78,26 +73,6 @@ func FindFieldByIndex(modelType reflect.Type, fieldIndex int) (fieldName, jsonTa return "", "" } -func MakeMapJson(modelType reflect.Type) map[string]string { - maps := make(map[string]string) - numField := modelType.NumField() - for i := 0; i < numField; i++ { - key1 := modelType.Field(i).Name - fields, _ := modelType.FieldByName(key1) - if tag, ok := fields.Tag.Lookup("json"); ok { - if strings.Contains(tag, ",") { - a := strings.Split(tag, ",") - maps[key1] = a[0] - } else { - maps[key1] = tag - } - } else { - maps[key1] = key1 - } - } - return maps -} - func BuildQueryWithoutIdFromObject(object interface{}) map[string]interface{} { valueOf := reflect.Indirect(reflect.ValueOf(object)) idIndex, _, _ := FindIdField(valueOf.Type()) @@ -111,19 +86,6 @@ func BuildQueryWithoutIdFromObject(object interface{}) map[string]interface{} { return result } -func BuildQueryMap(indexName string, query map[string]interface{}) map[string]interface{} { - return map[string]interface{}{} -} - -func MapToDBObject(object map[string]interface{}, objectMap map[string]string) map[string]interface{} { - result := make(map[string]interface{}) - for key, value := range object { - field := objectMap[key] - result[field] = value - } - return result -} - func Exist(ctx context.Context, es *elasticsearch.Client, indexName string, documentID string) (bool, error) { req := esapi.ExistsRequest{ Index: indexName, @@ -147,16 +109,7 @@ func Exist(ctx context.Context, es *elasticsearch.Client, indexName string, docu } } -func FindOneById(ctx context.Context, es *elasticsearch.Client, indexName string, documentID string, modelType reflect.Type) (interface{}, error) { - result := reflect.New(modelType).Interface() - if ok, err := FindOneByIdAndDecode(ctx, es, indexName, documentID, result); ok { - return result, nil - } else { - return nil, err - } -} - -func FindOneByIdAndDecode(ctx context.Context, es *elasticsearch.Client, indexName string, documentID string, result interface{}) (bool, error) { +func FindOne(ctx context.Context, es *elasticsearch.Client, indexName string, documentID string, result interface{}) (bool, error) { req := esapi.GetRequest{ Index: indexName, DocumentID: documentID, @@ -182,19 +135,10 @@ func FindOneByIdAndDecode(ctx context.Context, es *elasticsearch.Client, indexNa return false, errors.New("response error") } -func FindOne(ctx context.Context, es *elasticsearch.Client, index []string, query map[string]interface{}, modelType reflect.Type) (interface{}, error) { - result := reflect.New(modelType).Interface() - if ok, err := FindOneAndDecode(ctx, es, index, query, result); ok { - return result, nil - } else { - return nil, err - } -} - -func FindOneAndDecode(ctx context.Context, es *elasticsearch.Client, index []string, query map[string]interface{}, result interface{}) (bool, error) { +func FindOneByFilter(ctx context.Context, es *elasticsearch.Client, index []string, filter map[string]interface{}, result interface{}) (bool, error) { req := esapi.SearchRequest{ Index: index, - Body: esutil.NewJSONReader(query), + Body: esutil.NewJSONReader(filter), TrackTotalHits: true, Pretty: true, } @@ -224,17 +168,7 @@ func FindOneAndDecode(ctx context.Context, es *elasticsearch.Client, index []str } } -func Find(ctx context.Context, es *elasticsearch.Client, indexName []string, query map[string]interface{}, modelType reflect.Type) (interface{}, error) { - modelsType := reflect.Zero(reflect.SliceOf(modelType)).Type() - result := reflect.New(modelsType).Interface() - if ok, err := FindAndDecode(ctx, es, indexName, query, result); ok { - return result, nil - } else { - return nil, err - } -} - -func FindAndDecode(ctx context.Context, es *elasticsearch.Client, indexName []string, query map[string]interface{}, result interface{}) (bool, error) { +func Find(ctx context.Context, es *elasticsearch.Client, indexName []string, query map[string]interface{}, result interface{}) error { req := esapi.SearchRequest{ Index: indexName, Body: esutil.NewJSONReader(query), @@ -243,17 +177,17 @@ func FindAndDecode(ctx context.Context, es *elasticsearch.Client, indexName []st } res, err := req.Do(ctx, es) if err != nil { - return false, err + return err } defer res.Body.Close() - + modelType := reflect.TypeOf(result).Elem().Elem() if res.IsError() { - return false, errors.New("response error") + return errors.New("response error") } else { var r map[string]interface{} if err := json.NewDecoder(res.Body).Decode(&r); err != nil { - return false, err + return err } else { hits := r["hits"].(map[string]interface{})["hits"].([]interface{}) listResults := make([]interface{}, 0) @@ -269,66 +203,18 @@ func FindAndDecode(ctx context.Context, es *elasticsearch.Client, indexName []st } listResults = append(listResults, r) } - - err := json.NewDecoder(esutil.NewJSONReader(listResults)).Decode(result) - if err != nil { - return false, err - } - return true, nil + return json.NewDecoder(esutil.NewJSONReader(listResults)).Decode(result) } } } -func FindValueByJson(model interface{}, jsonTagName string) (index int, fieldName string, val string) { - object := reflect.Indirect(reflect.ValueOf(model)) - modelType := object.Type() - - numField := modelType.NumField() - for i := 0; i < numField; i++ { - val := object.Field(i) - field := modelType.Field(i) - tag1, ok1 := field.Tag.Lookup("json") - if ok1 && strings.Split(tag1, ",")[0] == jsonTagName { - return i, field.Name, val.String() - } - } - return -1, jsonTagName, "" -} - -func FindListIdField(modelType reflect.Type, model interface{}) (listIdS []interface{}) { - value := reflect.Indirect(reflect.ValueOf(model)) - - if value.Kind() == reflect.Slice { - for i := 0; i < value.Len(); i++ { - sliceValue := value.Index(i).Interface() - if idIndex, _, _ := FindIdField(modelType); idIndex >= 0 { - modelValue := reflect.Indirect(reflect.ValueOf(sliceValue)) - idValue := modelValue.Field(idIndex).String() - listIdS = append(listIdS, idValue) - } - } - } - return -} - -func InsertOne(ctx context.Context, es *elasticsearch.Client, indexName string, model interface{}, opts ...int) (int64, error) { - object := reflect.Indirect(reflect.ValueOf(model)) - modelType := object.Type() +func Create(ctx context.Context, es *elasticsearch.Client, indexName string, model interface{}, id *string) (int64, error) { var req esapi.CreateRequest - idIndex := -1 - if len(opts) > 0 && opts[0] >= 0 { - idIndex = opts[0] - } else { - idIndex, _, _ = FindIdField(modelType) - } - if idIndex >= 0 { - modelValue := reflect.Indirect(reflect.ValueOf(model)) - idValue := modelValue.Field(idIndex).String() - body := BuildQueryWithoutIdFromObject(model) + if id != nil { req = esapi.CreateRequest{ Index: indexName, - DocumentID: idValue, - Body: esutil.NewJSONReader(body), + DocumentID: *id, + Body: esutil.NewJSONReader(model), Refresh: "true", } } else { @@ -356,46 +242,13 @@ func InsertOne(ctx context.Context, es *elasticsearch.Client, indexName string, } } -func BuildIndicesResult(listIds, successIds, failIds []interface{}) (successIndices, failureIndices []int) { - if len(listIds) > 0 { - for _, idValue := range listIds { - for index, id := range successIds { - if id == idValue { - successIndices = append(successIndices, int(index)) - } - } - for index, id := range failIds { - if id == idValue { - failureIndices = append(failureIndices, int(index)) - } - } - } - } - return -} - -func UpdateOne(ctx context.Context, es *elasticsearch.Client, indexName string, model interface{}, opts ...int) (int64, error) { - object := reflect.Indirect(reflect.ValueOf(model)) - modelType := object.Type() - idIndex := -1 - if len(opts) > 0 && opts[0] >= 0 { - idIndex = opts[0] - } else { - idIndex, _, _ = FindIdField(modelType) - } - if idIndex < 0 { - return 0, errors.New("missing document ID in the object") - } - modelValue := reflect.ValueOf(model) - idValue := modelValue.Elem().Field(idIndex).String() - // body := BuildQueryWithoutIdFromObject(model) - +func Update(ctx context.Context, es *elasticsearch.Client, indexName string, model interface{}, id string) (int64, error) { query := map[string]interface{}{ "doc": model, } req := esapi.UpdateRequest{ Index: indexName, - DocumentID: idValue, + DocumentID: id, Body: esutil.NewJSONReader(query), Refresh: "true", } @@ -417,7 +270,7 @@ func UpdateOne(ctx context.Context, es *elasticsearch.Client, indexName string, } } -func UpsertOne(ctx context.Context, es *elasticsearch.Client, indexName string, id string, model interface{}) (int64, error) { +func Save(ctx context.Context, es *elasticsearch.Client, indexName string, model interface{}, id string) (int64, error) { // body := BuildQueryWithoutIdFromObject(model) query := map[string]interface{}{ "doc": model, @@ -444,22 +297,27 @@ func UpsertOne(ctx context.Context, es *elasticsearch.Client, indexName string, return successful, nil } -func PatchOne(ctx context.Context, es *elasticsearch.Client, indexName string, id string, model map[string]interface{}) (int64, error) { - idValue := reflect.ValueOf(model[id]) - if idValue.IsZero() { - return 0, errors.New("missing document ID in the map") +func Patch(ctx context.Context, es *elasticsearch.Client, indexName string, model map[string]interface{}, idName string) (int64, error) { + idValue, ok := model[idName] + if !ok { + return -1, fmt.Errorf("%s must be in map[string]interface{} for patch", idName) + } + id, ok2 := idValue.(string) + if !ok2 { + return -1, fmt.Errorf("%s map[string]interface{} must be a string for patch", idName) } - delete(model, id) + delete(model, idName) query := map[string]interface{}{ "doc": model, } req := esapi.UpdateRequest{ Index: indexName, - DocumentID: idValue.String(), + DocumentID: id, Body: esutil.NewJSONReader(query), Refresh: "true", } res, err := req.Do(ctx, es) + model[idName] = id if err != nil { return -1, err } @@ -477,7 +335,7 @@ func PatchOne(ctx context.Context, es *elasticsearch.Client, indexName string, i } } -func DeleteOne(ctx context.Context, es *elasticsearch.Client, indexName string, documentID string) (int64, error) { +func Delete(ctx context.Context, es *elasticsearch.Client, indexName string, documentID string) (int64, error) { req := esapi.DeleteRequest{ Index: indexName, DocumentID: documentID, @@ -499,41 +357,3 @@ func DeleteOne(ctx context.Context, es *elasticsearch.Client, indexName string, } } } - -func GetFieldByJson(modelType reflect.Type, jsonName string) (int, string, string) { - numField := modelType.NumField() - for i := 0; i < numField; i++ { - field := modelType.Field(i) - tag1, ok1 := field.Tag.Lookup("json") - if ok1 && strings.Split(tag1, ",")[0] == jsonName { - if tag2, ok2 := field.Tag.Lookup("bson"); ok2 { - return i, field.Name, strings.Split(tag2, ",")[0] - } - return i, field.Name, "" - } - } - return -1, jsonName, jsonName -} - -func MapModels(ctx context.Context, models interface{}, mp func(context.Context, interface{}) (interface{}, error)) (interface{}, error) { - valueModelObject := reflect.Indirect(reflect.ValueOf(models)) - if valueModelObject.Kind() == reflect.Ptr { - valueModelObject = reflect.Indirect(valueModelObject) - } - if valueModelObject.Kind() == reflect.Slice { - le := valueModelObject.Len() - for i := 0; i < le; i++ { - x := valueModelObject.Index(i) - k := x.Kind() - if k == reflect.Struct { - y := x.Addr().Interface() - mp(ctx, y) - } else { - y := x.Interface() - mp(ctx, y) - } - - } - } - return models, nil -} diff --git a/elasticsearch_loader.go b/elasticsearch_loader.go deleted file mode 100644 index 1926426..0000000 --- a/elasticsearch_loader.go +++ /dev/null @@ -1,76 +0,0 @@ -package elasticsearch - -import ( - "context" - "github.com/elastic/go-elasticsearch/v8" - "log" - "reflect" -) - -type ElasticSearchLoader struct { - client *elasticsearch.Client - indexName string - modelType reflect.Type - jsonIdName string - idIndex int - Map func(ctx context.Context, model interface{}) (interface{}, error) -} - -func NewElasticSearchLoader(client *elasticsearch.Client, indexName string, modelType reflect.Type, options ...func(context.Context, interface{}) (interface{}, error)) *ElasticSearchLoader { - idIndex, _, jsonIdName := FindIdField(modelType) - if idIndex < 0 { - log.Println(modelType.Name() + " repository can't use functions that need Id value (Ex Load, Exist, Save, Update) because don't have any fields of " + modelType.Name() + " struct define _id bson tag.") - } - var mp func(context.Context, interface{}) (interface{}, error) - if len(options) > 0 { - mp = options[0] - } - return &ElasticSearchLoader{client: client, indexName: indexName, modelType: modelType, jsonIdName: jsonIdName, idIndex: idIndex, Map: mp} -} - -func (m *ElasticSearchLoader) Id() string { - return m.indexName -} - -func (m *ElasticSearchLoader) All(ctx context.Context) (interface{}, error) { - query := BuildQueryMap(m.indexName, nil) - result, err := Find(ctx, m.client, []string{m.indexName}, query, m.modelType) - if m.Map != nil && err == nil && result != nil { - return MapModels(ctx, result, m.Map) - } - return result, err -} - -func (m *ElasticSearchLoader) Load(ctx context.Context, id string) (interface{}, error) { - r, er1 := FindOneById(ctx, m.client, m.indexName, id, m.modelType) - if er1 != nil { - return r, er1 - } - if m.Map != nil { - r2, er2 := m.Map(ctx, r) - if er2 != nil { - return r, er2 - } - return r2, er2 - } - return r, er1 -} - -func (m *ElasticSearchLoader) Get(ctx context.Context, id string, result interface{}) (bool, error) { - ok, er0 := FindOneByIdAndDecode(ctx, m.client, m.indexName, id, result) - if ok && er0 == nil && m.Map != nil { - _, er2 := m.Map(ctx, result) - if er2 != nil { - return ok, er2 - } - } - return ok, er0 -} - -func (m *ElasticSearchLoader) LoadAndDecode(ctx context.Context, id string, result interface{}) (bool, error) { - return m.Get(ctx, id, result) -} - -func (m *ElasticSearchLoader) Exist(ctx context.Context, id string) (bool, error) { - return Exist(ctx, m.client, m.indexName, id) -} diff --git a/generic_writer.go b/generic_writer.go deleted file mode 100644 index 34dd023..0000000 --- a/generic_writer.go +++ /dev/null @@ -1,66 +0,0 @@ -package elasticsearch - -import ( - "context" - es "github.com/elastic/go-elasticsearch/v8" - "reflect" -) - -type Repository interface { - Get(ctx context.Context, id string, result interface{}) (bool, error) - Exist(ctx context.Context, id string) (bool, error) - Insert(ctx context.Context, model interface{}) (int64, error) - Update(ctx context.Context, model interface{}) (int64, error) - Patch(ctx context.Context, model map[string]interface{}) (int64, error) - Delete(ctx context.Context, id string) (int64, error) -} - -type GenericWriter struct { - *ElasticSearchLoader - maps map[string]string - versionField string - versionIndex int - Mapper Mapper -} - -func NewGenericWriter(client *es.Client, indexName string, modelType reflect.Type, options ...string) *GenericWriter { - return NewGenericWriterWithMapper(client, indexName, modelType, nil, options...) -} -func NewGenericWriterWithMapper(client *es.Client, indexName string, modelType reflect.Type, mapper Mapper, options ...string) *GenericWriter { - var loader *ElasticSearchLoader - if mapper != nil { - loader = NewElasticSearchLoader(client, indexName, modelType, mapper.DbToModel) - } else { - loader = NewElasticSearchLoader(client, indexName, modelType) - } - var versionField string - if len(options) >= 1 && len(options[0]) > 0 { - versionField = options[0] - } - if len(versionField) > 0 { - index, _ := FindFieldByName(modelType, versionField) - if index >= 0 { - return &GenericWriter{ElasticSearchLoader: loader, maps: MakeMapJson(modelType), versionField: versionField, versionIndex: index} - } - } - return &GenericWriter{ElasticSearchLoader: loader, maps: MakeMapJson(modelType), Mapper: mapper, versionField: "", versionIndex: -1} -} - -func (m *GenericWriter) Insert(ctx context.Context, model interface{}) (int64, error) { - return InsertOne(ctx, m.client, m.indexName, model, m.idIndex) -} - -func (m *GenericWriter) Update(ctx context.Context, model interface{}) (int64, error) { - return UpdateOne(ctx, m.client, m.indexName, model, m.idIndex) -} -func (m *GenericWriter) Patch(ctx context.Context, model map[string]interface{}) (int64, error) { - return PatchOne(ctx, m.client, m.indexName, m.jsonIdName, model) -} -func (m *GenericWriter) Delete(ctx context.Context, id string) (int64, error) { - return DeleteOne(ctx, m.client, m.indexName, id) -} -func (m *GenericWriter) Save(ctx context.Context, model interface{}) (int64, error) { - modelValue := reflect.ValueOf(model) - id := modelValue.Field(m.idIndex).String() - return UpsertOne(ctx, m.client, m.indexName, id, model) -} diff --git a/health_checker.go b/health/health_checker.go similarity index 97% rename from health_checker.go rename to health/health_checker.go index 7b671ca..f99a531 100644 --- a/health_checker.go +++ b/health/health_checker.go @@ -1,4 +1,4 @@ -package elasticsearch +package health import ( "context" diff --git a/loader.go b/loader.go deleted file mode 100644 index 72acba5..0000000 --- a/loader.go +++ /dev/null @@ -1,79 +0,0 @@ -package elasticsearch - -import ( - "context" - "github.com/elastic/go-elasticsearch/v8" - "log" - "reflect" -) - -type Loader struct { - client *elasticsearch.Client - indexName string - modelType reflect.Type - jsonIdName string - idIndex int - Map func(ctx context.Context, model interface{}) (interface{}, error) -} - -func NewLoader(client *elasticsearch.Client, indexName string, modelType reflect.Type, options ...func(context.Context, interface{}) (interface{}, error)) *Loader { - idIndex, _, jsonIdName := FindIdField(modelType) - if idIndex < 0 { - log.Println(modelType.Name() + " repository can't use functions that need Id value (Ex Load, Exist, Save, Update) because don't have any fields of " + modelType.Name() + " struct define _id bson tag.") - } - var mp func(context.Context, interface{}) (interface{}, error) - if len(options) > 0 { - mp = options[0] - } - return &Loader{client: client, indexName: indexName, modelType: modelType, jsonIdName: jsonIdName, idIndex: idIndex, Map: mp} -} - -func (m *Loader) Id() string { - return m.indexName -} - -func (m *Loader) All(ctx context.Context) (interface{}, error) { - query := BuildQueryMap(m.indexName, nil) - result, err := Find(ctx, m.client, []string{m.indexName}, query, m.modelType) - if m.Map != nil && err == nil && result != nil { - return MapModels(ctx, result, m.Map) - } - return result, err -} - -func (m *Loader) Load(ctx context.Context, id interface{}) (interface{}, error) { - sid := id.(string) - r, er1 := FindOneById(ctx, m.client, m.indexName, sid, m.modelType) - if er1 != nil { - return r, er1 - } - if m.Map != nil { - r2, er2 := m.Map(ctx, r) - if er2 != nil { - return r, er2 - } - return r2, er2 - } - return r, er1 -} - -func (m *Loader) Get(ctx context.Context, id interface{}, result interface{}) (bool, error) { - sid := id.(string) - ok, er0 := FindOneByIdAndDecode(ctx, m.client, m.indexName, sid, result) - if ok && er0 == nil && m.Map != nil { - _, er2 := m.Map(ctx, result) - if er2 != nil { - return ok, er2 - } - } - return ok, er0 -} - -func (m *Loader) LoadAndDecode(ctx context.Context, id interface{}, result interface{}) (bool, error) { - return m.Get(ctx, id, result) -} - -func (m *Loader) Exist(ctx context.Context, id interface{}) (bool, error) { - sid := id.(string) - return Exist(ctx, m.client, m.indexName, sid) -} diff --git a/field_loader.go b/loader/field_loader.go similarity index 98% rename from field_loader.go rename to loader/field_loader.go index 0f719c8..2da421e 100644 --- a/field_loader.go +++ b/loader/field_loader.go @@ -1,4 +1,4 @@ -package elasticsearch +package loader import ( "context" diff --git a/passcode_repository.go b/passcode/passcode_repository.go similarity index 61% rename from passcode_repository.go rename to passcode/passcode_repository.go index 126559d..b1bb889 100644 --- a/passcode_repository.go +++ b/passcode/passcode_repository.go @@ -1,8 +1,9 @@ -package elasticsearch +package passcode import ( "context" "encoding/json" + "errors" "fmt" "github.com/elastic/go-elasticsearch/v8" "github.com/elastic/go-elasticsearch/v8/esapi" @@ -79,3 +80,51 @@ func (p *PasscodeRepository) Load(ctx context.Context, id string) (string, time. func (p *PasscodeRepository) Delete(ctx context.Context, id string) (int64, error) { return DeleteOne(ctx, p.client, p.indexName, id) } + +func FindOneByIdAndDecode(ctx context.Context, es *elasticsearch.Client, indexName string, documentID string, result interface{}) (bool, error) { + req := esapi.GetRequest{ + Index: indexName, + DocumentID: documentID, + } + res, err := req.Do(ctx, es) + if err != nil { + return false, err + } + defer res.Body.Close() + + if !res.IsError() { + var r map[string]interface{} + if err := json.NewDecoder(res.Body).Decode(&r); err == nil { + hit := r["_source"].(map[string]interface{}) + hit["id"] = r["_id"] + if err := json.NewDecoder(esutil.NewJSONReader(hit)).Decode(&result); err != nil { + return false, err + } + return true, nil + } + return false, err + } + return false, errors.New("response error") +} +func DeleteOne(ctx context.Context, es *elasticsearch.Client, indexName string, documentID string) (int64, error) { + req := esapi.DeleteRequest{ + Index: indexName, + DocumentID: documentID, + } + res, err := req.Do(ctx, es) + if err != nil { + return -1, err + } + defer res.Body.Close() + if res.IsError() { + return -1, errors.New("document ID not exists in the index") + } else { + var r map[string]interface{} + if err := json.NewDecoder(res.Body).Decode(&r); err != nil { + return -1, err + } else { + successful := int64(r["_shards"].(map[string]interface{})["successful"].(float64)) + return successful, nil + } + } +} diff --git a/query.go b/query.go index 73f7f89..8e898aa 100644 --- a/query.go +++ b/query.go @@ -11,12 +11,7 @@ import ( "strings" ) -func BuildSearchResult(ctx context.Context, db *elasticsearch.Client, results interface{}, indexName string, query map[string]interface{}, sort []map[string]interface{}, limit int64, offset int64, modelType reflect.Type, options ...func(context.Context, interface{}) (interface{}, error)) (int64, error) { - var mp func(context.Context, interface{}) (interface{}, error) - if len(options) > 0 { - mp = options[0] - } - +func BuildSearchResult(ctx context.Context, db *elasticsearch.Client, results interface{}, indexName string, query map[string]interface{}, sort []map[string]interface{}, limit int64, offset int64, modelType reflect.Type) (int64, error) { from := int(offset) size := int(limit) fullQuery := UpdateQuery(query) @@ -63,11 +58,6 @@ func BuildSearchResult(ctx context.Context, db *elasticsearch.Client, results in if err != nil { return count, err } - - if mp != nil { - MapModels(ctx, results, mp) - } - return count, err } } diff --git a/query/query_builder.go b/query/builder/query_builder.go similarity index 99% rename from query/query_builder.go rename to query/builder/query_builder.go index fb10997..33f1dab 100644 --- a/query/query_builder.go +++ b/query/builder/query_builder.go @@ -1,4 +1,4 @@ -package query +package builder import ( "github.com/core-go/search" diff --git a/repository.go b/repository.go deleted file mode 100644 index 5f9275c..0000000 --- a/repository.go +++ /dev/null @@ -1,20 +0,0 @@ -package elasticsearch - -import ( - es "github.com/elastic/go-elasticsearch/v8" - "reflect" -) - -func NewRepository(client *es.Client, indexName string, modelType reflect.Type, options ...string) *GenericWriter { - return NewGenericWriterWithMapper(client, indexName, modelType, nil, options...) -} -func NewRepositoryWithMapper(client *es.Client, indexName string, modelType reflect.Type, mapper Mapper, options ...string) *GenericWriter { - return NewGenericWriterWithMapper(client, indexName, modelType, mapper, options...) -} - -func NewAdapter(client *es.Client, indexName string, modelType reflect.Type, options ...string) *Writer { - return NewWriterWithMapper(client, indexName, modelType, nil, options...) -} -func NewAdapterWithMapper(client *es.Client, indexName string, modelType reflect.Type, mapper Mapper, options ...string) *Writer { - return NewWriterWithMapper(client, indexName, modelType, mapper, options...) -} diff --git a/search_builder.go b/search_builder.go index 2fc1e8b..6e72971 100644 --- a/search_builder.go +++ b/search_builder.go @@ -7,30 +7,22 @@ import ( "github.com/elastic/go-elasticsearch/v8" ) -type SearchQuery struct { +type SearchBuilder struct { Client *elasticsearch.Client IndexName string BuildQuery func(searchModel interface{}) map[string]interface{} GetSort func(m interface{}) string - Map func(ctx context.Context, model interface{}) (interface{}, error) ModelType reflect.Type } -func NewSearchBuilder(client *elasticsearch.Client, indexName string, modelType reflect.Type, buildQuery func(interface{}) map[string]interface{}, getSort func(m interface{}) string, options ...func(context.Context, interface{}) (interface{}, error)) *SearchQuery { - return NewSearchQuery(client, indexName, modelType, buildQuery, getSort, options...) +func NewSearchBuilder(client *elasticsearch.Client, indexName string, modelType reflect.Type, buildQuery func(interface{}) map[string]interface{}, getSort func(m interface{}) string) *SearchBuilder { + return &SearchBuilder{Client: client, IndexName: indexName, BuildQuery: buildQuery, GetSort: getSort, ModelType: modelType} } -func NewSearchQuery(client *elasticsearch.Client, indexName string, modelType reflect.Type, buildQuery func(interface{}) map[string]interface{}, getSort func(m interface{}) string, options ...func(context.Context, interface{}) (interface{}, error)) *SearchQuery { - var mp func(context.Context, interface{}) (interface{}, error) - if len(options) > 0 { - mp = options[0] - } - return &SearchQuery{Client: client, IndexName: indexName, BuildQuery: buildQuery, GetSort: getSort, Map: mp, ModelType: modelType} -} -func (b *SearchQuery) Search(ctx context.Context, sm interface{}, results interface{}, pageSize int64, skip int64) (int64, error) { +func (b *SearchBuilder) Search(ctx context.Context, sm interface{}, results interface{}, pageSize int64, skip int64) (int64, error) { query := b.BuildQuery(sm) s := b.GetSort(sm) sort := BuildSort(s, b.ModelType) - total, err := BuildSearchResult(ctx, b.Client, results, b.IndexName, query, sort, pageSize, skip, b.ModelType, b.Map) + total, err := BuildSearchResult(ctx, b.Client, results, b.IndexName, query, sort, pageSize, skip, b.ModelType) return total, err } diff --git a/search_load.go b/search_load.go deleted file mode 100644 index fce6a47..0000000 --- a/search_load.go +++ /dev/null @@ -1,19 +0,0 @@ -package elasticsearch - -import ( - "context" - "github.com/elastic/go-elasticsearch/v8" - "reflect" -) - -func NewDefaultSearchLoader(client *elasticsearch.Client, indexName string, modelType reflect.Type, search func(context.Context, interface{}, interface{}, int64, int64) (int64, error), options ...func(context.Context, interface{}) (interface{}, error)) (*Searcher, *Loader) { - searcher := NewSearcher(search) - loader := NewLoader(client, indexName, modelType, options...) - return searcher, loader -} - -func NewSearchLoader(client *elasticsearch.Client, indexName string, modelType reflect.Type, buildQuery func(interface{}) map[string]interface{}, getSort func(m interface{}) string, options ...func(context.Context, interface{}) (interface{}, error)) (*Searcher, *Loader) { - searcher := NewSearcherWithQuery(client, indexName, modelType, buildQuery, getSort, options...) - loader := NewLoader(client, indexName, modelType, options...) - return searcher, loader -} diff --git a/search_write.go b/search_write.go deleted file mode 100644 index 3887074..0000000 --- a/search_write.go +++ /dev/null @@ -1,37 +0,0 @@ -package elasticsearch - -import ( - "context" - "github.com/elastic/go-elasticsearch/v8" - "reflect" -) - -func NewDefaultSearchWriter(client *elasticsearch.Client, indexName string, modelType reflect.Type, search func(context.Context, interface{}, interface{}, int64, int64) (int64, error), options ...string) (*Searcher, *Writer) { - return NewDefaultSearchWriterWithMapper(client, indexName, modelType, search, nil, options...) -} -func NewDefaultSearchWriterWithMapper(client *elasticsearch.Client, indexName string, modelType reflect.Type, search func(context.Context, interface{}, interface{}, int64, int64) (int64, error), mapper Mapper, options ...string) (*Searcher, *Writer) { - var versionField string - if len(options) >= 1 && len(options[0]) > 0 { - versionField = options[0] - } - writer := NewWriterWithMapper(client, indexName, modelType, mapper, versionField) - searcher := NewSearcher(search) - return searcher, writer -} -func NewSearchWriter(client *elasticsearch.Client, indexName string, modelType reflect.Type, buildQuery func(interface{}) map[string]interface{}, getSort func(m interface{}) string, options ...string) (*Searcher, *Writer) { - return NewSearchWriterWithMapper(client, indexName, modelType, buildQuery, getSort, nil, options...) -} -func NewSearchWriterWithMapper(client *elasticsearch.Client, indexName string, modelType reflect.Type, buildQuery func(interface{}) map[string]interface{}, getSort func(m interface{}) string, mapper Mapper, options ...string) (*Searcher, *Writer) { - var versionField string - if len(options) >= 1 && len(options[0]) > 0 { - versionField = options[0] - } - writer := NewWriterWithMapper(client, indexName, modelType, mapper, versionField) - var searcher *Searcher - if mapper != nil { - searcher = NewSearcherWithQuery(client, indexName, modelType, buildQuery, getSort, mapper.DbToModel) - } else { - searcher = NewSearcherWithQuery(client, indexName, modelType, buildQuery, getSort) - } - return searcher, writer -} diff --git a/searcher.go b/searcher.go deleted file mode 100644 index f7dd4f1..0000000 --- a/searcher.go +++ /dev/null @@ -1,24 +0,0 @@ -package elasticsearch - -import ( - "context" - "github.com/elastic/go-elasticsearch/v8" - "reflect" -) - -type Searcher struct { - search func(ctx context.Context, searchModel interface{}, results interface{}, limit int64, offset int64) (int64, error) -} - -func NewSearcher(search func(context.Context, interface{}, interface{}, int64, int64) (int64, error)) *Searcher { - return &Searcher{search: search} -} - -func (s *Searcher) Search(ctx context.Context, m interface{}, results interface{}, limit int64, offset int64) (int64, error) { - return s.search(ctx, m, results, limit, offset) -} - -func NewSearcherWithQuery(client *elasticsearch.Client, indexName string, modelType reflect.Type, buildQuery func(interface{}) map[string]interface{}, getSort func(m interface{}) string, options ...func(context.Context, interface{}) (interface{}, error)) *Searcher { - builder := NewSearchQuery(client, indexName, modelType, buildQuery, getSort, options...) - return NewSearcher(builder.Search) -} diff --git a/writer.go b/writer.go deleted file mode 100644 index d169733..0000000 --- a/writer.go +++ /dev/null @@ -1,63 +0,0 @@ -package elasticsearch - -import ( - "context" - es "github.com/elastic/go-elasticsearch/v8" - "reflect" -) - -type Mapper interface { - DbToModel(ctx context.Context, model interface{}) (interface{}, error) - ModelToDb(ctx context.Context, model interface{}) (interface{}, error) -} - -type Writer struct { - *Loader - maps map[string]string - versionField string - versionIndex int - Mapper Mapper -} - -func NewWriter(client *es.Client, indexName string, modelType reflect.Type, options ...string) *Writer { - return NewWriterWithMapper(client, indexName, modelType, nil, options...) -} -func NewWriterWithMapper(client *es.Client, indexName string, modelType reflect.Type, mapper Mapper, options ...string) *Writer { - var loader *Loader - if mapper != nil { - loader = NewLoader(client, indexName, modelType, mapper.DbToModel) - } else { - loader = NewLoader(client, indexName, modelType) - } - var versionField string - if len(options) >= 1 && len(options[0]) > 0 { - versionField = options[0] - } - if len(versionField) > 0 { - index, _ := FindFieldByName(modelType, versionField) - if index >= 0 { - return &Writer{Loader: loader, maps: MakeMapJson(modelType), versionField: versionField, versionIndex: index} - } - } - return &Writer{Loader: loader, maps: MakeMapJson(modelType), Mapper: mapper, versionField: "", versionIndex: -1} -} - -func (m *Writer) Insert(ctx context.Context, model interface{}) (int64, error) { - return InsertOne(ctx, m.client, m.indexName, model, m.idIndex) -} - -func (m *Writer) Update(ctx context.Context, model interface{}) (int64, error) { - return UpdateOne(ctx, m.client, m.indexName, model, m.idIndex) -} -func (m *Writer) Patch(ctx context.Context, model map[string]interface{}) (int64, error) { - return PatchOne(ctx, m.client, m.indexName, m.jsonIdName, MapToDBObject(model, m.maps)) -} -func (m *Writer) Delete(ctx context.Context, id interface{}) (int64, error) { - sid := id.(string) - return DeleteOne(ctx, m.client, m.indexName, sid) -} -func (m *Writer) Save(ctx context.Context, model interface{}) (int64, error) { - modelValue := reflect.ValueOf(model) - id := modelValue.Field(m.idIndex).String() - return UpsertOne(ctx, m.client, m.indexName, id, model) -} diff --git a/elasticsearch_writer.go b/writer/elasticsearch_writer.go similarity index 98% rename from elasticsearch_writer.go rename to writer/elasticsearch_writer.go index df824de..ed709ef 100644 --- a/elasticsearch_writer.go +++ b/writer/elasticsearch_writer.go @@ -1,4 +1,4 @@ -package elasticsearch +package writer import ( "context" diff --git a/inserter.go b/writer/inserter.go similarity index 98% rename from inserter.go rename to writer/inserter.go index 2d32a25..6e32e85 100644 --- a/inserter.go +++ b/writer/inserter.go @@ -1,4 +1,4 @@ -package elasticsearch +package writer import ( "context" diff --git a/updater.go b/writer/updater.go similarity index 98% rename from updater.go rename to writer/updater.go index a8f7d39..b106615 100644 --- a/updater.go +++ b/writer/updater.go @@ -1,4 +1,4 @@ -package elasticsearch +package writer import ( "context"