Feat/serialize to fix #182

ast/ArgumentList.go

@@ -38,6 +38,27 @@ type ArgumentList struct {
 	Arguments []*Expression
 }
 
+// MakeCatalog will create a catalog entry from ArgumentList node.
+func (e *ArgumentList) MakeCatalog(cat *Catalog) {
+	meta := &ArgumentListMeta{
+		NodeMeta: NodeMeta{
+			AstID:    e.AstID,
+			GrlText:  e.GrlText,
+			Snapshot: e.GetSnapshot(),
+		},
+		ArgumentASTIDs: nil,
+	}
+	if cat.AddMeta(e.AstID, meta) {
+		if e.Arguments != nil && len(e.Arguments) > 0 {
+			meta.ArgumentASTIDs = make([]string, len(e.Arguments))
+			for i, v := range e.Arguments {
+				meta.ArgumentASTIDs[i] = v.AstID
+				v.MakeCatalog(cat)
+			}
+		}
+	}
+}
+
 // Clone will clone this ArgumentList. The new clone will have an identical structure
 func (e *ArgumentList) Clone(cloneTable *pkg.CloneTable) *ArgumentList {
 	clone := &ArgumentList{

ast/ArrayMapSelector.go

@@ -39,6 +39,23 @@ type ArrayMapSelector struct {
 	Value reflect.Value
 }
 
+// MakeCatalog will create a catalog entry from ArrayMapSelector node.
+func (e *ArrayMapSelector) MakeCatalog(cat *Catalog) {
+	meta := &ArrayMapSelectorMeta{
+		NodeMeta: NodeMeta{
+			AstID:    e.AstID,
+			GrlText:  e.GrlText,
+			Snapshot: e.GetSnapshot(),
+		},
+	}
+	if cat.AddMeta(e.AstID, meta) {
+		if e.Expression != nil {
+			meta.ExpressionID = e.Expression.AstID
+			e.Expression.MakeCatalog(cat)
+		}
+	}
+}
+
 // ArrayMapSelectorReceiver must be implemented by all other ast graph that uses map/array selector
 type ArrayMapSelectorReceiver interface {
 	AcceptArrayMapSelector(sel *ArrayMapSelector) error

ast/Assignment.go

@@ -42,6 +42,32 @@ type Assignment struct {
 	IsMulAssign   bool
 }
 
+// MakeCatalog will create a catalog entry from Assignment node.
+func (e *Assignment) MakeCatalog(cat *Catalog) {
+	meta := &AssigmentMeta{
+		NodeMeta: NodeMeta{
+			AstID:    e.AstID,
+			GrlText:  e.GrlText,
+			Snapshot: e.GetSnapshot(),
+		},
+	}
+	if cat.AddMeta(e.AstID, meta) {
+		if e.Variable != nil {
+			meta.VariableID = e.Variable.AstID
+			e.Variable.MakeCatalog(cat)
+		}
+		if e.Expression != nil {
+			meta.ExpressionID = e.Expression.AstID
+			e.Expression.MakeCatalog(cat)
+		}
+		meta.IsAssign = e.IsAssign
+		meta.IsPlusAssign = e.IsPlusAssign
+		meta.IsMinusAssign = e.IsMinusAssign
+		meta.IsDivAssign = e.IsDivAssign
+		meta.IsMulAssign = e.IsMulAssign
+	}
+}
+
 // AssignmentReceiver must be implemented by all other ast graph that uses an assigment expression
 type AssignmentReceiver interface {
 	AcceptAssignment(assignment *Assignment) error

ast/Ast.go

@@ -61,4 +61,5 @@ type Node interface {
 	GetGrlText() string
 	GetSnapshot() string
 	SetGrlText(grlText string)
+	MakeCatalog(cat *Catalog)
 }

ast/Constant.go

@@ -16,8 +16,10 @@ package ast
 
 import (
 	"bytes"
+	"encoding/binary"
 	"fmt"
 	"github.com/hyperjumptech/grule-rule-engine/ast/unique"
+	"math"
 	"reflect"
 
 	"github.com/hyperjumptech/grule-rule-engine/pkg"
@@ -41,6 +43,48 @@ type Constant struct {
 	IsNil         bool
 }
 
+// MakeCatalog will create a catalog entry from Constant node.
+func (e *Constant) MakeCatalog(cat *Catalog) {
+	meta := &ConstantMeta{
+		NodeMeta: NodeMeta{
+			AstID:    e.AstID,
+			GrlText:  e.GrlText,
+			Snapshot: e.GetSnapshot(),
+		},
+	}
+	if cat.AddMeta(e.AstID, meta) {
+		var buff bytes.Buffer
+		switch e.Value.Kind() {
+		case reflect.String:
+			meta.ValueType = TypeString
+			length := make([]byte, 8)
+			data := []byte(e.Value.String())
+			binary.LittleEndian.PutUint64(length, uint64(len(data)))
+			buff.Write(length)
+			buff.Write(data)
+		case reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uint, reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8, reflect.Int:
+			meta.ValueType = TypeInteger
+			intData := make([]byte, 8)
+			binary.LittleEndian.PutUint64(intData, uint64(e.Value.Int()))
+			buff.Write(intData)
+		case reflect.Float32, reflect.Float64:
+			meta.ValueType = TypeFloat
+			floatData := make([]byte, 8)
+			binary.LittleEndian.PutUint64(floatData, math.Float64bits(e.Value.Float()))
+			buff.Write(floatData)
+		case reflect.Bool:
+			meta.ValueType = TypeBoolean
+			if e.Value.Bool() {
+				buff.WriteByte(1)
+			} else {
+				buff.WriteByte(0)
+			}
+		}
+		meta.ValueBytes = buff.Bytes()
+		meta.IsNil = e.IsNil
+	}
+}
+
 // Clone will clone this Constant. The new clone will have an identical structure
 func (e *Constant) Clone(cloneTable *pkg.CloneTable) *Constant {
 	clone := &Constant{

ast/Expression.go

@@ -81,6 +81,37 @@ type Expression struct {
 	Evaluated bool
 }
 
+// MakeCatalog will create a catalog entry from Expression node.
+func (e *Expression) MakeCatalog(cat *Catalog) {
+	meta := &ExpressionMeta{
+		NodeMeta: NodeMeta{
+			AstID:    e.AstID,
+			GrlText:  e.GrlText,
+			Snapshot: e.GetSnapshot(),
+		},
+	}
+	if cat.AddMeta(e.AstID, meta) {
+		if e.LeftExpression != nil {
+			meta.LeftExpressionID = e.LeftExpression.AstID
+			e.LeftExpression.MakeCatalog(cat)
+		}
+		if e.RightExpression != nil {
+			meta.RightExpressionID = e.RightExpression.AstID
+			e.RightExpression.MakeCatalog(cat)
+		}
+		if e.SingleExpression != nil {
+			meta.SingleExpressionID = e.SingleExpression.AstID
+			e.SingleExpression.MakeCatalog(cat)
+		}
+		if e.ExpressionAtom != nil {
+			meta.ExpressionAtomID = e.ExpressionAtom.AstID
+			e.ExpressionAtom.MakeCatalog(cat)
+		}
+		meta.Operator = e.Operator
+		meta.Negated = e.Negated
+	}
+}
+
 // Clone will clone this Expression. The new clone will have an identical structure
 func (e *Expression) Clone(cloneTable *pkg.CloneTable) *Expression {
 	clone := &Expression{

ast/ExpressionAtom.go

@@ -43,13 +43,49 @@ type ExpressionAtom struct {
 	Variable         *Variable
 	Negated          bool
 	ExpressionAtom   *ExpressionAtom
-	Value            reflect.Value
-	ValueNode        model.ValueNode
 	ArrayMapSelector *ArrayMapSelector
 
+	Value     reflect.Value
+	ValueNode model.ValueNode
+
 	Evaluated bool
 }
 
+// MakeCatalog will create a catalog entry from ExpressionAtom node.
+func (e *ExpressionAtom) MakeCatalog(cat *Catalog) {
+	meta := &ExpressionAtomMeta{
+		NodeMeta: NodeMeta{
+			AstID:    e.AstID,
+			GrlText:  e.GrlText,
+			Snapshot: e.GetSnapshot(),
+		},
+	}
+	if cat.AddMeta(e.AstID, meta) {
+		if e.Constant != nil {
+			meta.ConstantID = e.Constant.AstID
+			e.Constant.MakeCatalog(cat)
+		}
+		if e.FunctionCall != nil {
+			meta.FunctionCallID = e.FunctionCall.AstID
+			e.FunctionCall.MakeCatalog(cat)
+		}
+		if e.Variable != nil {
+			meta.VariableID = e.Variable.AstID
+			e.Variable.MakeCatalog(cat)
+		}
+		if e.ExpressionAtom != nil {
+			meta.ExpressionAtomID = e.ExpressionAtom.AstID
+			e.ExpressionAtom.MakeCatalog(cat)
+		}
+		if e.ArrayMapSelector != nil {
+			meta.ArrayMapSelectorID = e.ArrayMapSelector.AstID
+			e.ArrayMapSelector.MakeCatalog(cat)
+		}
+		meta.VariableName = e.VariableName
+		meta.Negated = e.Negated
+	}
+}
+
 // ExpressionAtomReceiver contains function to be implemented by other AST graph to receive an ExpressionAtom AST graph
 type ExpressionAtomReceiver interface {
 	AcceptExpressionAtom(exp *ExpressionAtom) error

ast/FunctionCall.go

@@ -40,6 +40,24 @@ type FunctionCall struct {
 	Value        reflect.Value
 }
 
+// MakeCatalog will create a catalog entry from FunctionCall node.
+func (e *FunctionCall) MakeCatalog(cat *Catalog) {
+	meta := &FunctionCallMeta{
+		NodeMeta: NodeMeta{
+			AstID:    e.AstID,
+			GrlText:  e.GrlText,
+			Snapshot: e.GetSnapshot(),
+		},
+	}
+	if cat.AddMeta(e.AstID, meta) {
+		if e.ArgumentList != nil {
+			meta.ArgumentListID = e.ArgumentList.AstID
+			e.ArgumentList.MakeCatalog(cat)
+		}
+		meta.FunctionName = e.FunctionName
+	}
+}
+
 // Clone will clone this FunctionCall. The new clone will have an identical structure
 func (e *FunctionCall) Clone(cloneTable *pkg.CloneTable) *FunctionCall {
 	clone := &FunctionCall{

ast/KnowledgeBase.go

@@ -17,6 +17,8 @@ package ast
 import (
 	"bytes"
 	"fmt"
+	"github.com/sirupsen/logrus"
+	"io"
 	"sort"
 	"strings"
 	"sync"
@@ -54,6 +56,50 @@ func (lib *KnowledgeLibrary) GetKnowledgeBase(name, version string) *KnowledgeBa
 	return kb
 }
 
+// LoadKnowledgeBaseFromReader will load the KnowledgeBase stored using StoreKnowledgeBaseToWriter function
+// be it from file, or anywhere. The reader we needed is a plain io.Reader, thus closing the source stream is your responsibility.
+// This should hopefully speedup loading huge ruleset by storing and reading them
+// without having to parse the GRL.
+func (lib *KnowledgeLibrary) LoadKnowledgeBaseFromReader(reader io.Reader, overwrite bool) (retKb *KnowledgeBase, retErr error) {
+	defer func() {
+		if r := recover(); r != nil {
+			retKb = nil
+			logrus.Panicf("panic recovered during LoadKnowledgeBaseFromReader. send us your report to https://github.com/hyperjumptech/grule-rule-engine/issues")
+		}
+	}()
+
+	catalog := &Catalog{}
+	err := catalog.ReadCatalogFromReader(reader)
+	if err != nil {
+		return nil, err
+	}
+	kb := catalog.BuildKnowledgeBase()
+	if overwrite {
+		lib.Library[fmt.Sprintf("%s:%s", kb.Name, kb.Version)] = kb
+		return kb, nil
+	}
+	if _, ok := lib.Library[fmt.Sprintf("%s:%s", kb.Name, kb.Version)]; !ok {
+		lib.Library[fmt.Sprintf("%s:%s", kb.Name, kb.Version)] = kb
+		return kb, nil
+	}
+	return nil, fmt.Errorf("KnowledgeBase %s version %s exist", kb.Name, kb.Version)
+}
+
+// StoreKnowledgeBaseToWriter will store a KnowledgeBase in binary form
+// once store, the binary stream can be read using LoadKnowledgeBaseFromReader function.
+// This should hopefully speedup loading huge ruleset by storing and reading them
+// without having to parse the GRL.
+//
+// The stored binary file is greatly increased (easily 10x fold) due to lots of generated keys for AST Nodes
+// that was also saved. To overcome this, the use of archive/zip package for Readers and Writers could cut down the
+// binary size quite a lot.
+func (lib *KnowledgeLibrary) StoreKnowledgeBaseToWriter(writer io.Writer, name, version string) error {
+	kb := lib.GetKnowledgeBase(name, version)
+	cat := kb.MakeCatalog()
+	err := cat.WriteCatalogToWriter(writer)
+	return err
+}
+
 // NewKnowledgeBaseInstance will create a new instance based on KnowledgeBase blue print
 // identified by its name and version
 func (lib *KnowledgeLibrary) NewKnowledgeBaseInstance(name, version string) *KnowledgeBase {
@@ -81,8 +127,33 @@ type KnowledgeBase struct {
 	RuleEntries   map[string]*RuleEntry
 }
 
+// MakeCatalog will create a catalog entry for all AST Nodes under the KnowledgeBase
+// the catalog can be used to save the knowledge base into a Writer, or to
+// rebuild the KnowledgeBase from it.
+// This function also will catalog the WorkingMemory.
+func (e *KnowledgeBase) MakeCatalog() *Catalog {
+	catalog := &Catalog{
+		KnowledgeBaseName:               e.Name,
+		KnowledgeBaseVersion:            e.Version,
+		Data:                            nil,
+		MemoryName:                      "",
+		MemoryVersion:                   "",
+		MemoryVariableSnapshotMap:       nil,
+		MemoryExpressionSnapshotMap:     nil,
+		MemoryExpressionAtomSnapshotMap: nil,
+		MemoryExpressionVariableMap:     nil,
+		MemoryExpressionAtomVariableMap: nil,
+	}
+	for _, v := range e.RuleEntries {
+		v.MakeCatalog(catalog)
+	}
+	e.WorkingMemory.MakeCatalog(catalog)
+	return catalog
+}
+
 // IsIdentical will validate if two KnoledgeBase is identical. Used to validate if the origin and clone is identical.
 func (e *KnowledgeBase) IsIdentical(that *KnowledgeBase) bool {
+	// fmt.Printf("%s\n%s\n", e.GetSnapshot(), that.GetSnapshot())
 	return e.GetSnapshot() == that.GetSnapshot()
 }
 

ast/RuleEntry.go

@@ -47,6 +47,30 @@ type RuleEntry struct {
 	Retracted bool
 }
 
+// MakeCatalog will create a catalog entry from RuleEntry node.
+func (e *RuleEntry) MakeCatalog(cat *Catalog) {
+	meta := &RuleEntryMeta{
+		NodeMeta: NodeMeta{
+			AstID:    e.AstID,
+			GrlText:  e.GrlText,
+			Snapshot: e.GetSnapshot(),
+		},
+	}
+	if cat.AddMeta(e.AstID, meta) {
+		if e.WhenScope != nil {
+			meta.WhenScopeID = e.WhenScope.AstID
+			e.WhenScope.MakeCatalog(cat)
+		}
+		if e.ThenScope != nil {
+			meta.ThenScopeID = e.ThenScope.AstID
+			e.ThenScope.MakeCatalog(cat)
+		}
+		meta.RuleName = e.RuleName
+		meta.RuleDescription = e.RuleDescription
+		meta.Salience = e.Salience
+	}
+}
+
 // RuleEntryReceiver should be implemented by any rule AST object that receive a RuleEntry
 type RuleEntryReceiver interface {
 	ReceiveRuleEntry(entry *RuleEntry) error