Skip to content

Commit

Permalink
Feat/serialize to fix #182 (#183)
Browse files Browse the repository at this point in the history
* added catalog to save into binnary, whats left is the reverse process

* Finished serializer, whats left is to save/load to file

* Get all test and function in library to read and write knowledgebase from reader and into writer

* Committed some license notice and a little bit more test

* Some additional node of the size of binary that inflated compared to the original GRL size
newm4n authored Apr 12, 2021
1 parent 90f4cf9 commit 8c7b275
Showing 22 changed files with 3,036 additions and 3 deletions.
21 changes: 21 additions & 0 deletions ast/ArgumentList.go
Original file line number Diff line number Diff line change
@@ -38,6 +38,27 @@ type ArgumentList struct {
Arguments []*Expression
}

// MakeCatalog will create a catalog entry from ArgumentList node.
func (e *ArgumentList) MakeCatalog(cat *Catalog) {
meta := &ArgumentListMeta{
NodeMeta: NodeMeta{
AstID: e.AstID,
GrlText: e.GrlText,
Snapshot: e.GetSnapshot(),
},
ArgumentASTIDs: nil,
}
if cat.AddMeta(e.AstID, meta) {
if e.Arguments != nil && len(e.Arguments) > 0 {
meta.ArgumentASTIDs = make([]string, len(e.Arguments))
for i, v := range e.Arguments {
meta.ArgumentASTIDs[i] = v.AstID
v.MakeCatalog(cat)
}
}
}
}

// Clone will clone this ArgumentList. The new clone will have an identical structure
func (e *ArgumentList) Clone(cloneTable *pkg.CloneTable) *ArgumentList {
clone := &ArgumentList{
17 changes: 17 additions & 0 deletions ast/ArrayMapSelector.go
Original file line number Diff line number Diff line change
@@ -39,6 +39,23 @@ type ArrayMapSelector struct {
Value reflect.Value
}

// MakeCatalog will create a catalog entry from ArrayMapSelector node.
func (e *ArrayMapSelector) MakeCatalog(cat *Catalog) {
meta := &ArrayMapSelectorMeta{
NodeMeta: NodeMeta{
AstID: e.AstID,
GrlText: e.GrlText,
Snapshot: e.GetSnapshot(),
},
}
if cat.AddMeta(e.AstID, meta) {
if e.Expression != nil {
meta.ExpressionID = e.Expression.AstID
e.Expression.MakeCatalog(cat)
}
}
}

// ArrayMapSelectorReceiver must be implemented by all other ast graph that uses map/array selector
type ArrayMapSelectorReceiver interface {
AcceptArrayMapSelector(sel *ArrayMapSelector) error
26 changes: 26 additions & 0 deletions ast/Assignment.go
Original file line number Diff line number Diff line change
@@ -42,6 +42,32 @@ type Assignment struct {
IsMulAssign bool
}

// MakeCatalog will create a catalog entry from Assignment node.
func (e *Assignment) MakeCatalog(cat *Catalog) {
meta := &AssigmentMeta{
NodeMeta: NodeMeta{
AstID: e.AstID,
GrlText: e.GrlText,
Snapshot: e.GetSnapshot(),
},
}
if cat.AddMeta(e.AstID, meta) {
if e.Variable != nil {
meta.VariableID = e.Variable.AstID
e.Variable.MakeCatalog(cat)
}
if e.Expression != nil {
meta.ExpressionID = e.Expression.AstID
e.Expression.MakeCatalog(cat)
}
meta.IsAssign = e.IsAssign
meta.IsPlusAssign = e.IsPlusAssign
meta.IsMinusAssign = e.IsMinusAssign
meta.IsDivAssign = e.IsDivAssign
meta.IsMulAssign = e.IsMulAssign
}
}

// AssignmentReceiver must be implemented by all other ast graph that uses an assigment expression
type AssignmentReceiver interface {
AcceptAssignment(assignment *Assignment) error
1 change: 1 addition & 0 deletions ast/Ast.go
Original file line number Diff line number Diff line change
@@ -61,4 +61,5 @@ type Node interface {
GetGrlText() string
GetSnapshot() string
SetGrlText(grlText string)
MakeCatalog(cat *Catalog)
}
44 changes: 44 additions & 0 deletions ast/Constant.go
Original file line number Diff line number Diff line change
@@ -16,8 +16,10 @@ package ast

import (
"bytes"
"encoding/binary"
"fmt"
"github.com/hyperjumptech/grule-rule-engine/ast/unique"
"math"
"reflect"

"github.com/hyperjumptech/grule-rule-engine/pkg"
@@ -41,6 +43,48 @@ type Constant struct {
IsNil bool
}

// MakeCatalog will create a catalog entry from Constant node.
func (e *Constant) MakeCatalog(cat *Catalog) {
meta := &ConstantMeta{
NodeMeta: NodeMeta{
AstID: e.AstID,
GrlText: e.GrlText,
Snapshot: e.GetSnapshot(),
},
}
if cat.AddMeta(e.AstID, meta) {
var buff bytes.Buffer
switch e.Value.Kind() {
case reflect.String:
meta.ValueType = TypeString
length := make([]byte, 8)
data := []byte(e.Value.String())
binary.LittleEndian.PutUint64(length, uint64(len(data)))
buff.Write(length)
buff.Write(data)
case reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8, reflect.Uint, reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8, reflect.Int:
meta.ValueType = TypeInteger
intData := make([]byte, 8)
binary.LittleEndian.PutUint64(intData, uint64(e.Value.Int()))
buff.Write(intData)
case reflect.Float32, reflect.Float64:
meta.ValueType = TypeFloat
floatData := make([]byte, 8)
binary.LittleEndian.PutUint64(floatData, math.Float64bits(e.Value.Float()))
buff.Write(floatData)
case reflect.Bool:
meta.ValueType = TypeBoolean
if e.Value.Bool() {
buff.WriteByte(1)
} else {
buff.WriteByte(0)
}
}
meta.ValueBytes = buff.Bytes()
meta.IsNil = e.IsNil
}
}

// Clone will clone this Constant. The new clone will have an identical structure
func (e *Constant) Clone(cloneTable *pkg.CloneTable) *Constant {
clone := &Constant{
31 changes: 31 additions & 0 deletions ast/Expression.go
Original file line number Diff line number Diff line change
@@ -81,6 +81,37 @@ type Expression struct {
Evaluated bool
}

// MakeCatalog will create a catalog entry from Expression node.
func (e *Expression) MakeCatalog(cat *Catalog) {
meta := &ExpressionMeta{
NodeMeta: NodeMeta{
AstID: e.AstID,
GrlText: e.GrlText,
Snapshot: e.GetSnapshot(),
},
}
if cat.AddMeta(e.AstID, meta) {
if e.LeftExpression != nil {
meta.LeftExpressionID = e.LeftExpression.AstID
e.LeftExpression.MakeCatalog(cat)
}
if e.RightExpression != nil {
meta.RightExpressionID = e.RightExpression.AstID
e.RightExpression.MakeCatalog(cat)
}
if e.SingleExpression != nil {
meta.SingleExpressionID = e.SingleExpression.AstID
e.SingleExpression.MakeCatalog(cat)
}
if e.ExpressionAtom != nil {
meta.ExpressionAtomID = e.ExpressionAtom.AstID
e.ExpressionAtom.MakeCatalog(cat)
}
meta.Operator = e.Operator
meta.Negated = e.Negated
}
}

// Clone will clone this Expression. The new clone will have an identical structure
func (e *Expression) Clone(cloneTable *pkg.CloneTable) *Expression {
clone := &Expression{
40 changes: 38 additions & 2 deletions ast/ExpressionAtom.go
Original file line number Diff line number Diff line change
@@ -43,13 +43,49 @@ type ExpressionAtom struct {
Variable *Variable
Negated bool
ExpressionAtom *ExpressionAtom
Value reflect.Value
ValueNode model.ValueNode
ArrayMapSelector *ArrayMapSelector

Value reflect.Value
ValueNode model.ValueNode

Evaluated bool
}

// MakeCatalog will create a catalog entry from ExpressionAtom node.
func (e *ExpressionAtom) MakeCatalog(cat *Catalog) {
meta := &ExpressionAtomMeta{
NodeMeta: NodeMeta{
AstID: e.AstID,
GrlText: e.GrlText,
Snapshot: e.GetSnapshot(),
},
}
if cat.AddMeta(e.AstID, meta) {
if e.Constant != nil {
meta.ConstantID = e.Constant.AstID
e.Constant.MakeCatalog(cat)
}
if e.FunctionCall != nil {
meta.FunctionCallID = e.FunctionCall.AstID
e.FunctionCall.MakeCatalog(cat)
}
if e.Variable != nil {
meta.VariableID = e.Variable.AstID
e.Variable.MakeCatalog(cat)
}
if e.ExpressionAtom != nil {
meta.ExpressionAtomID = e.ExpressionAtom.AstID
e.ExpressionAtom.MakeCatalog(cat)
}
if e.ArrayMapSelector != nil {
meta.ArrayMapSelectorID = e.ArrayMapSelector.AstID
e.ArrayMapSelector.MakeCatalog(cat)
}
meta.VariableName = e.VariableName
meta.Negated = e.Negated
}
}

// ExpressionAtomReceiver contains function to be implemented by other AST graph to receive an ExpressionAtom AST graph
type ExpressionAtomReceiver interface {
AcceptExpressionAtom(exp *ExpressionAtom) error
18 changes: 18 additions & 0 deletions ast/FunctionCall.go
Original file line number Diff line number Diff line change
@@ -40,6 +40,24 @@ type FunctionCall struct {
Value reflect.Value
}

// MakeCatalog will create a catalog entry from FunctionCall node.
func (e *FunctionCall) MakeCatalog(cat *Catalog) {
meta := &FunctionCallMeta{
NodeMeta: NodeMeta{
AstID: e.AstID,
GrlText: e.GrlText,
Snapshot: e.GetSnapshot(),
},
}
if cat.AddMeta(e.AstID, meta) {
if e.ArgumentList != nil {
meta.ArgumentListID = e.ArgumentList.AstID
e.ArgumentList.MakeCatalog(cat)
}
meta.FunctionName = e.FunctionName
}
}

// Clone will clone this FunctionCall. The new clone will have an identical structure
func (e *FunctionCall) Clone(cloneTable *pkg.CloneTable) *FunctionCall {
clone := &FunctionCall{
71 changes: 71 additions & 0 deletions ast/KnowledgeBase.go
Original file line number Diff line number Diff line change
@@ -17,6 +17,8 @@ package ast
import (
"bytes"
"fmt"
"github.com/sirupsen/logrus"
"io"
"sort"
"strings"
"sync"
@@ -54,6 +56,50 @@ func (lib *KnowledgeLibrary) GetKnowledgeBase(name, version string) *KnowledgeBa
return kb
}

// LoadKnowledgeBaseFromReader will load the KnowledgeBase stored using StoreKnowledgeBaseToWriter function
// be it from file, or anywhere. The reader we needed is a plain io.Reader, thus closing the source stream is your responsibility.
// This should hopefully speedup loading huge ruleset by storing and reading them
// without having to parse the GRL.
func (lib *KnowledgeLibrary) LoadKnowledgeBaseFromReader(reader io.Reader, overwrite bool) (retKb *KnowledgeBase, retErr error) {
defer func() {
if r := recover(); r != nil {
retKb = nil
logrus.Panicf("panic recovered during LoadKnowledgeBaseFromReader. send us your report to https://github.com/hyperjumptech/grule-rule-engine/issues")
}
}()

catalog := &Catalog{}
err := catalog.ReadCatalogFromReader(reader)
if err != nil {
return nil, err
}
kb := catalog.BuildKnowledgeBase()
if overwrite {
lib.Library[fmt.Sprintf("%s:%s", kb.Name, kb.Version)] = kb
return kb, nil
}
if _, ok := lib.Library[fmt.Sprintf("%s:%s", kb.Name, kb.Version)]; !ok {
lib.Library[fmt.Sprintf("%s:%s", kb.Name, kb.Version)] = kb
return kb, nil
}
return nil, fmt.Errorf("KnowledgeBase %s version %s exist", kb.Name, kb.Version)
}

// StoreKnowledgeBaseToWriter will store a KnowledgeBase in binary form
// once store, the binary stream can be read using LoadKnowledgeBaseFromReader function.
// This should hopefully speedup loading huge ruleset by storing and reading them
// without having to parse the GRL.
//
// The stored binary file is greatly increased (easily 10x fold) due to lots of generated keys for AST Nodes
// that was also saved. To overcome this, the use of archive/zip package for Readers and Writers could cut down the
// binary size quite a lot.
func (lib *KnowledgeLibrary) StoreKnowledgeBaseToWriter(writer io.Writer, name, version string) error {
kb := lib.GetKnowledgeBase(name, version)
cat := kb.MakeCatalog()
err := cat.WriteCatalogToWriter(writer)
return err
}

// NewKnowledgeBaseInstance will create a new instance based on KnowledgeBase blue print
// identified by its name and version
func (lib *KnowledgeLibrary) NewKnowledgeBaseInstance(name, version string) *KnowledgeBase {
@@ -81,8 +127,33 @@ type KnowledgeBase struct {
RuleEntries map[string]*RuleEntry
}

// MakeCatalog will create a catalog entry for all AST Nodes under the KnowledgeBase
// the catalog can be used to save the knowledge base into a Writer, or to
// rebuild the KnowledgeBase from it.
// This function also will catalog the WorkingMemory.
func (e *KnowledgeBase) MakeCatalog() *Catalog {
catalog := &Catalog{
KnowledgeBaseName: e.Name,
KnowledgeBaseVersion: e.Version,
Data: nil,
MemoryName: "",
MemoryVersion: "",
MemoryVariableSnapshotMap: nil,
MemoryExpressionSnapshotMap: nil,
MemoryExpressionAtomSnapshotMap: nil,
MemoryExpressionVariableMap: nil,
MemoryExpressionAtomVariableMap: nil,
}
for _, v := range e.RuleEntries {
v.MakeCatalog(catalog)
}
e.WorkingMemory.MakeCatalog(catalog)
return catalog
}

// IsIdentical will validate if two KnoledgeBase is identical. Used to validate if the origin and clone is identical.
func (e *KnowledgeBase) IsIdentical(that *KnowledgeBase) bool {
// fmt.Printf("%s\n%s\n", e.GetSnapshot(), that.GetSnapshot())
return e.GetSnapshot() == that.GetSnapshot()
}

24 changes: 24 additions & 0 deletions ast/RuleEntry.go
Original file line number Diff line number Diff line change
@@ -47,6 +47,30 @@ type RuleEntry struct {
Retracted bool
}

// MakeCatalog will create a catalog entry from RuleEntry node.
func (e *RuleEntry) MakeCatalog(cat *Catalog) {
meta := &RuleEntryMeta{
NodeMeta: NodeMeta{
AstID: e.AstID,
GrlText: e.GrlText,
Snapshot: e.GetSnapshot(),
},
}
if cat.AddMeta(e.AstID, meta) {
if e.WhenScope != nil {
meta.WhenScopeID = e.WhenScope.AstID
e.WhenScope.MakeCatalog(cat)
}
if e.ThenScope != nil {
meta.ThenScopeID = e.ThenScope.AstID
e.ThenScope.MakeCatalog(cat)
}
meta.RuleName = e.RuleName
meta.RuleDescription = e.RuleDescription
meta.Salience = e.Salience
}
}

// RuleEntryReceiver should be implemented by any rule AST object that receive a RuleEntry
type RuleEntryReceiver interface {
ReceiveRuleEntry(entry *RuleEntry) error
Loading

0 comments on commit 8c7b275

Please sign in to comment.