From 964fd917e6d295a375191dbd3779221d630d55ba Mon Sep 17 00:00:00 2001 From: "A. Athan" <24279435+aathan@users.noreply.github.com> Date: Wed, 9 Aug 2023 00:55:47 +0000 Subject: [PATCH 1/5] better detection of pointer receivers --- encode.go | 59 ++++++++++++++++++++++++++----------------- yaml.go | 75 ++++++++++++++++++++++++++----------------------------- 2 files changed, 71 insertions(+), 63 deletions(-) diff --git a/encode.go b/encode.go index de9e72a3..2ac567ba 100644 --- a/encode.go +++ b/encode.go @@ -24,7 +24,6 @@ import ( "sort" "strconv" "strings" - "time" "unicode/utf8" ) @@ -110,31 +109,34 @@ func (e *encoder) marshalDoc(tag string, in reflect.Value) { func (e *encoder) marshal(tag string, in reflect.Value) { tag = shortTag(tag) - if !in.IsValid() || in.Kind() == reflect.Ptr && in.IsNil() { + if !in.IsValid() { //|| in.Kind() == reflect.Ptr && in.IsNil() { e.nilv() return } + iface := in.Interface() - switch value := iface.(type) { - case *Node: - e.nodev(in) + if iface == nil { + e.nilv() return - case Node: - if !in.CanAddr() { - var n = reflect.New(in.Type()).Elem() + } + + if in.Kind() != reflect.Ptr && in.Kind() != reflect.Interface { + if in.CanAddr() { + iface = in.Addr().Interface() + } else { + // uh, not sure about this but look at this project's original Node: case... + n := reflect.New(in.Type()).Elem() n.Set(in) - in = n + iface = n.Addr().Interface() } - e.nodev(in.Addr()) - return - case time.Time: - e.timev(tag, in) - return - case *time.Time: - e.timev(tag, in.Elem()) - return - case time.Duration: - e.stringv(tag, reflect.ValueOf(value.String())) + } + + switch value := iface.(type) { + //case *time.Duration: + // e.stringv(tag, reflect.ValueOf(value.String())) + // return + case *Node: + e.nodev(in) return case Marshaler: v, err := value.MarshalYAML() @@ -143,20 +145,25 @@ func (e *encoder) marshal(tag string, in reflect.Value) { } if v == nil { e.nilv() - return + } else { + e.marshal(tag, reflect.ValueOf(v)) } - e.marshal(tag, reflect.ValueOf(v)) return case encoding.TextMarshaler: text, err := value.MarshalText() if err != nil { fail(err) } - in = reflect.ValueOf(string(text)) + e.rawstringv(tag, string(text)) + return + case fmt.Stringer: + e.rawstringv(tag, value.String()) + return case nil: e.nilv() return } + switch in.Kind() { case reflect.Interface: e.marshal(tag, in.Elem()) @@ -322,8 +329,12 @@ func isOldBool(s string) (result bool) { } func (e *encoder) stringv(tag string, in reflect.Value) { - var style yaml_scalar_style_t s := in.String() + e.rawstringv(tag, s) +} + +func (e *encoder) rawstringv(tag string, s string) { + var style yaml_scalar_style_t canUsePlain := true switch { case !utf8.ValidString(s): @@ -382,11 +393,13 @@ func (e *encoder) uintv(tag string, in reflect.Value) { e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil) } +/* Handled by TextMarshaler in time.Time func (e *encoder) timev(tag string, in reflect.Value) { t := in.Interface().(time.Time) s := t.Format(time.RFC3339Nano) e.emitScalar(s, "", tag, yaml_PLAIN_SCALAR_STYLE, nil, nil, nil, nil) } +*/ func (e *encoder) floatv(tag string, in reflect.Value) { // Issue #352: When formatting, use the precision of the underlying value diff --git a/yaml.go b/yaml.go index 8cec6da4..f0bedf3d 100644 --- a/yaml.go +++ b/yaml.go @@ -17,8 +17,7 @@ // // Source code and other details for the project are available at GitHub: // -// https://github.com/go-yaml/yaml -// +// https://github.com/go-yaml/yaml package yaml import ( @@ -75,16 +74,15 @@ type Marshaler interface { // // For example: // -// type T struct { -// F int `yaml:"a,omitempty"` -// B int -// } -// var t T -// yaml.Unmarshal([]byte("a: 1\nb: 2"), &t) +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// var t T +// yaml.Unmarshal([]byte("a: 1\nb: 2"), &t) // // See the documentation of Marshal for the format of tags and a list of // supported tag options. -// func Unmarshal(in []byte, out interface{}) (err error) { return unmarshal(in, out, false) } @@ -185,36 +183,35 @@ func unmarshal(in []byte, out interface{}, strict bool) (err error) { // // The field tag format accepted is: // -// `(...) yaml:"[][,[,]]" (...)` +// `(...) yaml:"[][,[,]]" (...)` // // The following flags are currently supported: // -// omitempty Only include the field if it's not set to the zero -// value for the type or to empty slices or maps. -// Zero valued structs will be omitted if all their public -// fields are zero, unless they implement an IsZero -// method (see the IsZeroer interface type), in which -// case the field will be excluded if IsZero returns true. +// omitempty Only include the field if it's not set to the zero +// value for the type or to empty slices or maps. +// Zero valued structs will be omitted if all their public +// fields are zero, unless they implement an IsZero +// method (see the IsZeroer interface type), in which +// case the field will be excluded if IsZero returns true. // -// flow Marshal using a flow style (useful for structs, -// sequences and maps). +// flow Marshal using a flow style (useful for structs, +// sequences and maps). // -// inline Inline the field, which must be a struct or a map, -// causing all of its fields or keys to be processed as if -// they were part of the outer struct. For maps, keys must -// not conflict with the yaml keys of other struct fields. +// inline Inline the field, which must be a struct or a map, +// causing all of its fields or keys to be processed as if +// they were part of the outer struct. For maps, keys must +// not conflict with the yaml keys of other struct fields. // // In addition, if the key is "-", the field is ignored. // // For example: // -// type T struct { -// F int `yaml:"a,omitempty"` -// B int -// } -// yaml.Marshal(&T{B: 2}) // Returns "b: 2\n" -// yaml.Marshal(&T{F: 1}} // Returns "a: 1\nb: 0\n" -// +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// yaml.Marshal(&T{B: 2}) // Returns "b: 2\n" +// yaml.Marshal(&T{F: 1}} // Returns "a: 1\nb: 0\n" func Marshal(in interface{}) (out []byte, err error) { defer handleErr(&err) e := newEncoder() @@ -358,22 +355,21 @@ const ( // // For example: // -// var person struct { -// Name string -// Address yaml.Node -// } -// err := yaml.Unmarshal(data, &person) -// -// Or by itself: +// var person struct { +// Name string +// Address yaml.Node +// } +// err := yaml.Unmarshal(data, &person) // -// var person Node -// err := yaml.Unmarshal(data, &person) +// Or by itself: // +// var person Node +// err := yaml.Unmarshal(data, &person) type Node struct { // Kind defines whether the node is a document, a mapping, a sequence, // a scalar value, or an alias to another node. The specific data type of // scalar nodes may be obtained via the ShortTag and LongTag methods. - Kind Kind + Kind Kind // Style allows customizing the apperance of the node in the tree. Style Style @@ -421,7 +417,6 @@ func (n *Node) IsZero() bool { n.HeadComment == "" && n.LineComment == "" && n.FootComment == "" && n.Line == 0 && n.Column == 0 } - // LongTag returns the long form of the tag that indicates the data type for // the node. If the Tag field isn't explicitly defined, one will be computed // based on the node properties. From 037bd8c766557c84028d861853e48e27d46d9bea Mon Sep 17 00:00:00 2001 From: "A. Athan" <24279435+aathan@users.noreply.github.com> Date: Wed, 9 Aug 2023 03:27:17 +0000 Subject: [PATCH 2/5] marshal opts --- decode.go | 2 +- encode.go | 37 ++++++++++++------ go.mod | 8 ++-- yaml.go | 111 +++++++++++++++++++++++++++++++----------------------- 4 files changed, 94 insertions(+), 64 deletions(-) diff --git a/decode.go b/decode.go index 0173b698..2241e647 100644 --- a/decode.go +++ b/decode.go @@ -876,7 +876,7 @@ func isStringMap(n *Node) bool { } func (d *decoder) mappingStruct(n *Node, out reflect.Value) (good bool) { - sinfo, err := getStructInfo(out.Type()) + sinfo, err := getStructInfo(out.Type(), false) if err != nil { panic(err) } diff --git a/encode.go b/encode.go index de9e72a3..c6dcba09 100644 --- a/encode.go +++ b/encode.go @@ -29,24 +29,37 @@ import ( ) type encoder struct { - emitter yaml_emitter_t - event yaml_event_t - out []byte - flow bool - indent int - doneInit bool + emitter yaml_emitter_t + event yaml_event_t + out []byte + flow bool + indent int + doneInit bool + originalCase bool + arg any } -func newEncoder() *encoder { - e := &encoder{} +func newEncoder_(arg any, opts ...MarshalOpt) (e *encoder) { + e = &encoder{} + e.arg = arg + for _, o := range opts { + if o == OriginalCase { + e.originalCase = true + } + } + return +} + +func newEncoder(arg any, opts ...MarshalOpt) *encoder { + e := newEncoder_(arg, opts...) yaml_emitter_initialize(&e.emitter) yaml_emitter_set_output_string(&e.emitter, &e.out) yaml_emitter_set_unicode(&e.emitter, true) return e } -func newEncoderWithWriter(w io.Writer) *encoder { - e := &encoder{} +func newEncoderWithWriter(w io.Writer, arg any, opts ...MarshalOpt) *encoder { + e := newEncoder_(arg, opts...) yaml_emitter_initialize(&e.emitter) yaml_emitter_set_output_writer(&e.emitter, w) yaml_emitter_set_unicode(&e.emitter, true) @@ -137,7 +150,7 @@ func (e *encoder) marshal(tag string, in reflect.Value) { e.stringv(tag, reflect.ValueOf(value.String())) return case Marshaler: - v, err := value.MarshalYAML() + v, err := value.MarshalYAML(e.arg) if err != nil { fail(err) } @@ -212,7 +225,7 @@ func (e *encoder) fieldByIndex(v reflect.Value, index []int) (field reflect.Valu } func (e *encoder) structv(tag string, in reflect.Value) { - sinfo, err := getStructInfo(in.Type()) + sinfo, err := getStructInfo(in.Type(), e.originalCase) if err != nil { panic(err) } diff --git a/go.mod b/go.mod index f407ea32..057d59b1 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,5 @@ -module "gopkg.in/yaml.v3" +module gopkg.in/yaml.v3 -require ( - "gopkg.in/check.v1" v0.0.0-20161208181325-20d25e280405 -) +go 1.19 + +require gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 diff --git a/yaml.go b/yaml.go index 8cec6da4..25a45dc0 100644 --- a/yaml.go +++ b/yaml.go @@ -17,8 +17,7 @@ // // Source code and other details for the project are available at GitHub: // -// https://github.com/go-yaml/yaml -// +// https://github.com/go-yaml/yaml package yaml import ( @@ -48,7 +47,7 @@ type obsoleteUnmarshaler interface { // If an error is returned by MarshalYAML, the marshaling procedure stops // and returns with the provided error. type Marshaler interface { - MarshalYAML() (interface{}, error) + MarshalYAML(arg any) (interface{}, error) } // Unmarshal decodes the first document found within the in byte slice @@ -75,16 +74,15 @@ type Marshaler interface { // // For example: // -// type T struct { -// F int `yaml:"a,omitempty"` -// B int -// } -// var t T -// yaml.Unmarshal([]byte("a: 1\nb: 2"), &t) +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// var t T +// yaml.Unmarshal([]byte("a: 1\nb: 2"), &t) // // See the documentation of Marshal for the format of tags and a list of // supported tag options. -// func Unmarshal(in []byte, out interface{}) (err error) { return unmarshal(in, out, false) } @@ -185,39 +183,42 @@ func unmarshal(in []byte, out interface{}, strict bool) (err error) { // // The field tag format accepted is: // -// `(...) yaml:"[][,[,]]" (...)` +// `(...) yaml:"[][,[,]]" (...)` // // The following flags are currently supported: // -// omitempty Only include the field if it's not set to the zero -// value for the type or to empty slices or maps. -// Zero valued structs will be omitted if all their public -// fields are zero, unless they implement an IsZero -// method (see the IsZeroer interface type), in which -// case the field will be excluded if IsZero returns true. +// omitempty Only include the field if it's not set to the zero +// value for the type or to empty slices or maps. +// Zero valued structs will be omitted if all their public +// fields are zero, unless they implement an IsZero +// method (see the IsZeroer interface type), in which +// case the field will be excluded if IsZero returns true. // -// flow Marshal using a flow style (useful for structs, -// sequences and maps). +// flow Marshal using a flow style (useful for structs, +// sequences and maps). // -// inline Inline the field, which must be a struct or a map, -// causing all of its fields or keys to be processed as if -// they were part of the outer struct. For maps, keys must -// not conflict with the yaml keys of other struct fields. +// inline Inline the field, which must be a struct or a map, +// causing all of its fields or keys to be processed as if +// they were part of the outer struct. For maps, keys must +// not conflict with the yaml keys of other struct fields. // // In addition, if the key is "-", the field is ignored. // // For example: // -// type T struct { -// F int `yaml:"a,omitempty"` -// B int -// } -// yaml.Marshal(&T{B: 2}) // Returns "b: 2\n" -// yaml.Marshal(&T{F: 1}} // Returns "a: 1\nb: 0\n" -// +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// yaml.Marshal(&T{B: 2}) // Returns "b: 2\n" +// yaml.Marshal(&T{F: 1}} // Returns "a: 1\nb: 0\n" func Marshal(in interface{}) (out []byte, err error) { + return MarshalWithOpts(in, nil) +} + +func MarshalWithOpts(in interface{}, arg any, opts ...MarshalOpt) (out []byte, err error) { defer handleErr(&err) - e := newEncoder() + e := newEncoder(arg, opts...) defer e.destroy() e.marshalDoc("", reflect.ValueOf(in)) e.finish() @@ -225,6 +226,12 @@ func Marshal(in interface{}) (out []byte, err error) { return } +type MarshalOpt int + +const ( + OriginalCase MarshalOpt = 1 +) + // An Encoder writes YAML values to an output stream. type Encoder struct { encoder *encoder @@ -234,8 +241,12 @@ type Encoder struct { // The Encoder should be closed after use to flush all data // to w. func NewEncoder(w io.Writer) *Encoder { + return NewEncoderWithOpts(w, nil) +} + +func NewEncoderWithOpts(w io.Writer, arg any, opts ...MarshalOpt) *Encoder { return &Encoder{ - encoder: newEncoderWithWriter(w), + encoder: newEncoderWithWriter(w, arg, opts...), } } @@ -257,8 +268,12 @@ func (e *Encoder) Encode(v interface{}) (err error) { // See the documentation for Marshal for details about the // conversion of Go values into YAML. func (n *Node) Encode(v interface{}) (err error) { + return n.EncodeWithOpts(v, nil) +} + +func (n *Node) EncodeWithOpts(v interface{}, arg any, opts ...MarshalOpt) (err error) { defer handleErr(&err) - e := newEncoder() + e := newEncoder(arg, opts...) defer e.destroy() e.marshalDoc("", reflect.ValueOf(v)) e.finish() @@ -358,22 +373,21 @@ const ( // // For example: // -// var person struct { -// Name string -// Address yaml.Node -// } -// err := yaml.Unmarshal(data, &person) -// -// Or by itself: +// var person struct { +// Name string +// Address yaml.Node +// } +// err := yaml.Unmarshal(data, &person) // -// var person Node -// err := yaml.Unmarshal(data, &person) +// Or by itself: // +// var person Node +// err := yaml.Unmarshal(data, &person) type Node struct { // Kind defines whether the node is a document, a mapping, a sequence, // a scalar value, or an alias to another node. The specific data type of // scalar nodes may be obtained via the ShortTag and LongTag methods. - Kind Kind + Kind Kind // Style allows customizing the apperance of the node in the tree. Style Style @@ -421,7 +435,6 @@ func (n *Node) IsZero() bool { n.HeadComment == "" && n.LineComment == "" && n.FootComment == "" && n.Line == 0 && n.Column == 0 } - // LongTag returns the long form of the tag that indicates the data type for // the node. If the Tag field isn't explicitly defined, one will be computed // based on the node properties. @@ -524,7 +537,7 @@ func init() { unmarshalerType = reflect.ValueOf(&v).Elem().Type() } -func getStructInfo(st reflect.Type) (*structInfo, error) { +func getStructInfo(st reflect.Type, originalCase bool) (*structInfo, error) { fieldMapMutex.RLock() sinfo, found := structMap[st] fieldMapMutex.RUnlock() @@ -592,7 +605,7 @@ func getStructInfo(st reflect.Type) (*structInfo, error) { if reflect.PtrTo(ftype).Implements(unmarshalerType) { inlineUnmarshalers = append(inlineUnmarshalers, []int{i}) } else { - sinfo, err := getStructInfo(ftype) + sinfo, err := getStructInfo(ftype, originalCase) if err != nil { return nil, err } @@ -623,7 +636,11 @@ func getStructInfo(st reflect.Type) (*structInfo, error) { if tag != "" { info.Key = tag } else { - info.Key = strings.ToLower(field.Name) + if originalCase { + info.Key = field.Name + } else { + info.Key = strings.ToLower(field.Name) + } } if _, found = fieldsMap[info.Key]; found { From 968d7bfa2b0f60dee5c79218804f44f645a070be Mon Sep 17 00:00:00 2001 From: "A. Athan" <24279435+aathan@users.noreply.github.com> Date: Wed, 9 Aug 2023 04:19:35 +0000 Subject: [PATCH 3/5] fix isZero for pointer receivers --- yaml.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/yaml.go b/yaml.go index f0bedf3d..bd2c9845 100644 --- a/yaml.go +++ b/yaml.go @@ -654,7 +654,22 @@ type IsZeroer interface { func isZero(v reflect.Value) bool { kind := v.Kind() - if z, ok := v.Interface().(IsZeroer); ok { + + var iface any + if kind != reflect.Ptr && kind != reflect.Interface { + if v.CanAddr() { + iface = v.Addr().Interface() + } else { + // uh, not sure about this but look at this project's original Node: case... + n := reflect.New(v.Type()).Elem() + n.Set(v) + iface = n.Addr().Interface() + } + } else { + iface = v.Interface() + } + + if z, ok := iface.(IsZeroer); ok { if (kind == reflect.Ptr || kind == reflect.Interface) && v.IsNil() { return true } From 0ec620964088e64c5712a8d13c9eb646ffdadfa0 Mon Sep 17 00:00:00 2001 From: "A. Athan" <24279435+aathan@users.noreply.github.com> Date: Wed, 9 Aug 2023 05:16:24 +0000 Subject: [PATCH 4/5] struct_opter --- encode.go | 12 +++++++++--- yaml.go | 5 +++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/encode.go b/encode.go index 4f230460..b18c3410 100644 --- a/encode.go +++ b/encode.go @@ -177,6 +177,11 @@ func (e *encoder) marshal(tag string, in reflect.Value) { return } + omitAllEmptyFields := false + if opter, ok := iface.(StructEncoderOpter); ok { + omitAllEmptyFields = opter.OmitAllEmptyFieldsOpt() + } + switch in.Kind() { case reflect.Interface: e.marshal(tag, in.Elem()) @@ -185,7 +190,7 @@ func (e *encoder) marshal(tag string, in reflect.Value) { case reflect.Ptr: e.marshal(tag, in.Elem()) case reflect.Struct: - e.structv(tag, in) + e.structv(tag, in, omitAllEmptyFields) case reflect.Slice, reflect.Array: e.slicev(tag, in) case reflect.String: @@ -231,8 +236,9 @@ func (e *encoder) fieldByIndex(v reflect.Value, index []int) (field reflect.Valu return v } -func (e *encoder) structv(tag string, in reflect.Value) { +func (e *encoder) structv(tag string, in reflect.Value, omitAllEmptyFields bool) { sinfo, err := getStructInfo(in.Type(), e.originalCase) + if err != nil { panic(err) } @@ -247,7 +253,7 @@ func (e *encoder) structv(tag string, in reflect.Value) { continue } } - if info.OmitEmpty && isZero(value) { + if (omitAllEmptyFields || info.OmitEmpty) && isZero(value) { continue } e.marshal("", reflect.ValueOf(info.Key)) diff --git a/yaml.go b/yaml.go index 2c392adf..4f21f53d 100644 --- a/yaml.go +++ b/yaml.go @@ -666,6 +666,11 @@ func getStructInfo(st reflect.Type, originalCase bool) (*structInfo, error) { return sinfo, nil } +// StructOpter +type StructEncoderOpter interface { + OmitAllEmptyFieldsOpt() bool +} + // IsZeroer is used to check whether an object is zero to // determine whether it should be omitted when marshaling // with the omitempty flag. One notable implementation From c585ce6e3f7735fd2fcb6b9cd00bb6b30bacba8c Mon Sep 17 00:00:00 2001 From: "A. Athan" <24279435+aathan@users.noreply.github.com> Date: Wed, 9 Aug 2023 07:00:21 +0000 Subject: [PATCH 5/5] pass tag to isZero --- encode.go | 2 +- yaml.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/encode.go b/encode.go index b18c3410..387020b9 100644 --- a/encode.go +++ b/encode.go @@ -253,7 +253,7 @@ func (e *encoder) structv(tag string, in reflect.Value, omitAllEmptyFields bool) continue } } - if (omitAllEmptyFields || info.OmitEmpty) && isZero(value) { + if (omitAllEmptyFields || info.OmitEmpty) && isZero(info.Key, value) { continue } e.marshal("", reflect.ValueOf(info.Key)) diff --git a/yaml.go b/yaml.go index 4f21f53d..6233a4b8 100644 --- a/yaml.go +++ b/yaml.go @@ -679,7 +679,7 @@ type IsZeroer interface { IsZero() bool } -func isZero(v reflect.Value) bool { +func isZero(tag string, v reflect.Value) bool { kind := v.Kind() var iface any @@ -725,7 +725,7 @@ func isZero(v reflect.Value) bool { if vt.Field(i).PkgPath != "" { continue // Private field } - if !isZero(v.Field(i)) { + if !isZero(tag+"."+vt.Field(i).Name, v.Field(i)) { return false } }