diff --git a/changelog/current.md b/changelog/current.md index 185aef5e..632052ee 100644 --- a/changelog/current.md +++ b/changelog/current.md @@ -1,13 +1,36 @@ -**All the changes described come from a single PR: [#PR414](https://github.com/biojppm/rapidyaml/pull/414).** +Most of the changes are from the giant Parser refactor described below. Before getting to that, a couple of other minor changes first. +### Fixes + +- [#PR431](https://github.com/biojppm/rapidyaml/pull/431) - Emitter: prevent stack overflows when emitting malicious trees by providing a max tree depth for the emit visitor. This was done by adding an `EmitOptions` structure as an argument both to the emitter and to the emit functions, which is then forwarded to the emitter. This `EmitOptions` structure has a max tree depth setting with a default value of 64. +- [#PR431](https://github.com/biojppm/rapidyaml/pull/431) - Fix `_RYML_CB_ALLOC()` using `(T)` in parenthesis, making the macro unusable. + + +### New features + +- [#PR431](https://github.com/biojppm/rapidyaml/pull/431) - append-emitting to existing containers in the `emitrs_` functions, suggested in [#345](https://github.com/biojppm/rapidyaml/issues/345). This was achieved by adding a `bool append=false` as the last parameter of these functions. +- [#PR431](https://github.com/biojppm/rapidyaml/pull/431) - add depth query methods: + ```cpp + Tree::depth_asc(id_type) const; // O(log(num_tree_nodes)) get the depth of a node ascending (ie, from root to node) + Tree::depth_desc(id_type) const; // O(num_tree_nodes) get the depth of a node descending (ie, from node to deep-most leaf node) + ConstNodeRef::depth_asc() const; // likewise + ConstNodeRef::depth_desc() const; + NodeRef::depth_asc() const; + NodeRef::depth_desc() const; + ``` + + +------ +All other changes come from [#PR414](https://github.com/biojppm/rapidyaml/pull/414). + ### Parser refactor -The parser was completely refactored ([#PR414](https://github.com/biojppm/rapidyaml/pull/414)). This was a large and hard job carried out over several months, and the result is: +The parser was completely refactored ([#PR414](https://github.com/biojppm/rapidyaml/pull/414)). This was a large and hard job carried out over several months, but it brings important improvements. -- A new event-based parser engine is now in place, enabling the improvements described below. This engine uses a templated event handler, where each event is a function call, which spares branches on the event handler. The parsing code was fully rewritten, and is now much more simple (albeit longer), and much easier to work with and fix. +- The new parser is an event-based parser, based on an event dispatcher engine. This engine is templated on event handler, where each event is a function call, which spares branches on the event handler. The parsing code was fully rewritten, and is now much more simple (albeit longer), and much easier to work with and fix. - YAML standard-conformance was improved significantly. Along with many smaller fixes and additions, (too many to list here), the main changes are the following: - - The parser engine can now successfully parse container keys, emitting all the events in the correct , **but** as before, the ryml tree cannot accomodate these (and this constraint is no longer enforced by the parser, but instead by `EventHandlerTree`). For an example of a handler which can accomodate key containers, see the one which is used for the test suite at `test/test_suite/test_suite_event_handler.hpp` + - The parser engine can now successfully parse container keys, emitting all the events in correctly, **but** as before, the ryml tree cannot accomodate these (and this constraint is no longer enforced by the parser, but instead by `EventHandlerTree`). For an example of a handler which can accomodate key containers, see the one which is used for the test suite at `test/test_suite/test_suite_event_handler.hpp` - Anchor keys can now be terminated with colon (eg, `&anchor: key: val`), as dictated by the standard. - The parser engine can now be used to create native trees in other programming languages, or in cases where the user *must* have container keys. - Performance of both parsing and emitting improved significantly; see some figures below. diff --git a/src/c4/yml/common.hpp b/src/c4/yml/common.hpp index cdeac049..6b574710 100644 --- a/src/c4/yml/common.hpp +++ b/src/c4/yml/common.hpp @@ -468,7 +468,7 @@ do \ } \ } while(0) #define _RYML_CB_ALLOC_HINT(cb, T, num, hint) (T*) (cb).m_allocate((num) * sizeof(T), (hint), (cb).m_user_data) -#define _RYML_CB_ALLOC(cb, T, num) _RYML_CB_ALLOC_HINT((cb), (T), (num), nullptr) +#define _RYML_CB_ALLOC(cb, T, num) _RYML_CB_ALLOC_HINT((cb), T, (num), nullptr) #define _RYML_CB_FREE(cb, buf, T, num) \ do { \ (cb).m_free((buf), (num) * sizeof(T), (cb).m_user_data); \ diff --git a/src/c4/yml/emit.def.hpp b/src/c4/yml/emit.def.hpp index 7f9b0594..efbaa351 100644 --- a/src/c4/yml/emit.def.hpp +++ b/src/c4/yml/emit.def.hpp @@ -29,28 +29,13 @@ substr Emitter::emit_as(EmitType_e type, Tree const& t, id_type id, bool if(type == EMIT_YAML) _emit_yaml(id); else if(type == EMIT_JSON) - _do_visit_json(id); + _do_visit_json(id, 0); else _RYML_CB_ERR(m_tree->callbacks(), "unknown emit type"); m_tree = nullptr; return this->Writer::_get(error_on_excess); } -template -substr Emitter::emit_as(EmitType_e type, Tree const& t, bool error_on_excess) -{ - if(t.empty()) - return {}; - return this->emit_as(type, t, t.root_id(), error_on_excess); -} - -template -substr Emitter::emit_as(EmitType_e type, ConstNodeRef const& n, bool error_on_excess) -{ - _RYML_CB_CHECK(n.tree()->callbacks(), n.readable()); - return this->emit_as(type, *n.tree(), n.id(), error_on_excess); -} - //----------------------------------------------------------------------------- @@ -82,7 +67,7 @@ void Emitter::_emit_yaml(id_type id) this->Writer::_do_write(":\n"); ++ilevel; } - _do_visit_block_container(id, ilevel, ilevel); + _do_visit_block_container(id, 0, ilevel, ilevel); return; } } @@ -256,13 +241,15 @@ void Emitter::_write_doc(id_type id) } template -void Emitter::_do_visit_flow_sl(id_type node, id_type ilevel) +void Emitter::_do_visit_flow_sl(id_type node, id_type depth, id_type ilevel) { const bool prev_flow = m_flow; m_flow = true; RYML_ASSERT(!m_tree->is_stream(node)); RYML_ASSERT(m_tree->is_container(node) || m_tree->is_doc(node)); RYML_ASSERT(m_tree->is_root(node) || (m_tree->parent_is_map(node) || m_tree->parent_is_seq(node))); + if(C4_UNLIKELY(depth > m_opts.max_depth())) + _RYML_CB_ERR(m_tree->callbacks(), "max depth exceeded"); if(m_tree->is_doc(node)) { @@ -345,7 +332,7 @@ void Emitter::_do_visit_flow_sl(id_type node, id_type ilevel) else { // with single-line flow, we can never go back to block - _do_visit_flow_sl(child, ilevel + 1); + _do_visit_flow_sl(child, depth + 1, ilevel + 1); } } @@ -363,19 +350,25 @@ void Emitter::_do_visit_flow_sl(id_type node, id_type ilevel) C4_SUPPRESS_WARNING_MSVC_WITH_PUSH(4702) // unreachable error, triggered by flow_ml not implemented template -void Emitter::_do_visit_flow_ml(id_type id, id_type ilevel, id_type do_indent) +void Emitter::_do_visit_flow_ml(id_type id, id_type depth, id_type ilevel, id_type do_indent) { C4_UNUSED(id); + C4_UNUSED(depth); C4_UNUSED(ilevel); C4_UNUSED(do_indent); + c4::yml::error("not implemented"); + #ifdef THIS_IS_A_WORK_IN_PROGRESS + if(C4_UNLIKELY(depth > m_opts.max_depth())) + _RYML_CB_ERR(m_tree->callbacks(), "max depth exceeded"); const bool prev_flow = m_flow; m_flow = true; - c4::yml::error("not implemented"); + // do it... m_flow = prev_flow; + #endif } template -void Emitter::_do_visit_block_container(id_type node, id_type level, bool do_indent) +void Emitter::_do_visit_block_container(id_type node, id_type depth, id_type level, bool do_indent) { if(m_tree->is_seq(node)) { @@ -397,19 +390,19 @@ void Emitter::_do_visit_block_container(id_type node, id_type level, boo { _indent(level, do_indent); this->Writer::_do_write("- "); - _do_visit_flow_sl(child, 0u); + _do_visit_flow_sl(child, depth+1, 0u); this->Writer::_do_write('\n'); } else if(ty.is_flow_ml()) { _indent(level, do_indent); this->Writer::_do_write("- "); - _do_visit_flow_ml(child, 0u, do_indent); + _do_visit_flow_ml(child, depth+1, 0u, do_indent); this->Writer::_do_write('\n'); } else { - _do_visit_block(child, level, do_indent); // same level + _do_visit_block(child, depth+1, level, do_indent); // same indentation level } } do_indent = true; @@ -436,18 +429,18 @@ void Emitter::_do_visit_block_container(id_type node, id_type level, boo if(ty.is_flow_sl()) { _indent(level, do_indent); - _do_visit_flow_sl(ich, 0u); + _do_visit_flow_sl(ich, depth+1, 0u); this->Writer::_do_write('\n'); } else if(ty.is_flow_ml()) { _indent(level, do_indent); - _do_visit_flow_ml(ich, 0u); + _do_visit_flow_ml(ich, depth+1, 0u); this->Writer::_do_write('\n'); } else { - _do_visit_block(ich, level, do_indent); // same level! + _do_visit_block(ich, depth+1, level, do_indent); // same level! } } // keyval vs container do_indent = true; @@ -456,11 +449,13 @@ void Emitter::_do_visit_block_container(id_type node, id_type level, boo } template -void Emitter::_do_visit_block(id_type node, id_type ilevel, id_type do_indent) +void Emitter::_do_visit_block(id_type node, id_type depth, id_type ilevel, id_type do_indent) { RYML_ASSERT(!m_tree->is_stream(node)); RYML_ASSERT(m_tree->is_container(node) || m_tree->is_doc(node)); RYML_ASSERT(m_tree->is_root(node) || (m_tree->parent_is_map(node) || m_tree->parent_is_seq(node))); + if(C4_UNLIKELY(depth > m_opts.max_depth())) + _RYML_CB_ERR(m_tree->callbacks(), "max depth exceeded"); if(m_tree->is_doc(node)) { _write_doc(node); @@ -537,16 +532,18 @@ void Emitter::_do_visit_block(id_type node, id_type ilevel, id_type do_i if(m_tree->is_root(node) || m_tree->is_doc(node)) next_level = ilevel; // do not indent at top level - _do_visit_block_container(node, next_level, do_indent); + _do_visit_block_container(node, depth, next_level, do_indent); } C4_SUPPRESS_WARNING_MSVC_POP template -void Emitter::_do_visit_json(id_type id) +void Emitter::_do_visit_json(id_type id, id_type depth) { _RYML_CB_CHECK(m_tree->callbacks(), !m_tree->is_stream(id)); // JSON does not have streams + if(C4_UNLIKELY(depth > m_opts.max_depth())) + _RYML_CB_ERR(m_tree->callbacks(), "max depth exceeded"); if(m_tree->is_keyval(id)) { _writek_json(id); @@ -574,7 +571,7 @@ void Emitter::_do_visit_json(id_type id) { if(ich != m_tree->first_child(id)) this->Writer::_do_write(','); - _do_visit_json(ich); + _do_visit_json(ich, depth+1); } if(m_tree->is_seq(id)) diff --git a/src/c4/yml/emit.hpp b/src/c4/yml/emit.hpp index 80fbc6ec..8c788b61 100644 --- a/src/c4/yml/emit.hpp +++ b/src/c4/yml/emit.hpp @@ -51,6 +51,15 @@ using EmitterOStream = Emitter>; using EmitterFile = Emitter; using EmitterBuf = Emitter; +namespace detail { +inline bool is_set_(ConstNodeRef n) { return n.tree() && (n.id() != NONE); } +} + + +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- + /** Specifies the type of content to emit */ typedef enum { EMIT_YAML = 0, ///< emit YAML @@ -58,6 +67,29 @@ typedef enum { } EmitType_e; +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- +//----------------------------------------------------------------------------- + +/** A lightweight object containing options to be used when emitting. */ +struct EmitOptions +{ +public: + /** @name max depth for the emitted tree + * + * This makes the emitter fail when emitting trees exceeding the + * max_depth. + * + * @{ */ + C4_ALWAYS_INLINE id_type max_depth() const noexcept { return m_max_depth; } + EmitOptions& max_depth(id_type d) noexcept { m_max_depth = d; return *this; } + static constexpr const id_type max_depth_default = 64; + /** @} */ +private: + id_type m_max_depth{max_depth_default}; +}; + + //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- @@ -69,12 +101,21 @@ class Emitter : public Writer { public: - /** Construct the emitter and its internal Writer state. Every - * parameter is forwarded to the constructor of the writer. */ + /** Construct the emitter and its internal Writer state, using default emit options. + * @param args arguments to be forwarded to the constructor of the writer. + * */ template - Emitter(Args &&...args) : Writer(std::forward(args)...), m_tree() {} - /** emit! + Emitter(Args &&...args) : Writer(std::forward(args)...), m_tree(), m_opts(), m_flow(false) {} + /** Construct the emitter and its internal Writer state. + * + * @param opts EmitOptions + * @param args arguments to be forwarded to the constructor of the writer. + * */ + template + Emitter(EmitOptions const& opts, Args &&...args) : Writer(std::forward(args)...), m_tree(), m_opts(opts), m_flow(false) {} + + /** emit! * * When writing to a buffer, returns a substr of the emitted YAML. * If the given buffer has insufficient space, the returned substr @@ -93,21 +134,45 @@ class Emitter : public Writer * */ substr emit_as(EmitType_e type, Tree const& t, id_type id, bool error_on_excess); /** emit starting at the root node */ - substr emit_as(EmitType_e type, Tree const& t, bool error_on_excess=true); - /** emit the given node */ - substr emit_as(EmitType_e type, ConstNodeRef const& n, bool error_on_excess=true); + substr emit_as(EmitType_e type, Tree const& t, bool error_on_excess=true) + { + if(t.empty()) + return {}; + return this->emit_as(type, t, t.root_id(), error_on_excess); + } + /** emit starting at the given node */ + substr emit_as(EmitType_e type, ConstNodeRef const& n, bool error_on_excess=true) + { + if(!detail::is_set_(n)) + return {}; + _RYML_CB_CHECK(n.tree()->callbacks(), n.readable()); + return this->emit_as(type, *n.tree(), n.id(), error_on_excess); + } + +public: + + /** get the emit options for this object */ + EmitOptions const& options() const noexcept { return m_opts; } + + /** set the max depth for emitted trees (to prevent a stack overflow) */ + void max_depth(id_type max_depth) noexcept { m_opts.max_depth(max_depth); } + /** get the max depth for emitted trees (to prevent a stack overflow) */ + id_type max_depth() const noexcept { return m_opts.max_depth(); } private: Tree const* C4_RESTRICT m_tree; + EmitOptions m_opts; bool m_flow; +private: + void _emit_yaml(id_type id); - void _do_visit_flow_sl(id_type id, id_type ilevel=0); - void _do_visit_flow_ml(id_type id, id_type ilevel=0, id_type do_indent=1); - void _do_visit_block(id_type id, id_type ilevel=0, id_type do_indent=1); - void _do_visit_block_container(id_type id, id_type next_level, bool do_indent); - void _do_visit_json(id_type id); + void _do_visit_flow_sl(id_type id, id_type depth, id_type ilevel=0); + void _do_visit_flow_ml(id_type id, id_type depth, id_type ilevel=0, id_type do_indent=1); + void _do_visit_block(id_type id, id_type depth, id_type ilevel=0, id_type do_indent=1); + void _do_visit_block_container(id_type id, id_type depth, id_type next_level, bool do_indent); + void _do_visit_json(id_type id, id_type depth); private: @@ -168,15 +233,29 @@ class Emitter : public Writer */ -/** emit YAML to the given file. A null file defaults to stdout. - * Return the number of bytes written. */ +// emit from tree and node id ----------------------- + +/** (1) emit YAML to the given file, starting at the given node. A null + * file defaults to stdout. Return the number of bytes written. */ +inline size_t emit_yaml(Tree const& t, id_type id, EmitOptions const& opts, FILE *f) +{ + EmitterFile em(opts, f); + return em.emit_as(EMIT_YAML, t, id, /*error_on_excess*/true).len; +} +/** (2) like (1), but use default emit options */ inline size_t emit_yaml(Tree const& t, id_type id, FILE *f) { EmitterFile em(f); return em.emit_as(EMIT_YAML, t, id, /*error_on_excess*/true).len; } -/** emit JSON to the given file. A null file defaults to stdout. - * Return the number of bytes written. */ +/** (1) emit JSON to the given file, starting at the given node. A null + * file defaults to stdout. Return the number of bytes written. */ +inline size_t emit_json(Tree const& t, id_type id, EmitOptions const& opts, FILE *f) +{ + EmitterFile em(opts, f); + return em.emit_as(EMIT_JSON, t, id, /*error_on_excess*/true).len; +} +/** (2) like (1), but use default emit options */ inline size_t emit_json(Tree const& t, id_type id, FILE *f) { EmitterFile em(f); @@ -184,17 +263,29 @@ inline size_t emit_json(Tree const& t, id_type id, FILE *f) } -/** emit YAML to the given file. A null file defaults to stdout. - * Return the number of bytes written. - * @overload */ +// emit from root ------------------------- + +/** (1) emit YAML to the given file, starting at the root node. A null file defaults to stdout. + * Return the number of bytes written. */ +inline size_t emit_yaml(Tree const& t, EmitOptions const& opts, FILE *f=nullptr) +{ + EmitterFile em(opts, f); + return em.emit_as(EMIT_YAML, t, /*error_on_excess*/true).len; +} +/** (2) like (1), but use default emit options */ inline size_t emit_yaml(Tree const& t, FILE *f=nullptr) { EmitterFile em(f); return em.emit_as(EMIT_YAML, t, /*error_on_excess*/true).len; } -/** emit JSON to the given file. A null file defaults to stdout. - * Return the number of bytes written. - * @overload */ +/** (1) emit JSON to the given file. A null file defaults to stdout. + * Return the number of bytes written. */ +inline size_t emit_json(Tree const& t, EmitOptions const& opts, FILE *f=nullptr) +{ + EmitterFile em(opts, f); + return em.emit_as(EMIT_JSON, t, /*error_on_excess*/true).len; +} +/** (2) like (1), but use default emit options */ inline size_t emit_json(Tree const& t, FILE *f=nullptr) { EmitterFile em(f); @@ -202,19 +293,39 @@ inline size_t emit_json(Tree const& t, FILE *f=nullptr) } -/** emit YAML to the given file. A null file defaults to stdout. - * Return the number of bytes written. - * @overload */ +// emit from ConstNodeRef ------------------------ + +/** (1) emit YAML to the given file. A null file defaults to stdout. + * Return the number of bytes written. */ +inline size_t emit_yaml(ConstNodeRef const& r, EmitOptions const& opts, FILE *f=nullptr) +{ + if(!detail::is_set_(r)) + return {}; + EmitterFile em(opts, f); + return em.emit_as(EMIT_YAML, r, /*error_on_excess*/true).len; +} +/** (2) like (1), but use default emit options */ inline size_t emit_yaml(ConstNodeRef const& r, FILE *f=nullptr) { + if(!detail::is_set_(r)) + return {}; EmitterFile em(f); return em.emit_as(EMIT_YAML, r, /*error_on_excess*/true).len; } -/** emit JSON to the given file. A null file defaults to stdout. - * Return the number of bytes written. - * @overload */ +/** (1) emit JSON to the given file. A null file defaults to stdout. + * Return the number of bytes written. */ +inline size_t emit_json(ConstNodeRef const& r, EmitOptions const& opts, FILE *f=nullptr) +{ + if(!detail::is_set_(r)) + return {}; + EmitterFile em(opts, f); + return em.emit_as(EMIT_JSON, r, /*error_on_excess*/true).len; +} +/** (2) like (1), but use default emit options */ inline size_t emit_json(ConstNodeRef const& r, FILE *f=nullptr) { + if(!detail::is_set_(r)) + return {}; EmitterFile em(f); return em.emit_as(EMIT_JSON, r, /*error_on_excess*/true).len; } @@ -229,24 +340,6 @@ inline size_t emit_json(ConstNodeRef const& r, FILE *f=nullptr) * @{ */ -/** mark a tree or node to be emitted as json when using @ref operator<< . For example: - * - * ```cpp - * Tree t = parse_in_arena("{foo: bar}"); - * std::cout << t; // emits YAML - * std::cout << as_json(t); // emits JSON - * ``` - * - * @see @ref operator<< */ -struct as_json -{ - Tree const* tree; - size_t node; - as_json(Tree const& t) : tree(&t), node(t.empty() ? NONE : t.root_id()) {} - as_json(Tree const& t, size_t id) : tree(&t), node(id) {} - as_json(ConstNodeRef const& n) : tree(n.tree()), node(n.id()) {} -}; - /** emit YAML to an STL-like ostream */ template inline OStream& operator<< (OStream& s, Tree const& t) @@ -261,20 +354,77 @@ inline OStream& operator<< (OStream& s, Tree const& t) template inline OStream& operator<< (OStream& s, ConstNodeRef const& n) { + if(!detail::is_set_(n)) + return s; EmitterOStream em(s); em.emit_as(EMIT_YAML, n); return s; } +/** mark a tree or node to be emitted as yaml when using @ref + * operator<<, with options. For example: + * + * ```cpp + * Tree t = parse_in_arena("{foo: bar}"); + * std::cout << t; // emits YAML + * std::cout << as_yaml(t); // emits YAML, same as above + * std::cout << as_yaml(t, EmitOptions().max_depth(10)); // emits JSON with a max tree depth + * ``` + * + * @see @ref operator<< */ +struct as_json +{ + Tree const* tree; + size_t node; + EmitOptions options; + as_json(Tree const& t, EmitOptions const& opts={}) : tree(&t), node(t.empty() ? NONE : t.root_id()), options(opts) {} + as_json(Tree const& t, size_t id, EmitOptions const& opts={}) : tree(&t), node(id), options(opts) {} + as_json(ConstNodeRef const& n, EmitOptions const& opts={}) : tree(n.tree()), node(n.id()), options(opts) {} +}; + +/** mark a tree or node to be emitted as yaml when using @ref + * operator<< . For example: + * + * ```cpp + * Tree t = parse_in_arena("{foo: bar}"); + * std::cout << t; // emits YAML + * std::cout << as_json(t); // emits JSON + * std::cout << as_json(t, EmitOptions().max_depth(10)); // emits JSON with a max tree depth + * ``` + * + * @see @ref operator<< */ +struct as_yaml +{ + Tree const* tree; + size_t node; + EmitOptions options; + as_yaml(Tree const& t, EmitOptions const& opts={}) : tree(&t), node(t.empty() ? NONE : t.root_id()), options(opts) {} + as_yaml(Tree const& t, size_t id, EmitOptions const& opts={}) : tree(&t), node(id), options(opts) {} + as_yaml(ConstNodeRef const& n, EmitOptions const& opts={}) : tree(n.tree()), node(n.id()), options(opts) {} +}; + /** emit json to an STL-like stream */ template inline OStream& operator<< (OStream& s, as_json const& j) { - EmitterOStream em(s); + if(!j.tree || j.node == NONE) + return s; + EmitterOStream em(j.options, s); em.emit_as(EMIT_JSON, *j.tree, j.node, true); return s; } +/** emit yaml to an STL-like stream */ +template +inline OStream& operator<< (OStream& s, as_yaml const& y) +{ + if(!y.tree || y.node == NONE) + return s; + EmitterOStream em(y.options, s); + em.emit_as(EMIT_YAML, *y.tree, y.node, true); + return s; +} + /** @} */ @@ -285,29 +435,43 @@ inline OStream& operator<< (OStream& s, as_json const& j) * @{ */ -/** emit YAML to the given buffer. Return a substr trimmed to the emitted YAML. +// emit from tree and node id ----------------------- + +/** (1) emit YAML to the given buffer. Return a substr trimmed to the emitted YAML. * @param t the tree to emit. * @param id the node where to start emitting. + * @param opts emit options. * @param buf the output buffer. * @param error_on_excess Raise an error if the space in the buffer is insufficient. - * @return a substr trimmed to the result. If the buffer is - * insufficient (and error_on_excess is false), the pointer of the - * result will be set to null. - * @overload */ + * @return a substr trimmed to the result in the output buffer. If the buffer is + * insufficient (when error_on_excess is false), the string pointer of the + * result will be set to null, and the length will report the required buffer size. */ +inline substr emit_yaml(Tree const& t, id_type id, EmitOptions const& opts, substr buf, bool error_on_excess=true) +{ + EmitterBuf em(opts, buf); + return em.emit_as(EMIT_YAML, t, id, error_on_excess); +} +/** (2) like (1), but use default emit options */ inline substr emit_yaml(Tree const& t, id_type id, substr buf, bool error_on_excess=true) { EmitterBuf em(buf); return em.emit_as(EMIT_YAML, t, id, error_on_excess); } -/** emit JSON to the given buffer. Return a substr trimmed to the emitted JSON. +/** (1) emit JSON to the given buffer. Return a substr trimmed to the emitted JSON. * @param t the tree to emit. * @param id the node where to start emitting. + * @param opts emit options. * @param buf the output buffer. * @param error_on_excess Raise an error if the space in the buffer is insufficient. - * @return a substr trimmed to the result. If the buffer is - * insufficient (and error_on_excess is false), the pointer of the - * result will be set to null. - * @overload */ + * @return a substr trimmed to the result in the output buffer. If the buffer is + * insufficient (when error_on_excess is false), the string pointer of the + * result will be set to null, and the length will report the required buffer size. */ +inline substr emit_json(Tree const& t, id_type id, EmitOptions const& opts, substr buf, bool error_on_excess=true) +{ + EmitterBuf em(opts, buf); + return em.emit_as(EMIT_JSON, t, id, error_on_excess); +} +/** (2) like (1), but use default emit options */ inline substr emit_json(Tree const& t, id_type id, substr buf, bool error_on_excess=true) { EmitterBuf em(buf); @@ -315,27 +479,39 @@ inline substr emit_json(Tree const& t, id_type id, substr buf, bool error_on_exc } -/** emit YAML to the given buffer. Return a substr trimmed to the emitted YAML. +// emit from root ------------------------- + +/** (1) emit YAML to the given buffer. Return a substr trimmed to the emitted YAML. * @param t the tree; will be emitted from the root node. - * @param error_on_excess Raise an error if the space in the buffer is insufficient. * @param buf the output buffer. - * @return a substr trimmed to the result. If the buffer is - * insufficient (and error_on_excess is false), the pointer of the - * result will be set to null. - * @overload */ + * @param error_on_excess Raise an error if the space in the buffer is insufficient. + * @return a substr trimmed to the result in the output buffer. If the buffer is + * insufficient (when error_on_excess is false), the string pointer of the + * result will be set to null, and the length will report the required buffer size. */ +inline substr emit_yaml(Tree const& t, EmitOptions const& opts, substr buf, bool error_on_excess=true) +{ + EmitterBuf em(opts, buf); + return em.emit_as(EMIT_YAML, t, error_on_excess); +} +/** (2) like (1), but use default emit options */ inline substr emit_yaml(Tree const& t, substr buf, bool error_on_excess=true) { EmitterBuf em(buf); return em.emit_as(EMIT_YAML, t, error_on_excess); } -/** emit JSON to the given buffer. Return a substr trimmed to the emitted JSON. +/** (1) emit JSON to the given buffer. Return a substr trimmed to the emitted JSON. * @param t the tree; will be emitted from the root node. * @param buf the output buffer. * @param error_on_excess Raise an error if the space in the buffer is insufficient. - * @return a substr trimmed to the result. If the buffer is - * insufficient (and error_on_excess is false), the pointer of the - * result will be set to null. - * @overload */ + * @return a substr trimmed to the result in the output buffer. If the buffer is + * insufficient (when error_on_excess is false), the string pointer of the + * result will be set to null, and the length will report the required buffer size. */ +inline substr emit_json(Tree const& t, EmitOptions const& opts, substr buf, bool error_on_excess=true) +{ + EmitterBuf em(opts, buf); + return em.emit_as(EMIT_JSON, t, error_on_excess); +} +/** (2) like (1), but use default emit options */ inline substr emit_json(Tree const& t, substr buf, bool error_on_excess=true) { EmitterBuf em(buf); @@ -343,31 +519,51 @@ inline substr emit_json(Tree const& t, substr buf, bool error_on_excess=true) } -/** emit YAML to the given buffer. Return a substr trimmed to the emitted YAML. +// emit from ConstNodeRef ------------------------ + +/** (1) emit YAML to the given buffer. Return a substr trimmed to the emitted YAML. * @param r the starting node. * @param buf the output buffer. + * @param opts emit options. * @param error_on_excess Raise an error if the space in the buffer is insufficient. - * @return a substr trimmed to the result. If the buffer is - * insufficient (and error_on_excess is false), the pointer of the - * result will be set to null. - * @overload - */ + * @return a substr trimmed to the result in the output buffer. If the buffer is + * insufficient (when error_on_excess is false), the string pointer of the + * result will be set to null, and the length will report the required buffer size. */ +inline substr emit_yaml(ConstNodeRef const& r, EmitOptions const& opts, substr buf, bool error_on_excess=true) +{ + if(!detail::is_set_(r)) + return {}; + EmitterBuf em(opts, buf); + return em.emit_as(EMIT_YAML, r, error_on_excess); +} +/** (2) like (1), but use default emit options */ inline substr emit_yaml(ConstNodeRef const& r, substr buf, bool error_on_excess=true) { + if(!detail::is_set_(r)) + return {}; EmitterBuf em(buf); return em.emit_as(EMIT_YAML, r, error_on_excess); } -/** emit JSON to the given buffer. Return a substr trimmed to the emitted JSON. +/** (1) emit JSON to the given buffer. Return a substr trimmed to the emitted JSON. * @param r the starting node. * @param buf the output buffer. + * @param opts emit options. * @param error_on_excess Raise an error if the space in the buffer is insufficient. - * @return a substr trimmed to the result. If the buffer is - * insufficient (and error_on_excess is false), the pointer of the - * result will be set to null. - * @overload - */ + * @return a substr trimmed to the result in the output buffer. If the buffer is + * insufficient (when error_on_excess is false), the string pointer of the + * result will be set to null, and the length will report the required buffer size. */ +inline substr emit_json(ConstNodeRef const& r, EmitOptions const& opts, substr buf, bool error_on_excess=true) +{ + if(!detail::is_set_(r)) + return {}; + EmitterBuf em(opts, buf); + return em.emit_as(EMIT_JSON, r, error_on_excess); +} +/** (2) like (1), but use default emit options */ inline substr emit_json(ConstNodeRef const& r, substr buf, bool error_on_excess=true) { + if(!detail::is_set_(r)) + return {}; EmitterBuf em(buf); return em.emit_as(EMIT_JSON, r, error_on_excess); } @@ -375,141 +571,225 @@ inline substr emit_json(ConstNodeRef const& r, substr buf, bool error_on_excess= //----------------------------------------------------------------------------- -/** emit+resize: emit YAML to the given `std::string`/`std::vector`-like - * container, resizing it as needed to fit the emitted YAML. */ +/** @defgroup doc_emit_to_container Emit to resizeable container + * + * @{ + */ + +// emit from tree and node id --------------------------- + +/** (1) emit+resize: emit YAML to the given `std::string`/`std::vector`-like + * container, resizing it as needed to fit the emitted YAML. If @p append is + * set to true, the emitted YAML is appended at the end of the container. + * + * @return a substr trimmed to the emitted YAML (excluding the initial contents, when appending) */ template -substr emitrs_yaml(Tree const& t, id_type id, CharOwningContainer * cont) +substr emitrs_yaml(Tree const& t, id_type id, EmitOptions const& opts, CharOwningContainer * cont, bool append=false) { - substr buf = to_substr(*cont); - substr ret = emit_yaml(t, id, buf, /*error_on_excess*/false); + size_t startpos = append ? cont->size() : 0u; + cont->resize(cont->capacity()); // otherwise the first emit would be certain to fail + substr buf = to_substr(*cont).sub(startpos); + substr ret = emit_yaml(t, id, opts, buf, /*error_on_excess*/false); if(ret.str == nullptr && ret.len > 0) { - cont->resize(ret.len); - buf = to_substr(*cont); - ret = emit_yaml(t, id, buf, /*error_on_excess*/true); + cont->resize(startpos + ret.len); + buf = to_substr(*cont).sub(startpos); + ret = emit_yaml(t, id, opts, buf, /*error_on_excess*/true); + } + else + { + cont->resize(startpos + ret.len); } return ret; } -/** emit+resize: emit JSON to the given `std::string`/`std::vector`-like - * container, resizing it as needed to fit the emitted JSON. */ +/** (2) like (1), but use default emit options */ +template +substr emitrs_yaml(Tree const& t, id_type id, CharOwningContainer * cont, bool append=false) +{ + return emitrs_yaml(t, id, EmitOptions{}, cont, append); +} +/** (1) emit+resize: emit JSON to the given `std::string`/`std::vector`-like + * container, resizing it as needed to fit the emitted JSON. If @p append is + * set to true, the emitted YAML is appended at the end of the container. + * + * @return a substr trimmed to the emitted JSON (excluding the initial contents, when appending) */ template -substr emitrs_json(Tree const& t, id_type id, CharOwningContainer * cont) +substr emitrs_json(Tree const& t, id_type id, EmitOptions const& opts, CharOwningContainer * cont, bool append=false) { - substr buf = to_substr(*cont); - substr ret = emit_json(t, id, buf, /*error_on_excess*/false); + const size_t startpos = append ? cont->size() : 0u; + cont->resize(cont->capacity()); // otherwise the first emit would be certain to fail + substr buf = to_substr(*cont).sub(startpos); + EmitterBuf em(opts, buf); + substr ret = emit_json(t, id, opts, buf, /*error_on_excess*/false); if(ret.str == nullptr && ret.len > 0) { - cont->resize(ret.len); - buf = to_substr(*cont); - ret = emit_json(t, id, buf, /*error_on_excess*/true); + cont->resize(startpos + ret.len); + buf = to_substr(*cont).sub(startpos); + ret = emit_json(t, id, opts, buf, /*error_on_excess*/true); + } + else + { + cont->resize(startpos + ret.len); } return ret; } +/** (2) like (1), but use default emit options */ +template +substr emitrs_json(Tree const& t, id_type id, CharOwningContainer * cont, bool append=false) +{ + return emitrs_json(t, id, EmitOptions{}, cont, append); +} -/** emit+resize: emit YAML to the given `std::string`/`std::vector`-like - * container, resizing it as needed to fit the emitted YAML. */ +/** (3) emit+resize: YAML to a newly-created `std::string`/`std::vector`-like container. */ template -CharOwningContainer emitrs_yaml(Tree const& t, id_type id) +CharOwningContainer emitrs_yaml(Tree const& t, id_type id, EmitOptions const& opts={}) { CharOwningContainer c; - emitrs_yaml(t, id, &c); + emitrs_yaml(t, id, opts, &c); return c; } -/** emit+resize: emit JSON to the given `std::string`/`std::vector`-like - * container, resizing it as needed to fit the emitted JSON. */ +/** (3) emit+resize: JSON to a newly-created `std::string`/`std::vector`-like container. */ template -CharOwningContainer emitrs_json(Tree const& t, id_type id) +CharOwningContainer emitrs_json(Tree const& t, id_type id, EmitOptions const& opts={}) { CharOwningContainer c; - emitrs_json(t, id, &c); + emitrs_json(t, id, opts, &c); return c; } -/** emit+resize: YAML to the given `std::string`/`std::vector`-like - * container, resizing it as needed to fit the emitted YAML. */ +// emit from root ------------------------- + +/** (1) emit+resize: YAML to the given `std::string`/`std::vector`-like + * container, resizing it as needed to fit the emitted YAML. + * @return a substr trimmed to the new emitted contents. */ +template +substr emitrs_yaml(Tree const& t, EmitOptions const& opts, CharOwningContainer * cont, bool append=false) +{ + if(t.empty()) + return {}; + return emitrs_yaml(t, t.root_id(), opts, cont, append); +} +/** (2) like (1), but use default emit options */ +template +substr emitrs_yaml(Tree const& t, CharOwningContainer * cont, bool append=false) +{ + if(t.empty()) + return {}; + return emitrs_yaml(t, t.root_id(), EmitOptions{}, cont, append); +} +/** (1) emit+resize: JSON to the given `std::string`/`std::vector`-like + * container, resizing it as needed to fit the emitted JSON. + * @return a substr trimmed to the new emitted contents. */ template -substr emitrs_yaml(Tree const& t, CharOwningContainer * cont) +substr emitrs_json(Tree const& t, EmitOptions const& opts, CharOwningContainer * cont, bool append=false) { if(t.empty()) return {}; - return emitrs_yaml(t, t.root_id(), cont); + return emitrs_json(t, t.root_id(), opts, cont, append); } -/** emit+resize: JSON to the given `std::string`/`std::vector`-like - * container, resizing it as needed to fit the emitted JSON. */ +/** (2) like (1), but use default emit options */ template -substr emitrs_json(Tree const& t, CharOwningContainer * cont) +substr emitrs_json(Tree const& t, CharOwningContainer * cont, bool append=false) { if(t.empty()) return {}; - return emitrs_json(t, t.root_id(), cont); + return emitrs_json(t, t.root_id(), EmitOptions{}, cont, append); } -/** emit+resize: YAML to the given `std::string`/`std::vector`-like container, - * resizing it as needed to fit the emitted YAML. */ +/** (3) emit+resize: YAML to a newly-created `std::string`/`std::vector`-like container. */ template -CharOwningContainer emitrs_yaml(Tree const& t) +CharOwningContainer emitrs_yaml(Tree const& t, EmitOptions const& opts={}) { CharOwningContainer c; if(t.empty()) return c; - emitrs_yaml(t, t.root_id(), &c); + emitrs_yaml(t, t.root_id(), opts, &c); return c; } -/** emit+resize: JSON to the given `std::string`/`std::vector`-like container, - * resizing it as needed to fit the emitted JSON. */ +/** (3) emit+resize: JSON to a newly-created `std::string`/`std::vector`-like container. */ template -CharOwningContainer emitrs_json(Tree const& t) +CharOwningContainer emitrs_json(Tree const& t, EmitOptions const& opts={}) { CharOwningContainer c; if(t.empty()) return c; - emitrs_json(t, t.root_id(), &c); + emitrs_json(t, t.root_id(), opts, &c); return c; } -/** emit+resize: YAML to the given `std::string`/`std::vector`-like container, - * resizing it as needed to fit the emitted YAML. */ +// emit from ConstNodeRef ------------------------ + + +/** (1) emit+resize: YAML to the given `std::string`/`std::vector`-like container, + * resizing it as needed to fit the emitted YAML. + * @return a substr trimmed to the new emitted contents */ +template +substr emitrs_yaml(ConstNodeRef const& n, EmitOptions const& opts, CharOwningContainer * cont, bool append=false) +{ + if(!detail::is_set_(n)) + return {}; + _RYML_CB_CHECK(n.tree()->callbacks(), n.readable()); + return emitrs_yaml(*n.tree(), n.id(), opts, cont, append); +} +/** (2) like (1), but use default emit options */ +template +substr emitrs_yaml(ConstNodeRef const& n, CharOwningContainer * cont, bool append=false) +{ + if(!detail::is_set_(n)) + return {}; + _RYML_CB_CHECK(n.tree()->callbacks(), n.readable()); + return emitrs_yaml(*n.tree(), n.id(), EmitOptions{}, cont, append); +} +/** (1) emit+resize: JSON to the given `std::string`/`std::vector`-like container, + * resizing it as needed to fit the emitted JSON. + * @return a substr trimmed to the new emitted contents */ template -substr emitrs_yaml(ConstNodeRef const& n, CharOwningContainer * cont) +substr emitrs_json(ConstNodeRef const& n, EmitOptions const& opts, CharOwningContainer * cont, bool append=false) { + if(!detail::is_set_(n)) + return {}; _RYML_CB_CHECK(n.tree()->callbacks(), n.readable()); - return emitrs_yaml(*n.tree(), n.id(), cont); + return emitrs_json(*n.tree(), n.id(), opts, cont, append); } -/** emit+resize: JSON to the given `std::string`/`std::vector`-like container, - * resizing it as needed to fit the emitted JSON. */ +/** (2) like (1), but use default emit options */ template -substr emitrs_json(ConstNodeRef const& n, CharOwningContainer * cont) +substr emitrs_json(ConstNodeRef const& n, CharOwningContainer * cont, bool append=false) { + if(!detail::is_set_(n)) + return {}; _RYML_CB_CHECK(n.tree()->callbacks(), n.readable()); - return emitrs_json(*n.tree(), n.id(), cont); + return emitrs_json(*n.tree(), n.id(), EmitOptions{}, cont, append); } -/** emit+resize: YAML to the given `std::string`/`std::vector`-like container, - * resizing it as needed to fit the emitted YAML. */ +/** (3) emit+resize: YAML to a newly-created `std::string`/`std::vector`-like container. */ template -CharOwningContainer emitrs_yaml(ConstNodeRef const& n) +CharOwningContainer emitrs_yaml(ConstNodeRef const& n, EmitOptions const& opts={}) { + if(!detail::is_set_(n)) + return {}; _RYML_CB_CHECK(n.tree()->callbacks(), n.readable()); CharOwningContainer c; - emitrs_yaml(*n.tree(), n.id(), &c); + emitrs_yaml(*n.tree(), n.id(), opts, &c); return c; } -/** emit+resize: JSON to the given `std::string`/`std::vector`-like container, - * resizing it as needed to fit the emitted JSON. */ +/** (3) emit+resize: JSON to a newly-created `std::string`/`std::vector`-like container. */ template -CharOwningContainer emitrs_json(ConstNodeRef const& n) +CharOwningContainer emitrs_json(ConstNodeRef const& n, EmitOptions const& opts={}) { + if(!detail::is_set_(n)) + return {}; _RYML_CB_CHECK(n.tree()->callbacks(), n.readable()); CharOwningContainer c; - emitrs_json(*n.tree(), n.id(), &c); + emitrs_json(*n.tree(), n.id(), opts, &c); return c; } + /** @} */ diff --git a/src/c4/yml/event_handler_stack.hpp b/src/c4/yml/event_handler_stack.hpp index 34b0e4ea..4bb78077 100644 --- a/src/c4/yml/event_handler_stack.hpp +++ b/src/c4/yml/event_handler_stack.hpp @@ -89,6 +89,28 @@ struct EventHandlerStack #endif } + substr _stack_relocate_to_new_arena(csubstr s, csubstr prev, substr curr) + { + _RYML_CB_ASSERT(m_stack.m_callbacks, prev.is_super(s)); + auto pos = s.str - prev.str; + substr out = {curr.str + pos, s.len}; + _RYML_CB_ASSERT(m_stack.m_callbacks, curr.is_super(out)); + return out; + } + + void _stack_relocate_to_new_arena(csubstr prev, substr curr) + { + for(state &st : m_stack) + { + if(st.line_contents.rem.is_sub(prev)) + st.line_contents.rem = _stack_relocate_to_new_arena(st.line_contents.rem, prev, curr); + if(st.line_contents.full.is_sub(prev)) + st.line_contents.full = _stack_relocate_to_new_arena(st.line_contents.full, prev, curr); + if(st.line_contents.stripped.is_sub(prev)) + st.line_contents.stripped = _stack_relocate_to_new_arena(st.line_contents.stripped, prev, curr); + } + } + protected: // undefined at the end diff --git a/src/c4/yml/event_handler_tree.hpp b/src/c4/yml/event_handler_tree.hpp index 5852e2c6..f9bf597f 100644 --- a/src/c4/yml/event_handler_tree.hpp +++ b/src/c4/yml/event_handler_tree.hpp @@ -229,6 +229,7 @@ struct EventHandlerTree : public EventHandlerStacknode_id); + _RYML_CB_CHECK(m_stack.m_callbacks, !_has_any_(VAL)); _enable_(MAP|FLOW_SL); _save_loc(); _push(); @@ -236,6 +237,7 @@ struct EventHandlerTree : public EventHandlerStacknode_id); + _RYML_CB_CHECK(m_stack.m_callbacks, !_has_any_(VAL)); _enable_(MAP|BLOCK); _save_loc(); _push(); @@ -266,6 +268,7 @@ struct EventHandlerTree : public EventHandlerStacknode_id); + _RYML_CB_CHECK(m_stack.m_callbacks, !_has_any_(VAL)); _enable_(SEQ|FLOW_SL); _save_loc(); _push(); @@ -273,6 +276,7 @@ struct EventHandlerTree : public EventHandlerStacknode_id); + _RYML_CB_CHECK(m_stack.m_callbacks, !_has_any_(VAL)); _enable_(SEQ|BLOCK); _save_loc(); _push(); @@ -515,7 +519,24 @@ struct EventHandlerTree : public EventHandlerStackalloc_arena(len); + csubstr prev = m_tree->arena(); + substr out = m_tree->alloc_arena(len); + substr curr = m_tree->arena(); + if(curr.str != prev.str) + _stack_relocate_to_new_arena(prev, curr); + return out; + } + + substr alloc_arena(size_t len, substr *relocated) + { + csubstr prev = m_tree->arena(); + if(!prev.is_super(*relocated)) + return alloc_arena(len); + substr out = alloc_arena(len); + substr curr = m_tree->arena(); + if(curr.str != prev.str) + *relocated = _stack_relocate_to_new_arena(*relocated, prev, curr); + return out; } /** @} */ @@ -675,6 +696,7 @@ struct EventHandlerTree : public EventHandlerStack_p(m_curr->node_id)->m_val.scalar.len == 0); m_tree->_p(m_curr->node_id)->m_val.scalar.str = m_curr->line_contents.rem.str; } diff --git a/src/c4/yml/node.hpp b/src/c4/yml/node.hpp index 288521dd..510e0bfc 100644 --- a/src/c4/yml/node.hpp +++ b/src/c4/yml/node.hpp @@ -383,6 +383,9 @@ struct RoNodeMethods /** O(num_siblings). Forward to Tree::sibling_pos(). */ C4_ALWAYS_INLINE id_type sibling_pos(ConstImpl const& n) const RYML_NOEXCEPT { _C4RR(); _RYML_CB_ASSERT(tree_->callbacks(), n.readable()); return tree_->child_pos(tree_->parent(id_), n.m_id); } + C4_ALWAYS_INLINE id_type depth_asc() const RYML_NOEXCEPT { _C4RR(); return tree_->depth_asc(id_); } /** O(log(num_nodes)). Forward to Tree::depth_asc(). Node must be readable. */ + C4_ALWAYS_INLINE id_type depth_desc() const RYML_NOEXCEPT { _C4RR(); return tree_->depth_desc(id_); } /** O(num_nodes). Forward to Tree::depth_desc(). Node must be readable. */ + /** @} */ public: diff --git a/src/c4/yml/parse_engine.def.hpp b/src/c4/yml/parse_engine.def.hpp index b28e290e..c5d912c4 100644 --- a/src/c4/yml/parse_engine.def.hpp +++ b/src/c4/yml/parse_engine.def.hpp @@ -1213,7 +1213,7 @@ bool ParseEngine::_scan_scalar_plain_blck(ScannedScalar *C4_RESTRI { case '-': next_peeked = next_peeked.trimr("\n\r"); - _c4dbgpf("doc begin? peeked=[{}]~~~{}{}~~~", next_peeked.len, next_peeked.first(3), next_peeked.len > 3 ? "..." : ""); + _c4dbgpf("doc begin? peeked=[{}]~~~{}{}~~~", next_peeked.len, next_peeked.len >= 3 ? next_peeked.first(3) : next_peeked, next_peeked.len > 3 ? "..." : ""); if(_is_doc_begin_token(next_peeked)) { _c4dbgp("doc begin! scalar ended"); @@ -1222,7 +1222,7 @@ bool ParseEngine::_scan_scalar_plain_blck(ScannedScalar *C4_RESTRI break; case '.': next_peeked = next_peeked.trimr("\n\r"); - _c4dbgpf("doc end? peeked=[{}]~~~{}{}~~~", next_peeked.len, next_peeked.first(3), next_peeked.len > 3 ? "..." : ""); + _c4dbgpf("doc end? peeked=[{}]~~~{}{}~~~", next_peeked.len, next_peeked.len >= 3 ? next_peeked.first(3) : next_peeked, next_peeked.len > 3 ? "..." : ""); if(_is_doc_end_token(next_peeked)) { _c4dbgp("doc end! scalar ended"); @@ -2136,7 +2136,7 @@ void ParseEngine::_scan_block(ScannedBlock *C4_RESTRICT sb, size_t template template -bool ParseEngine::_filter_ws_handle_to_first_non_space(FilterProcessor &proc) noexcept +bool ParseEngine::_filter_ws_handle_to_first_non_space(FilterProcessor &proc) { _c4dbgfws("found whitespace '{}'", _c4prc(proc.curr())); _RYML_CB_ASSERT(this->callbacks(), proc.curr() == ' ' || proc.curr() == '\t'); @@ -2164,7 +2164,7 @@ bool ParseEngine::_filter_ws_handle_to_first_non_space(FilterProce template template -void ParseEngine::_filter_ws_copy_trailing(FilterProcessor &proc) noexcept +void ParseEngine::_filter_ws_copy_trailing(FilterProcessor &proc) { if(!_filter_ws_handle_to_first_non_space(proc)) { @@ -2175,7 +2175,7 @@ void ParseEngine::_filter_ws_copy_trailing(FilterProcessor &proc) template template -void ParseEngine::_filter_ws_skip_trailing(FilterProcessor &proc) noexcept +void ParseEngine::_filter_ws_skip_trailing(FilterProcessor &proc) { if(!_filter_ws_handle_to_first_non_space(proc)) { @@ -2201,7 +2201,7 @@ void ParseEngine::_filter_ws_skip_trailing(FilterProcessor &proc) template template -void ParseEngine::_filter_nl_plain(FilterProcessor &C4_RESTRICT proc, size_t indentation) noexcept +void ParseEngine::_filter_nl_plain(FilterProcessor &C4_RESTRICT proc, size_t indentation) { _RYML_CB_ASSERT(this->callbacks(), proc.curr() == '\n'); @@ -2232,7 +2232,7 @@ void ParseEngine::_filter_nl_plain(FilterProcessor &C4_RESTRICT pr template template -auto ParseEngine::_filter_plain(FilterProcessor &C4_RESTRICT proc, size_t indentation) noexcept -> decltype(proc.result()) +auto ParseEngine::_filter_plain(FilterProcessor &C4_RESTRICT proc, size_t indentation) -> decltype(proc.result()) { _RYML_CB_ASSERT(this->callbacks(), indentation != npos); _c4dbgfps("before=[{}]~~~{}~~~", proc.src.len, proc.src); @@ -2271,14 +2271,14 @@ auto ParseEngine::_filter_plain(FilterProcessor &C4_RESTRICT proc, template -FilterResult ParseEngine::filter_scalar_plain(csubstr scalar, substr dst, size_t indentation) noexcept +FilterResult ParseEngine::filter_scalar_plain(csubstr scalar, substr dst, size_t indentation) { FilterProcessorSrcDst proc(scalar, dst); return _filter_plain(proc, indentation); } template -FilterResult ParseEngine::filter_scalar_plain_in_place(substr dst, size_t cap, size_t indentation) noexcept +FilterResult ParseEngine::filter_scalar_plain_in_place(substr dst, size_t cap, size_t indentation) { FilterProcessorInplaceEndExtending proc(dst, cap); return _filter_plain(proc, indentation); @@ -2299,7 +2299,7 @@ FilterResult ParseEngine::filter_scalar_plain_in_place(substr dst, template template -void ParseEngine::_filter_nl_squoted(FilterProcessor &C4_RESTRICT proc) noexcept +void ParseEngine::_filter_nl_squoted(FilterProcessor &C4_RESTRICT proc) { _RYML_CB_ASSERT(this->callbacks(), proc.curr() == '\n'); @@ -2330,7 +2330,7 @@ void ParseEngine::_filter_nl_squoted(FilterProcessor &C4_RESTRICT template template -auto ParseEngine::_filter_squoted(FilterProcessor &C4_RESTRICT proc) noexcept -> decltype(proc.result()) +auto ParseEngine::_filter_squoted(FilterProcessor &C4_RESTRICT proc) -> decltype(proc.result()) { _c4dbgfsq("before=[{}]~~~{}~~~", proc.src.len, proc.src); @@ -2363,6 +2363,10 @@ auto ParseEngine::_filter_squoted(FilterProcessor &C4_RESTRICT pro proc.skip(); proc.copy(); } + else + { + _c4err("filter error"); + } break; default: proc.copy(); @@ -2378,14 +2382,14 @@ auto ParseEngine::_filter_squoted(FilterProcessor &C4_RESTRICT pro #undef _c4dbgfsq template -FilterResult ParseEngine::filter_scalar_squoted(csubstr scalar, substr dst) noexcept +FilterResult ParseEngine::filter_scalar_squoted(csubstr scalar, substr dst) { FilterProcessorSrcDst proc(scalar, dst); return _filter_squoted(proc); } template -FilterResult ParseEngine::filter_scalar_squoted_in_place(substr dst, size_t cap) noexcept +FilterResult ParseEngine::filter_scalar_squoted_in_place(substr dst, size_t cap) { FilterProcessorInplaceEndExtending proc(dst, cap); return _filter_squoted(proc); @@ -2406,7 +2410,7 @@ FilterResult ParseEngine::filter_scalar_squoted_in_place(substr ds template template -void ParseEngine::_filter_nl_dquoted(FilterProcessor &C4_RESTRICT proc) noexcept +void ParseEngine::_filter_nl_dquoted(FilterProcessor &C4_RESTRICT proc) { _RYML_CB_ASSERT(this->callbacks(), proc.curr() == '\n'); @@ -2482,7 +2486,7 @@ void ParseEngine::_filter_dquoted_backslash(FilterProcessor &C4_RE } else if(next == '\r') { - //proc.skip(); + proc.skip(); } else if(next == 'n') { @@ -2686,7 +2690,7 @@ FilterResultExtending ParseEngine::filter_scalar_dquoted_in_place( template template -void ParseEngine::_filter_chomp(FilterProcessor &C4_RESTRICT proc, BlockChomp_e chomp, size_t indentation) noexcept +void ParseEngine::_filter_chomp(FilterProcessor &C4_RESTRICT proc, BlockChomp_e chomp, size_t indentation) { _RYML_CB_ASSERT(this->callbacks(), chomp == CHOMP_CLIP || chomp == CHOMP_KEEP || chomp == CHOMP_STRIP); _RYML_CB_ASSERT(this->callbacks(), proc.rem().first_not_of(" \n\r") == npos); @@ -2832,7 +2836,7 @@ void ParseEngine::_filter_chomp(FilterProcessor &C4_RESTRICT proc, template template -void ParseEngine::_filter_block_indentation(FilterProcessor &C4_RESTRICT proc, size_t indentation) noexcept +void ParseEngine::_filter_block_indentation(FilterProcessor &C4_RESTRICT proc, size_t indentation) { csubstr rem = proc.rem(); // remaining if(rem.len) @@ -2877,7 +2881,7 @@ void ParseEngine::_filter_block_indentation(FilterProcessor &C4_RE template template -size_t ParseEngine::_handle_all_whitespace(FilterProcessor &C4_RESTRICT proc, BlockChomp_e chomp) noexcept +size_t ParseEngine::_handle_all_whitespace(FilterProcessor &C4_RESTRICT proc, BlockChomp_e chomp) { csubstr contents = proc.src.trimr(" \n\r"); _c4dbgfb("ws: contents_len={} wslen={}", contents.len, proc.src.len-contents.len); @@ -2906,7 +2910,7 @@ size_t ParseEngine::_handle_all_whitespace(FilterProcessor &C4_RES template template -size_t ParseEngine::_extend_to_chomp(FilterProcessor &C4_RESTRICT proc, size_t contents_len) noexcept +size_t ParseEngine::_extend_to_chomp(FilterProcessor &C4_RESTRICT proc, size_t contents_len) { _c4dbgfb("contents_len={}", contents_len); @@ -2945,7 +2949,7 @@ size_t ParseEngine::_extend_to_chomp(FilterProcessor &C4_RESTRICT template template -auto ParseEngine::_filter_block_literal(FilterProcessor &C4_RESTRICT proc, size_t indentation, BlockChomp_e chomp) noexcept -> decltype(proc.result()) +auto ParseEngine::_filter_block_literal(FilterProcessor &C4_RESTRICT proc, size_t indentation, BlockChomp_e chomp) -> decltype(proc.result()) { _c4dbgfbl("indentation={} before=[{}]~~~{}~~~", indentation, proc.src.len, proc.src); @@ -2994,14 +2998,14 @@ auto ParseEngine::_filter_block_literal(FilterProcessor &C4_RESTRI #undef _c4dbgfbl template -FilterResult ParseEngine::filter_scalar_block_literal(csubstr scalar, substr dst, size_t indentation, BlockChomp_e chomp) noexcept +FilterResult ParseEngine::filter_scalar_block_literal(csubstr scalar, substr dst, size_t indentation, BlockChomp_e chomp) { FilterProcessorSrcDst proc(scalar, dst); return _filter_block_literal(proc, indentation, chomp); } template -FilterResult ParseEngine::filter_scalar_block_literal_in_place(substr scalar, size_t cap, size_t indentation, BlockChomp_e chomp) noexcept +FilterResult ParseEngine::filter_scalar_block_literal_in_place(substr scalar, size_t cap, size_t indentation, BlockChomp_e chomp) { FilterProcessorInplaceEndExtending proc(scalar, cap); return _filter_block_literal(proc, indentation, chomp); @@ -3022,7 +3026,7 @@ FilterResult ParseEngine::filter_scalar_block_literal_in_place(sub template template -void ParseEngine::_filter_block_folded_newlines_leading(FilterProcessor &C4_RESTRICT proc, size_t indentation, size_t len) noexcept +void ParseEngine::_filter_block_folded_newlines_leading(FilterProcessor &C4_RESTRICT proc, size_t indentation, size_t len) { _filter_block_indentation(proc, indentation); while(proc.has_more_chars(len)) @@ -3059,7 +3063,7 @@ void ParseEngine::_filter_block_folded_newlines_leading(FilterProc template template -size_t ParseEngine::_filter_block_folded_newlines_compress(FilterProcessor &C4_RESTRICT proc, size_t num_newl, size_t wpos_at_first_newl) noexcept +size_t ParseEngine::_filter_block_folded_newlines_compress(FilterProcessor &C4_RESTRICT proc, size_t num_newl, size_t wpos_at_first_newl) { switch(num_newl) { @@ -3088,7 +3092,7 @@ size_t ParseEngine::_filter_block_folded_newlines_compress(FilterP template template -void ParseEngine::_filter_block_folded_newlines(FilterProcessor &C4_RESTRICT proc, size_t indentation, size_t len) noexcept +void ParseEngine::_filter_block_folded_newlines(FilterProcessor &C4_RESTRICT proc, size_t indentation, size_t len) { _RYML_CB_ASSERT(this->callbacks(), proc.curr() == '\n'); size_t num_newl = 0; @@ -3232,7 +3236,7 @@ void ParseEngine::_filter_block_folded_indented_block(FilterProces template template -auto ParseEngine::_filter_block_folded(FilterProcessor &C4_RESTRICT proc, size_t indentation, BlockChomp_e chomp) noexcept -> decltype(proc.result()) +auto ParseEngine::_filter_block_folded(FilterProcessor &C4_RESTRICT proc, size_t indentation, BlockChomp_e chomp) -> decltype(proc.result()) { _c4dbgfbf("indentation={} before=[{}]~~~{}~~~", indentation, proc.src.len, proc.src); @@ -3280,14 +3284,14 @@ auto ParseEngine::_filter_block_folded(FilterProcessor &C4_RESTRIC #undef _c4dbgfbf template -FilterResult ParseEngine::filter_scalar_block_folded(csubstr scalar, substr dst, size_t indentation, BlockChomp_e chomp) noexcept +FilterResult ParseEngine::filter_scalar_block_folded(csubstr scalar, substr dst, size_t indentation, BlockChomp_e chomp) { FilterProcessorSrcDst proc(scalar, dst); return _filter_block_folded(proc, indentation, chomp); } template -FilterResult ParseEngine::filter_scalar_block_folded_in_place(substr scalar, size_t cap, size_t indentation, BlockChomp_e chomp) noexcept +FilterResult ParseEngine::filter_scalar_block_folded_in_place(substr scalar, size_t cap, size_t indentation, BlockChomp_e chomp) { FilterProcessorInplaceEndExtending proc(scalar, cap); return _filter_block_folded(proc, indentation, chomp); @@ -3337,7 +3341,7 @@ csubstr ParseEngine::_filter_scalar_dquot(substr s) { const size_t len = r.required_len(); _c4dbgpf("filtering dquo scalar: not enough space: needs {}, have {}", len, s.len); - substr dst = m_evt_handler->alloc_arena(len); + substr dst = m_evt_handler->alloc_arena(len, &s); _c4dbgpf("filtering dquo scalar: dst.len={}", dst.len); _RYML_CB_ASSERT(this->callbacks(), dst.len == len); FilterResult rsd = this->filter_scalar_dquoted(s, dst); @@ -3364,7 +3368,7 @@ csubstr ParseEngine::_filter_scalar_literal(substr s, size_t inden else { _c4dbgpf("filtering block literal scalar: not enough space: needs {}, have {}", r.required_len(), s.len); - substr dst = m_evt_handler->alloc_arena(r.required_len()); + substr dst = m_evt_handler->alloc_arena(r.required_len(), &s); FilterResult rsd = this->filter_scalar_block_literal(s, dst, indentation, chomp); _RYML_CB_CHECK(m_evt_handler->m_stack.m_callbacks, rsd.valid()); _c4dbgpf("filtering block literal scalar: success! s=[{}]~~~{}~~~", rsd.get().len, rsd.get()); @@ -3387,7 +3391,7 @@ csubstr ParseEngine::_filter_scalar_folded(substr s, size_t indent else { _c4dbgpf("filtering block folded scalar: not enough space: needs {}, have {}", r.required_len(), s.len); - substr dst = m_evt_handler->alloc_arena(r.required_len()); + substr dst = m_evt_handler->alloc_arena(r.required_len(), &s); FilterResult rsd = this->filter_scalar_block_folded(s, dst, indentation, chomp); _RYML_CB_CHECK(m_evt_handler->m_stack.m_callbacks, rsd.valid()); _c4dbgpf("filtering block folded scalar: success! s=[{}]~~~{}~~~", rsd.get().len, rsd.get()); @@ -4723,9 +4727,16 @@ void ParseEngine::_handle_seq_imap() _c4dbgt("seqimap: go again", 0); if(_finished_line()) { - _line_ended(); - _scan_line(); - _c4dbgnextline(); + if(C4_LIKELY(!_finished_file())) + { + _line_ended(); + _scan_line(); + _c4dbgnextline(); + } + else + { + _c4err("parse error"); + } } goto seqimap_start; @@ -5741,7 +5752,7 @@ void ParseEngine::_handle_seq_block() addrem_flags(RKEY, RNXT); goto seqblck_finish; } - else if(first != '*') + else //if(first != '*') { _c4err("parse error"); } @@ -6596,6 +6607,7 @@ void ParseEngine::_handle_map_block() // if(m_state->at_line_beginning()) { + _RYML_CB_ASSERT(m_evt_handler->m_stack.m_callbacks, m_state->line_contents.indentation != npos); if(m_state->indentation_eq()) { _c4dbgpf("mapblck[QMRK]: skip {} from indref", m_state->indref); diff --git a/src/c4/yml/parse_engine.hpp b/src/c4/yml/parse_engine.hpp index 8231b393..5cca1198 100644 --- a/src/c4/yml/parse_engine.hpp +++ b/src/c4/yml/parse_engine.hpp @@ -403,34 +403,34 @@ class ParseEngine public: - /** @name deprecated parse_methods + /** @name deprecated parse methods * @{ */ /** @cond dev */ - template RYML_DEPRECATED("deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_place(csubstr filename, substr yaml, Tree *t, size_t node_id); - template RYML_DEPRECATED("deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_place( substr yaml, Tree *t, size_t node_id); - template RYML_DEPRECATED("deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_place(csubstr filename, substr yaml, Tree *t ); - template RYML_DEPRECATED("deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_place( substr yaml, Tree *t ); - template RYML_DEPRECATED("deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_place(csubstr filename, substr yaml, NodeRef node ); - template RYML_DEPRECATED("deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_place( substr yaml, NodeRef node ); - template RYML_DEPRECATED("deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_place(csubstr filename, substr yaml ); - template RYML_DEPRECATED("deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_place( substr yaml ); - template RYML_DEPRECATED("deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_arena(csubstr filename, csubstr yaml, Tree *t, size_t node_id); - template RYML_DEPRECATED("deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_arena( csubstr yaml, Tree *t, size_t node_id); - template RYML_DEPRECATED("deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_arena(csubstr filename, csubstr yaml, Tree *t ); - template RYML_DEPRECATED("deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_arena( csubstr yaml, Tree *t ); - template RYML_DEPRECATED("deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_arena(csubstr filename, csubstr yaml, NodeRef node ); - template RYML_DEPRECATED("deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_arena( csubstr yaml, NodeRef node ); - template RYML_DEPRECATED("deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_arena(csubstr filename, csubstr yaml ); - template RYML_DEPRECATED("deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_arena( csubstr yaml ); - template RYML_DEPRECATED("deliberately undefined. use the freestanding csubstr version in parse.hpp.") typename std::enable_if::type parse_in_arena(csubstr filename, substr yaml, Tree *t, size_t node_id); - template RYML_DEPRECATED("deliberately undefined. use the freestanding csubstr version in parse.hpp.") typename std::enable_if::type parse_in_arena( substr yaml, Tree *t, size_t node_id); - template RYML_DEPRECATED("deliberately undefined. use the freestanding csubstr version in parse.hpp.") typename std::enable_if::type parse_in_arena(csubstr filename, substr yaml, Tree *t ); - template RYML_DEPRECATED("deliberately undefined. use the freestanding csubstr version in parse.hpp.") typename std::enable_if::type parse_in_arena( substr yaml, Tree *t ); - template RYML_DEPRECATED("deliberately undefined. use the freestanding csubstr version in parse.hpp.") typename std::enable_if::type parse_in_arena(csubstr filename, substr yaml, NodeRef node ); - template RYML_DEPRECATED("deliberately undefined. use the freestanding csubstr version in parse.hpp.") typename std::enable_if::type parse_in_arena( substr yaml, NodeRef node ); - template RYML_DEPRECATED("deliberately undefined. use the freestanding csubstr version in parse.hpp.") typename std::enable_if::type parse_in_arena(csubstr filename, substr yaml ); - template RYML_DEPRECATED("deliberately undefined. use the freestanding csubstr version in parse.hpp.") typename std::enable_if::type parse_in_arena( substr yaml ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_place(csubstr filename, substr yaml, Tree *t, size_t node_id); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_place( substr yaml, Tree *t, size_t node_id); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_place(csubstr filename, substr yaml, Tree *t ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_place( substr yaml, Tree *t ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_place(csubstr filename, substr yaml, NodeRef node ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_place( substr yaml, NodeRef node ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_place(csubstr filename, substr yaml ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_place( substr yaml ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_arena(csubstr filename, csubstr yaml, Tree *t, size_t node_id); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_arena( csubstr yaml, Tree *t, size_t node_id); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_arena(csubstr filename, csubstr yaml, Tree *t ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_arena( csubstr yaml, Tree *t ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_arena(csubstr filename, csubstr yaml, NodeRef node ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_arena( csubstr yaml, NodeRef node ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_arena(csubstr filename, csubstr yaml ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding function in parse.hpp.") typename std::enable_if::type parse_in_arena( csubstr yaml ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding csubstr version in parse.hpp.") typename std::enable_if::type parse_in_arena(csubstr filename, substr yaml, Tree *t, size_t node_id); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding csubstr version in parse.hpp.") typename std::enable_if::type parse_in_arena( substr yaml, Tree *t, size_t node_id); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding csubstr version in parse.hpp.") typename std::enable_if::type parse_in_arena(csubstr filename, substr yaml, Tree *t ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding csubstr version in parse.hpp.") typename std::enable_if::type parse_in_arena( substr yaml, Tree *t ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding csubstr version in parse.hpp.") typename std::enable_if::type parse_in_arena(csubstr filename, substr yaml, NodeRef node ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding csubstr version in parse.hpp.") typename std::enable_if::type parse_in_arena( substr yaml, NodeRef node ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding csubstr version in parse.hpp.") typename std::enable_if::type parse_in_arena(csubstr filename, substr yaml ); + template RYML_DEPRECATED("removed, deliberately undefined. use the freestanding csubstr version in parse.hpp.") typename std::enable_if::type parse_in_arena( substr yaml ); /** @endcond */ /** @} */ @@ -460,14 +460,14 @@ class ParseEngine /** @{*/ /** filter a plain scalar */ - FilterResult filter_scalar_plain(csubstr scalar, substr dst, size_t indentation) noexcept; + FilterResult filter_scalar_plain(csubstr scalar, substr dst, size_t indentation); /** filter a plain scalar in place */ - FilterResult filter_scalar_plain_in_place(substr scalar, size_t cap, size_t indentation) noexcept; + FilterResult filter_scalar_plain_in_place(substr scalar, size_t cap, size_t indentation); /** filter a single-quoted scalar */ - FilterResult filter_scalar_squoted(csubstr scalar, substr dst) noexcept; + FilterResult filter_scalar_squoted(csubstr scalar, substr dst); /** filter a single-quoted scalar in place */ - FilterResult filter_scalar_squoted_in_place(substr scalar, size_t cap) noexcept; + FilterResult filter_scalar_squoted_in_place(substr scalar, size_t cap); /** filter a double-quoted scalar */ FilterResult filter_scalar_dquoted(csubstr scalar, substr dst); @@ -475,14 +475,14 @@ class ParseEngine FilterResultExtending filter_scalar_dquoted_in_place(substr scalar, size_t cap); /** filter a block-literal scalar */ - FilterResult filter_scalar_block_literal(csubstr scalar, substr dst, size_t indentation, BlockChomp_e chomp) noexcept; + FilterResult filter_scalar_block_literal(csubstr scalar, substr dst, size_t indentation, BlockChomp_e chomp); /** filter a block-literal scalar in place */ - FilterResult filter_scalar_block_literal_in_place(substr scalar, size_t cap, size_t indentation, BlockChomp_e chomp) noexcept; + FilterResult filter_scalar_block_literal_in_place(substr scalar, size_t cap, size_t indentation, BlockChomp_e chomp); /** filter a block-folded scalar */ - FilterResult filter_scalar_block_folded(csubstr scalar, substr dst, size_t indentation, BlockChomp_e chomp) noexcept; + FilterResult filter_scalar_block_folded(csubstr scalar, substr dst, size_t indentation, BlockChomp_e chomp); /** filter a block-folded scalar in place */ - FilterResult filter_scalar_block_folded_in_place(substr scalar, size_t cap, size_t indentation, BlockChomp_e chomp) noexcept; + FilterResult filter_scalar_block_folded_in_place(substr scalar, size_t cap, size_t indentation, BlockChomp_e chomp); /** @} */ @@ -598,34 +598,35 @@ class ParseEngine public: /** @cond dev */ - template auto _filter_plain(FilterProcessor &C4_RESTRICT proc, size_t indentation) noexcept -> decltype(proc.result()); - template auto _filter_squoted(FilterProcessor &C4_RESTRICT proc) noexcept -> decltype(proc.result()); + template auto _filter_plain(FilterProcessor &C4_RESTRICT proc, size_t indentation) -> decltype(proc.result()); + template auto _filter_squoted(FilterProcessor &C4_RESTRICT proc) -> decltype(proc.result()); template auto _filter_dquoted(FilterProcessor &C4_RESTRICT proc) -> decltype(proc.result()); - template auto _filter_block_literal(FilterProcessor &C4_RESTRICT proc, size_t indentation, BlockChomp_e chomp) noexcept -> decltype(proc.result()); - template auto _filter_block_folded(FilterProcessor &C4_RESTRICT proc, size_t indentation, BlockChomp_e chomp) noexcept -> decltype(proc.result()); + template auto _filter_block_literal(FilterProcessor &C4_RESTRICT proc, size_t indentation, BlockChomp_e chomp) -> decltype(proc.result()); + template auto _filter_block_folded(FilterProcessor &C4_RESTRICT proc, size_t indentation, BlockChomp_e chomp) -> decltype(proc.result()); /** @endcond */ public: /** @cond dev */ - template void _filter_nl_plain(FilterProcessor &C4_RESTRICT proc, size_t indentation) noexcept; - template void _filter_nl_squoted(FilterProcessor &C4_RESTRICT proc) noexcept; - template void _filter_nl_dquoted(FilterProcessor &C4_RESTRICT proc) noexcept; + template void _filter_nl_plain(FilterProcessor &C4_RESTRICT proc, size_t indentation); + template void _filter_nl_squoted(FilterProcessor &C4_RESTRICT proc); + template void _filter_nl_dquoted(FilterProcessor &C4_RESTRICT proc); - template bool _filter_ws_handle_to_first_non_space(FilterProcessor &C4_RESTRICT proc) noexcept; - template void _filter_ws_copy_trailing(FilterProcessor &C4_RESTRICT proc) noexcept; - template void _filter_ws_skip_trailing(FilterProcessor &C4_RESTRICT proc) noexcept; + template bool _filter_ws_handle_to_first_non_space(FilterProcessor &C4_RESTRICT proc); + template void _filter_ws_copy_trailing(FilterProcessor &C4_RESTRICT proc); + template void _filter_ws_skip_trailing(FilterProcessor &C4_RESTRICT proc); template void _filter_dquoted_backslash(FilterProcessor &C4_RESTRICT proc); - template void _filter_chomp(FilterProcessor &C4_RESTRICT proc, BlockChomp_e chomp, size_t indentation) noexcept; - template size_t _handle_all_whitespace(FilterProcessor &C4_RESTRICT proc, BlockChomp_e chomp) noexcept; - template size_t _extend_to_chomp(FilterProcessor &C4_RESTRICT proc, size_t contents_len) noexcept; - template void _filter_block_indentation(FilterProcessor &C4_RESTRICT proc, size_t indentation) noexcept; - template void _filter_block_folded_newlines(FilterProcessor &C4_RESTRICT proc, size_t indentation, size_t len) noexcept; - template size_t _filter_block_folded_newlines_compress(FilterProcessor &C4_RESTRICT proc, size_t num_newl, size_t wpos_at_first_newl) noexcept; - template void _filter_block_folded_newlines_leading(FilterProcessor &C4_RESTRICT proc, size_t indentation, size_t len) noexcept; + template void _filter_chomp(FilterProcessor &C4_RESTRICT proc, BlockChomp_e chomp, size_t indentation); + template size_t _handle_all_whitespace(FilterProcessor &C4_RESTRICT proc, BlockChomp_e chomp); + template size_t _extend_to_chomp(FilterProcessor &C4_RESTRICT proc, size_t contents_len); + template void _filter_block_indentation(FilterProcessor &C4_RESTRICT proc, size_t indentation); + template void _filter_block_folded_newlines(FilterProcessor &C4_RESTRICT proc, size_t indentation, size_t len); + template size_t _filter_block_folded_newlines_compress(FilterProcessor &C4_RESTRICT proc, size_t num_newl, size_t wpos_at_first_newl); + template void _filter_block_folded_newlines_leading(FilterProcessor &C4_RESTRICT proc, size_t indentation, size_t len); template void _filter_block_folded_indented_block(FilterProcessor &C4_RESTRICT proc, size_t indentation, size_t len, size_t curr_indentation) noexcept; + /** @endcond */ private: diff --git a/src/c4/yml/parser_state.hpp b/src/c4/yml/parser_state.hpp index 9072f2a6..2c9ec678 100644 --- a/src/c4/yml/parser_state.hpp +++ b/src/c4/yml/parser_state.hpp @@ -11,12 +11,6 @@ namespace yml { /** data type for @ref ParserState_e */ using ParserFlag_t = int; -#ifdef RYML_DBG -namespace detail { -csubstr _parser_flags_to_str(substr buf, ParserFlag_t flags); -} // namespace -#endif - /** Enumeration of the state flags for the parser */ typedef enum : ParserFlag_t { RTOP = 0x01 << 0, ///< reading at top level @@ -42,6 +36,14 @@ typedef enum : ParserFlag_t { RSEQIMAP = 0x01 << 17, } ParserState_e; +#ifdef RYML_DBG +/** @cond dev */ +namespace detail { +csubstr _parser_flags_to_str(substr buf, ParserFlag_t flags); +} // namespace +/** @endcond */ +#endif + /** Helper to control the line contents while parsing a buffer */ struct LineContents diff --git a/src/c4/yml/tree.cpp b/src/c4/yml/tree.cpp index 8ccc06c6..ce4a8162 100644 --- a/src/c4/yml/tree.cpp +++ b/src/c4/yml/tree.cpp @@ -216,7 +216,8 @@ void Tree::_relocate(substr next_arena) { _RYML_CB_ASSERT(m_callbacks, next_arena.not_empty()); _RYML_CB_ASSERT(m_callbacks, next_arena.len >= m_arena.len); - memcpy(next_arena.str, m_arena.str, m_arena_pos); + if(m_arena_pos) + memcpy(next_arena.str, m_arena.str, m_arena_pos); for(NodeData *C4_RESTRICT n = m_buf, *e = m_buf + m_cap; n != e; ++n) { if(in_arena(n->m_key.scalar)) @@ -1191,6 +1192,37 @@ id_type Tree::find_child(id_type node, csubstr const& name) const # pragma GCC diagnostic pop #endif +namespace { +id_type depth_desc_(Tree const& C4_RESTRICT t, id_type id, id_type currdepth=0, id_type maxdepth=0) +{ + maxdepth = currdepth > maxdepth ? currdepth : maxdepth; + for(id_type child = t.first_child(id); child != NONE; child = t.next_sibling(child)) + { + const id_type d = depth_desc_(t, child, currdepth+1, maxdepth); + maxdepth = d > maxdepth ? d : maxdepth; + } + return maxdepth; +} +} + +id_type Tree::depth_desc(id_type node) const +{ + _RYML_CB_ASSERT(m_callbacks, node != NONE); + return depth_desc_(*this, node); +} + +id_type Tree::depth_asc(id_type node) const +{ + _RYML_CB_ASSERT(m_callbacks, node != NONE); + id_type depth = 0; + while(!is_root(node)) + { + ++depth; + node = parent(node); + } + return depth; +} + //----------------------------------------------------------------------------- diff --git a/src/c4/yml/tree.hpp b/src/c4/yml/tree.hpp index 60edc084..38e8e8fc 100644 --- a/src/c4/yml/tree.hpp +++ b/src/c4/yml/tree.hpp @@ -514,6 +514,9 @@ class RYML_EXPORT Tree id_type doc(id_type i) const { id_type rid = root_id(); _RYML_CB_ASSERT(m_callbacks, is_stream(rid)); return child(rid, i); } //!< gets the @p i document node index. requires that the root node is a stream. + id_type depth_asc(id_type node) const; /**< O(log(num_tree_nodes)) get the ascending depth of the node: number of levels between root and node */ + id_type depth_desc(id_type node) const; /**< O(num_tree_nodes) get the descending depth of the node: number of levels between node and deepest child */ + /** @} */ public: diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7a519882..905292ce 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -46,6 +46,9 @@ c4_add_library(ryml-_testgroup LIBRARY_TYPE STATIC test_lib/test_group.cpp LIBS ryml ryml-_testlib c4fs FOLDER test) +if(RYML_DBG) + target_compile_definitions(ryml-_testgroup PUBLIC RYML_DBG) +endif() function(ryml_add_test_case_group name) ryml_add_test(${name}) target_link_libraries(ryml-test-${name} PUBLIC ryml-_testgroup) @@ -59,6 +62,9 @@ c4_add_library(ryml-_evt_handler_yaml_std LIBRARY_STATIC LIBS ryml FOLDER test ) +if(RYML_DBG) + target_compile_definitions(ryml-_evt_handler_yaml_std PUBLIC RYML_DBG) +endif() function(ryml_add_engine_test name) ryml_add_test(${name} ryml-_evt_handler_yaml_std) endfunction() diff --git a/test/test_emit.cpp b/test/test_emit.cpp index 09fd967f..055afc96 100644 --- a/test/test_emit.cpp +++ b/test/test_emit.cpp @@ -27,6 +27,7 @@ std::string emit2file(Emit &&fn) fclose(f); std::string result = fs::file_get_contents(filename.c_str()); fs::rmfile(filename.c_str()); + _c4dbgpf("emit result: [{}]~~~{}~~~", result.size(), to_csubstr(result)); return result; C4_SUPPRESS_WARNING_MSVC_POP } @@ -36,7 +37,9 @@ std::string emit2stream(Emit &&fn) { std::ostringstream ss; fn(ss); - return ss.str(); + std::string result = ss.str(); + _c4dbgpf("emit result: [{}]~~~{}~~~", result.size(), to_csubstr(result)); + return result; } template @@ -51,6 +54,16 @@ std::string emit2buf(Emit &&fn) out = fn(to_substr(buf)); } buf.resize(out.len); + _c4dbgpf("emit result: [{}]~~~{}~~~", buf.size(), to_csubstr(buf)); + return buf; +} + +template +std::string emitrs_append(csubstr first_part, Emit &&fn) +{ + std::string buf(first_part.begin(), first_part.end()); + fn(&buf); + _c4dbgpf("emit result: [{}]~~~{}~~~", buf.size(), to_csubstr(buf)); return buf; } @@ -59,6 +72,60 @@ std::string emit2buf(Emit &&fn) //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- +bool operator== (const EmitOptions lhs, const EmitOptions rhs) +{ + return memcmp(&lhs, &rhs, sizeof(lhs)) == 0; +} + +TEST(as_yaml, basic) +{ + Tree et; + { + as_yaml j(et); + EXPECT_EQ(j.tree, &et); + } + Tree t = parse_in_arena("[foo, bar]"); + { + as_yaml j(t); + EXPECT_EQ(j.tree, &t); + EXPECT_EQ(j.node, t.root_id()); + EXPECT_EQ(j.options, EmitOptions{}); + } + { + EmitOptions opts = EmitOptions{}.max_depth(10); + as_yaml j(t, opts); + EXPECT_EQ(j.tree, &t); + EXPECT_EQ(j.node, t.root_id()); + EXPECT_EQ(j.options, opts); + } + { + as_yaml j(t, 2u); + EXPECT_EQ(j.tree, &t); + EXPECT_EQ(j.node, 2u); + EXPECT_EQ(j.options, EmitOptions{}); + } + { + EmitOptions opts = EmitOptions{}.max_depth(10); + as_yaml j(t, 2u, opts); + EXPECT_EQ(j.tree, &t); + EXPECT_EQ(j.node, 2u); + EXPECT_EQ(j.options, opts); + } + { + as_yaml j(t[0]); + EXPECT_EQ(j.tree, &t); + EXPECT_EQ(j.node, 1u); + EXPECT_EQ(j.options, EmitOptions{}); + } + { + EmitOptions opts = EmitOptions{}.max_depth(10); + as_yaml j(t[0], opts); + EXPECT_EQ(j.tree, &t); + EXPECT_EQ(j.node, 1u); + EXPECT_EQ(j.options, opts); + } +} + TEST(as_json, basic) { Tree et; @@ -71,16 +138,40 @@ TEST(as_json, basic) as_json j(t); EXPECT_EQ(j.tree, &t); EXPECT_EQ(j.node, t.root_id()); + EXPECT_EQ(j.options, EmitOptions{}); + } + { + EmitOptions opts = EmitOptions{}.max_depth(10); + as_json j(t, opts); + EXPECT_EQ(j.tree, &t); + EXPECT_EQ(j.node, t.root_id()); + EXPECT_EQ(j.options, opts); } { as_json j(t, 2u); EXPECT_EQ(j.tree, &t); EXPECT_EQ(j.node, 2u); + EXPECT_EQ(j.options, EmitOptions{}); + } + { + EmitOptions opts = EmitOptions{}.max_depth(10); + as_json j(t, 2u, opts); + EXPECT_EQ(j.tree, &t); + EXPECT_EQ(j.node, 2u); + EXPECT_EQ(j.options, opts); } { as_json j(t[0]); EXPECT_EQ(j.tree, &t); EXPECT_EQ(j.node, 1u); + EXPECT_EQ(j.options, EmitOptions{}); + } + { + EmitOptions opts = EmitOptions{}.max_depth(10); + as_json j(t[0], opts); + EXPECT_EQ(j.tree, &t); + EXPECT_EQ(j.node, 1u); + EXPECT_EQ(j.options, opts); } } @@ -89,41 +180,157 @@ TEST(as_json, basic) //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- -template -void test_emits(Tree const& t, size_t id, std::string const& expected, std::string const& expected_json) + +void test_emits(Tree const& t, id_type id, std::string const& expected, std::string const& expected_json) { - EXPECT_EQ(emit2buf([&](substr buf){ return emit_yaml(t, id, buf); }), expected); - EXPECT_EQ(emit2buf([&](substr buf){ return emit_json(t, id, buf); }), expected_json); - EXPECT_EQ(emit2file([&](FILE *f){ return emit_yaml(t, id, f); }), expected); - EXPECT_EQ(emit2file([&](FILE *f){ return emit_json(t, id, f); }), expected_json); - EXPECT_EQ(emitrs_yaml(t, id), expected); - EXPECT_EQ(emitrs_json(t, id), expected_json); + EXPECT_EQ(emit2buf([&](substr buf){ return emit_yaml(t, id, buf); }), expected); + EXPECT_EQ(emit2buf([&](substr buf){ return emit_yaml(t, id, EmitOptions{}, buf); }), expected); + EXPECT_EQ(emit2buf([&](substr buf){ return emit_yaml(t, id, EmitOptions{}, buf); }), expected); + EXPECT_EQ(emit2buf([&](substr buf){ return emit_json(t, id, buf); }), expected_json); + EXPECT_EQ(emit2buf([&](substr buf){ return emit_json(t, id, EmitOptions{}, buf); }), expected_json); + EXPECT_EQ(emit2buf([&](substr buf){ EmitterBuf em(buf); return em.emit_as(EMIT_YAML, t, id, /*error_on_excess*/true); }), expected); + EXPECT_EQ(emit2buf([&](substr buf){ EmitterBuf em(buf); return em.emit_as(EMIT_JSON, t, id, /*error_on_excess*/true); }), expected_json); + EXPECT_EQ(emit2file([&](FILE *f){ return emit_yaml(t, id, f); }), expected); + EXPECT_EQ(emit2file([&](FILE *f){ return emit_yaml(t, id, EmitOptions{}, f); }), expected); + EXPECT_EQ(emit2file([&](FILE *f){ return emit_json(t, id, f); }), expected_json); + EXPECT_EQ(emit2file([&](FILE *f){ return emit_json(t, id, EmitOptions{}, f); }), expected_json); + EXPECT_EQ(emit2file([&](FILE *f){ EmitterFile em(f); return em.emit_as(EMIT_YAML, t, id, /*error_on_excess*/true).len; }), expected); + EXPECT_EQ(emit2file([&](FILE *f){ EmitterFile em(f); return em.emit_as(EMIT_JSON, t, id, /*error_on_excess*/true).len; }), expected_json); + if(id != NONE) + { + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ oss << as_yaml(t, id); }), expected); + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ oss << as_yaml(t, id, EmitOptions{}); }), expected); + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ oss << as_json(t, id); }), expected_json); + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ oss << as_json(t, id, EmitOptions{}); }), expected_json); + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ EmitterOStream em({}, oss); em.emit_as(EMIT_YAML, t, id, /*error_on_excess*/true); }), expected); + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ EmitterOStream em({}, oss); em.emit_as(EMIT_JSON, t, id, /*error_on_excess*/true); }), expected_json); + } + EXPECT_EQ(emitrs_yaml(t, id), expected); + EXPECT_EQ(emitrs_yaml(t, id, EmitOptions{}), expected); + EXPECT_EQ(emitrs_json(t, id), expected_json); + EXPECT_EQ(emitrs_json(t, id, EmitOptions{}), expected_json); + std::string append_prefix = "#before\n"; + EXPECT_EQ(emitrs_append(to_csubstr(append_prefix), [&](std::string *s) { emitrs_yaml(t, id, s, /*append*/true); } ), append_prefix + expected); + EXPECT_EQ(emitrs_append(to_csubstr(append_prefix), [&](std::string *s) { emitrs_yaml(t, id, EmitOptions{}, s, /*append*/true); } ), append_prefix + expected); + EXPECT_EQ(emitrs_append(to_csubstr(append_prefix), [&](std::string *s) { emitrs_json(t, id, s, /*append*/true); } ), append_prefix + expected_json); + EXPECT_EQ(emitrs_append(to_csubstr(append_prefix), [&](std::string *s) { emitrs_json(t, id, EmitOptions{}, s, /*append*/true); } ), append_prefix + expected_json); + // error on max depth + if(id == NONE) + return; + id_type max_depth = t.depth_desc(id); + if(max_depth > 1) + { + EmitOptions opts = EmitOptions{}.max_depth(0); + ExpectError::do_check(&t, [&]{ return emit2buf([&](substr buf){ return emit_yaml(t, id, opts, buf); }); }); + ExpectError::do_check(&t, [&]{ return emit2buf([&](substr buf){ return emit_json(t, id, opts, buf); }); }); + ExpectError::do_check(&t, [&]{ return emit2file([&](FILE *f){ return emit_yaml(t, id, opts, f); }); }); + ExpectError::do_check(&t, [&]{ return emit2file([&](FILE *f){ return emit_json(t, id, opts, f); }); }); + ExpectError::do_check(&t, [&]{ return emit2stream([&](std::ostringstream &oss){ oss << as_yaml(t, id, opts); }); }); + ExpectError::do_check(&t, [&]{ return emit2stream([&](std::ostringstream &oss){ oss << as_json(t, id, opts); }); }); + ExpectError::do_check(&t, [&]{ return emitrs_yaml(t, id, opts); }); + ExpectError::do_check(&t, [&]{ return emitrs_json(t, id, opts); }); + } } -template void test_emits(Tree const& t, std::string const& expected, std::string const& expected_json) { - EXPECT_EQ(emit2buf([&](substr buf){ return emit_yaml(t, buf); }), expected); - EXPECT_EQ(emit2buf([&](substr buf){ return emit_json(t, buf); }), expected_json); - EXPECT_EQ(emit2file([&](FILE *f){ return emit_yaml(t, f); }), expected); - EXPECT_EQ(emit2file([&](FILE *f){ return emit_json(t, f); }), expected_json); - EXPECT_EQ(emit2stream([&](std::ostream& s){ s << t; }), expected); - EXPECT_EQ(emit2stream([&](std::ostream& s){ s << as_json(t); }), expected_json); - EXPECT_EQ(emitrs_yaml(t), expected); - EXPECT_EQ(emitrs_json(t), expected_json); + EXPECT_EQ(emit2buf([&](substr buf){ return emit_yaml(t, buf); }), expected); + EXPECT_EQ(emit2buf([&](substr buf){ return emit_yaml(t, EmitOptions{}, buf); }), expected); + EXPECT_EQ(emit2buf([&](substr buf){ return emit_json(t, buf); }), expected_json); + EXPECT_EQ(emit2buf([&](substr buf){ return emit_json(t, EmitOptions{}, buf); }), expected_json); + EXPECT_EQ(emit2buf([&](substr buf){ EmitterBuf em(buf); return em.emit_as(EMIT_YAML, t, /*error_on_excess*/true); }), expected); + EXPECT_EQ(emit2buf([&](substr buf){ EmitterBuf em(buf); return em.emit_as(EMIT_JSON, t, /*error_on_excess*/true); }), expected_json); + EXPECT_EQ(emit2file([&](FILE *f){ return emit_yaml(t, f); }), expected); + EXPECT_EQ(emit2file([&](FILE *f){ return emit_yaml(t, EmitOptions{}, f); }), expected); + EXPECT_EQ(emit2file([&](FILE *f){ return emit_json(t, f); }), expected_json); + EXPECT_EQ(emit2file([&](FILE *f){ return emit_json(t, EmitOptions{}, f); }), expected_json); + EXPECT_EQ(emit2file([&](FILE *f){ EmitterFile em(f); return em.emit_as(EMIT_YAML, t, /*error_on_excess*/true).len; }), expected); + EXPECT_EQ(emit2file([&](FILE *f){ EmitterFile em(f); return em.emit_as(EMIT_JSON, t, /*error_on_excess*/true).len; }), expected_json); + if(!t.empty()) + { + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ oss << as_yaml(t); }), expected); + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ oss << as_yaml(t, EmitOptions{}); }), expected); + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ oss << as_json(t); }), expected_json); + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ oss << as_json(t, EmitOptions{}); }), expected_json); + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ EmitterOStream em({}, oss); em.emit_as(EMIT_YAML, t, /*error_on_excess*/true); }), expected); + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ EmitterOStream em({}, oss); em.emit_as(EMIT_JSON, t, /*error_on_excess*/true); }), expected_json); + } + EXPECT_EQ(emitrs_yaml(t), expected); + EXPECT_EQ(emitrs_yaml(t, EmitOptions{}), expected); + EXPECT_EQ(emitrs_json(t), expected_json); + EXPECT_EQ(emitrs_json(t, EmitOptions{}), expected_json); + std::string append_prefix = "#before\n"; + EXPECT_EQ(emitrs_append(to_csubstr(append_prefix), [&](std::string *s) { emitrs_yaml(t, s, /*append*/true); } ), append_prefix + expected); + EXPECT_EQ(emitrs_append(to_csubstr(append_prefix), [&](std::string *s) { emitrs_yaml(t, EmitOptions{}, s, /*append*/true); } ), append_prefix + expected); + EXPECT_EQ(emitrs_append(to_csubstr(append_prefix), [&](std::string *s) { emitrs_json(t, s, /*append*/true); } ), append_prefix + expected_json); + EXPECT_EQ(emitrs_append(to_csubstr(append_prefix), [&](std::string *s) { emitrs_json(t, EmitOptions{}, s, /*append*/true); } ), append_prefix + expected_json); + // error on max depth + id_type max_depth = t.empty() ? 0 : t.depth_desc(t.root_id()); + if(max_depth > 1) + { + EmitOptions opts = EmitOptions{}.max_depth(0); + ExpectError::do_check(&t, [&]{ return emit2buf([&](substr buf){ return emit_yaml(t, opts, buf); }); }); + ExpectError::do_check(&t, [&]{ return emit2buf([&](substr buf){ return emit_json(t, opts, buf); }); }); + ExpectError::do_check(&t, [&]{ return emit2file([&](FILE *f){ return emit_yaml(t, opts, f); }); }); + ExpectError::do_check(&t, [&]{ return emit2file([&](FILE *f){ return emit_json(t, opts, f); }); }); + ExpectError::do_check(&t, [&]{ return emit2stream([&](std::ostringstream &oss){ oss << as_yaml(ConstNodeRef(&t), opts); }); }); + ExpectError::do_check(&t, [&]{ return emit2stream([&](std::ostringstream &oss){ oss << as_json(ConstNodeRef(&t), opts); }); }); + ExpectError::do_check(&t, [&]{ return emitrs_yaml(t, opts); }); + ExpectError::do_check(&t, [&]{ return emitrs_json(t, opts); }); + } } -template -void test_emits(ConstNodeRef t, std::string const& expected, std::string const& expected_json) +void test_emits(ConstNodeRef n, std::string const& expected, std::string const& expected_json) { - EXPECT_EQ(emit2buf([&](substr buf){ return emit_yaml(t, buf); }), expected); - EXPECT_EQ(emit2buf([&](substr buf){ return emit_json(t, buf); }), expected_json); - EXPECT_EQ(emit2file([&](FILE *f){ return emit_yaml(t, f); }), expected); - EXPECT_EQ(emit2file([&](FILE *f){ return emit_json(t, f); }), expected_json); - EXPECT_EQ(emit2stream([&](std::ostream& s){ s << t; }), expected); - EXPECT_EQ(emit2stream([&](std::ostream& s){ s << as_json(t); }), expected_json); - EXPECT_EQ(emitrs_yaml(t), expected); - EXPECT_EQ(emitrs_json(t), expected_json); + EXPECT_EQ(emit2buf([&](substr buf){ return emit_yaml(n, buf); }), expected); + EXPECT_EQ(emit2buf([&](substr buf){ return emit_yaml(n, EmitOptions{}, buf); }), expected); + EXPECT_EQ(emit2buf([&](substr buf){ return emit_json(n, buf); }), expected_json); + EXPECT_EQ(emit2buf([&](substr buf){ return emit_json(n, EmitOptions{}, buf); }), expected_json); + EXPECT_EQ(emit2buf([&](substr buf){ EmitterBuf em(buf); return em.emit_as(EMIT_YAML, n, /*error_on_excess*/true); }), expected); + EXPECT_EQ(emit2buf([&](substr buf){ EmitterBuf em(buf); return em.emit_as(EMIT_JSON, n, /*error_on_excess*/true); }), expected_json); + EXPECT_EQ(emit2file([&](FILE *f){ return emit_yaml(n, f); }), expected); + EXPECT_EQ(emit2file([&](FILE *f){ return emit_yaml(n, EmitOptions{}, f); }), expected); + EXPECT_EQ(emit2file([&](FILE *f){ return emit_json(n, f); }), expected_json); + EXPECT_EQ(emit2file([&](FILE *f){ return emit_json(n, EmitOptions{}, f); }), expected_json); + EXPECT_EQ(emit2file([&](FILE *f){ EmitterFile em(f); return em.emit_as(EMIT_YAML, n, /*error_on_excess*/true).len; }), expected); + EXPECT_EQ(emit2file([&](FILE *f){ EmitterFile em(f); return em.emit_as(EMIT_JSON, n, /*error_on_excess*/true).len; }), expected_json); + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ oss << n; }), expected); + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ oss << as_yaml(n); }), expected); + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ oss << as_yaml(n, EmitOptions{}); }), expected); + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ oss << as_json(n); }), expected_json); + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ oss << as_json(n, EmitOptions{}); }), expected_json); + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ EmitterOStream em({}, oss); em.emit_as(EMIT_YAML, n, /*error_on_excess*/true); }), expected); + EXPECT_EQ(emit2stream([&](std::ostringstream &oss){ EmitterOStream em({}, oss); em.emit_as(EMIT_JSON, n, /*error_on_excess*/true); }), expected_json); + EXPECT_EQ(emitrs_yaml(n), expected); + EXPECT_EQ(emitrs_yaml(n, EmitOptions{}), expected); + EXPECT_EQ(emitrs_json(n), expected_json); + EXPECT_EQ(emitrs_json(n, EmitOptions{}), expected_json); + std::string append_prefix = "#before\n"; + EXPECT_EQ(emitrs_append(to_csubstr(append_prefix), [&](std::string *s) { emitrs_yaml(n, s, /*append*/true); } ), append_prefix + expected); + EXPECT_EQ(emitrs_append(to_csubstr(append_prefix), [&](std::string *s) { emitrs_yaml(n, EmitOptions{}, s, /*append*/true); } ), append_prefix + expected); + EXPECT_EQ(emitrs_append(to_csubstr(append_prefix), [&](std::string *s) { emitrs_json(n, s, /*append*/true); } ), append_prefix + expected_json); + EXPECT_EQ(emitrs_append(to_csubstr(append_prefix), [&](std::string *s) { emitrs_json(n, EmitOptions{}, s, /*append*/true); } ), append_prefix + expected_json); + // error on max depth + if(n.tree() && n.id() != NONE) + { + id_type max_depth = n.depth_desc(); + if(max_depth > 1) + { + EmitOptions opts = EmitOptions{}.max_depth(0); + ExpectError::do_check(n.tree(), [&]{ + return emit2buf([&](substr buf){ + return emit_yaml(n, opts, buf); + }); + }); + ExpectError::do_check(n.tree(), [&]{ return emit2buf([&](substr buf){ return emit_json(n, opts, buf); }); }); + ExpectError::do_check(n.tree(), [&]{ return emit2file([&](FILE *f){ return emit_yaml(n, opts, f); }); }); + ExpectError::do_check(n.tree(), [&]{ return emit2file([&](FILE *f){ return emit_json(n, opts, f); }); }); + ExpectError::do_check(n.tree(), [&]{ return emit2stream([&](std::ostringstream &oss){ oss << as_yaml(n, opts); }); }); + ExpectError::do_check(n.tree(), [&]{ return emit2stream([&](std::ostringstream &oss){ oss << as_json(n, opts); }); }); + ExpectError::do_check(n.tree(), [&]{ return emitrs_yaml(n, opts); }); + ExpectError::do_check(n.tree(), [&]{ return emitrs_json(n, opts); }); + } + } } @@ -134,6 +341,13 @@ TEST(emit, empty_tree) test_emits(t, expected, expected); } +TEST(emit, empty_node) +{ + std::string expected = R"()"; + test_emits(NodeRef{}, expected, expected); + test_emits(ConstNodeRef{}, expected, expected); +} + TEST(emit, existing_tree) { const Tree t = parse_in_arena("[foo, bar]"); @@ -150,7 +364,7 @@ TEST(emit, no_node) test_emits(t, NONE, expected, expected_json); } -TEST(emit, existing_seq_node) +TEST(emit, existing_seq_node_flow) { Tree nct = parse_in_arena("[foo, bar, [nested, seq], {nested: map}]"); Tree const& t = nct; @@ -187,7 +401,7 @@ TEST(emit, existing_seq_node) nct._add_flags(n.id(), FLOW_SL); expected = "foo"; { - SCOPED_TRACE("t, id"); + SCOPED_TRACE("noderef"); test_emits(n, expected, expected_json); } { @@ -211,14 +425,13 @@ TEST(emit, existing_seq_node) nct._add_flags(n.id(), FLOW_SL); expected = "bar"; { - SCOPED_TRACE("t, id"); + SCOPED_TRACE("noderef"); test_emits(n, expected, expected_json); } { SCOPED_TRACE("t, id"); test_emits(t, n.id(), expected, expected_json); } - } { SCOPED_TRACE("t[2]"); @@ -236,7 +449,7 @@ TEST(emit, existing_seq_node) expected = "[nested,seq]"; nct._add_flags(n.id(), FLOW_SL); { - SCOPED_TRACE("t, id"); + SCOPED_TRACE("noderef"); test_emits(n, expected, expected_json); } { @@ -259,8 +472,127 @@ TEST(emit, existing_seq_node) } expected = "{nested: map}"; nct._add_flags(n.id(), FLOW_SL); + { + SCOPED_TRACE("noderef"); + test_emits(n, expected, expected_json); + } { SCOPED_TRACE("t, id"); + test_emits(t, n.id(), expected, expected_json); + } + } +} + +TEST(emit, existing_seq_node_block) +{ + Tree nct = parse_in_arena("- foo\n- bar\n- - nested\n - seq\n- nested: map\n"); + Tree const& t = nct; + { + SCOPED_TRACE("full"); + std::string expected = "- foo\n- bar\n- - nested\n - seq\n- nested: map\n"; + std::string expected_json = R"(["foo","bar",["nested","seq"],{"nested": "map"}])"; + { + SCOPED_TRACE("rootref"); + test_emits(t.crootref(), expected, expected_json); + } + { + SCOPED_TRACE("t"); + test_emits(t, expected, expected_json); + } + { + SCOPED_TRACE("t, id"); + test_emits(t, t.root_id(), expected, expected_json); + } + } + { + SCOPED_TRACE("t[0]"); + ConstNodeRef n = t[0]; + std::string expected = "foo\n"; + std::string expected_json = "\"foo\""; + { + SCOPED_TRACE("noderef"); + test_emits(n, expected, expected_json); + } + { + SCOPED_TRACE("t, id"); + test_emits(t, n.id(), expected, expected_json); + } + nct._add_flags(n.id(), FLOW_SL); + expected = "foo"; + { + SCOPED_TRACE("noderef"); + test_emits(n, expected, expected_json); + } + { + SCOPED_TRACE("t, id"); + test_emits(t, n.id(), expected, expected_json); + } + } + { + SCOPED_TRACE("t[1]"); + ConstNodeRef n = t[1]; + std::string expected = "bar\n"; + std::string expected_json = "\"bar\""; + { + SCOPED_TRACE("noderef"); + test_emits(n, expected, expected_json); + } + { + SCOPED_TRACE("t, id"); + test_emits(t, n.id(), expected, expected_json); + } + nct._add_flags(n.id(), FLOW_SL); + expected = "bar"; + { + SCOPED_TRACE("noderef"); + test_emits(n, expected, expected_json); + } + { + SCOPED_TRACE("t, id"); + test_emits(t, n.id(), expected, expected_json); + } + } + { + SCOPED_TRACE("t[2]"); + ConstNodeRef n = t[2]; + std::string expected = "- nested\n- seq\n"; + std::string expected_json = "[\"nested\",\"seq\"]"; + { + SCOPED_TRACE("noderef"); + test_emits(n, expected, expected_json); + } + { + SCOPED_TRACE("t, id"); + test_emits(t, n.id(), expected, expected_json); + } + expected = "[nested,seq]"; + nct._add_flags(n.id(), FLOW_SL); + { + SCOPED_TRACE("noderef"); + test_emits(n, expected, expected_json); + } + { + SCOPED_TRACE("t, id"); + test_emits(t, n.id(), expected, expected_json); + } + } + { + SCOPED_TRACE("t[3]"); + ConstNodeRef n = t[3]; + std::string expected = "nested: map\n"; + std::string expected_json = "{\"nested\": \"map\"}"; + { + SCOPED_TRACE("noderef"); + test_emits(n, expected, expected_json); + } + { + SCOPED_TRACE("t, id"); + test_emits(t, n.id(), expected, expected_json); + } + expected = "{nested: map}"; + nct._add_flags(n.id(), FLOW_SL); + { + SCOPED_TRACE("noderef"); test_emits(n, expected, expected_json); } { @@ -270,7 +602,7 @@ TEST(emit, existing_seq_node) } } -TEST(emit, existing_map_node) +TEST(emit, existing_map_node_flow) { Tree nct = parse_in_arena("{0: foo, 1: bar, 2: [nested, seq], 3: {nested: map}}"); Tree const& t = nct; @@ -391,6 +723,127 @@ TEST(emit, existing_map_node) } } +TEST(emit, existing_map_node_block) +{ + Tree nct = parse_in_arena("0: foo\n1: bar\n2:\n - nested\n - seq\n3:\n nested: map\n"); + Tree const& t = nct; + { + SCOPED_TRACE("root"); + std::string expected = "0: foo\n1: bar\n2:\n - nested\n - seq\n3:\n nested: map\n"; + std::string expected_json = R"({"0": "foo","1": "bar","2": ["nested","seq"],"3": {"nested": "map"}})"; + { + SCOPED_TRACE("rootref"); + test_emits(t.rootref(), expected, expected_json); + } + { + SCOPED_TRACE("t"); + test_emits(t, expected, expected_json); + } + { + SCOPED_TRACE("t, id"); + test_emits(t, t.root_id(), expected, expected_json); + } + } + { + SCOPED_TRACE("t[0]"); + ConstNodeRef n = t[0]; + std::string expected = "0: foo\n"; + std::string expected_json = "\"0\": \"foo\""; + { + SCOPED_TRACE("noderef"); + test_emits(n, expected, expected_json); + } + { + SCOPED_TRACE("t, id"); + test_emits(t, n.id(), expected, expected_json); + } + expected = "0: foo"; + nct._add_flags(n.id(), FLOW_SL); + { + SCOPED_TRACE("t, id"); + test_emits(n, expected, expected_json); + } + { + SCOPED_TRACE("t, id"); + test_emits(t, n.id(), expected, expected_json); + } + } + { + SCOPED_TRACE("t[1]"); + ConstNodeRef n = t[1]; + std::string expected = "1: bar\n"; + std::string expected_json = "\"1\": \"bar\""; + { + SCOPED_TRACE("noderef"); + test_emits(n, expected, expected_json); + } + { + SCOPED_TRACE("t, id"); + test_emits(t, n.id(), expected, expected_json); + } + expected = "1: bar"; + nct._add_flags(n.id(), FLOW_SL); + { + SCOPED_TRACE("t, id"); + test_emits(n, expected, expected_json); + } + { + SCOPED_TRACE("t, id"); + test_emits(t, n.id(), expected, expected_json); + } + } + { + SCOPED_TRACE("t[2]"); + ConstNodeRef n = t[2]; + std::string expected = "2:\n - nested\n - seq\n"; + std::string expected_json = "\"2\": [\"nested\",\"seq\"]"; + { + SCOPED_TRACE("noderef"); + test_emits(n, expected, expected_json); + } + { + SCOPED_TRACE("t, id"); + test_emits(t, n.id(), expected, expected_json); + } + expected = "2:\n - nested\n - seq\n"; + nct._rem_flags(n.id(), CONTAINER_STYLE); + nct._add_flags(n.id(), BLOCK); + { + SCOPED_TRACE("t, id"); + test_emits(n, expected, expected_json); + } + { + SCOPED_TRACE("t, id"); + test_emits(t, n.id(), expected, expected_json); + } + } + { + SCOPED_TRACE("t[3]"); + ConstNodeRef n = t[3]; + std::string expected = "3:\n nested: map\n"; + std::string expected_json = "\"3\": {\"nested\": \"map\"}"; + { + SCOPED_TRACE("noderef"); + test_emits(n, expected, expected_json); + } + { + SCOPED_TRACE("t, id"); + test_emits(t, n.id(), expected, expected_json); + } + expected = "3:\n nested: map\n"; + nct._rem_flags(n.id(), CONTAINER_STYLE); + nct._add_flags(n.id(), BLOCK); + { + SCOPED_TRACE("t, id"); + test_emits(n, expected, expected_json); + } + { + SCOPED_TRACE("t, id"); + test_emits(t, n.id(), expected, expected_json); + } + } +} + TEST(emit, percent_is_quoted) { Tree ti = parse_in_arena("{}"); diff --git a/test/test_lib/test_case.cpp b/test/test_lib/test_case.cpp index d27e6d91..ef591937 100644 --- a/test/test_lib/test_case.cpp +++ b/test/test_lib/test_case.cpp @@ -173,9 +173,12 @@ std::string format_error(const char* msg, size_t len, Location loc) #ifndef RYML_NO_DEFAULT_CALLBACKS report_error_impl(msg, len, loc, nullptr); #endif - if(!loc) - return msg; std::string out; + if(!loc) + { + out.assign(msg, len); + return out; + } if(!loc.name.empty()) c4::formatrs_append(&out, "{}:", loc.name); c4::formatrs_append(&out, "{}:{}:", loc.line, loc.col); @@ -262,6 +265,11 @@ void ExpectError::check_success(Tree *tree, std::function fn) EXPECT_FALSE(context.m_got_an_error); } +void ExpectError::do_check(Tree const* tree, std::function fn, Location expected_location) +{ + do_check(const_cast(tree), fn, expected_location); +} + void ExpectError::do_check(Tree *tree, std::function fn, Location expected_location) { auto context = ExpectError(tree, expected_location); diff --git a/test/test_lib/test_case.hpp b/test/test_lib/test_case.hpp index bc2a9939..e85e964b 100644 --- a/test/test_lib/test_case.hpp +++ b/test/test_lib/test_case.hpp @@ -203,8 +203,9 @@ struct ExpectError ExpectError(Tree *tree, Location loc={}); ~ExpectError(); - static void do_check( std::function fn, Location expected={}) { do_check(nullptr, fn, expected); } + static void do_check( std::function fn, Location expected={}) { do_check((const Tree*)nullptr, fn, expected); } static void do_check(Tree *tree, std::function fn, Location expected={}); + static void do_check(Tree const *tree, std::function fn, Location expected={}); static void check_assertion( std::function fn, Location expected={}) { check_assertion(nullptr, fn, expected); } static void check_assertion(Tree *tree, std::function fn, Location expected={}); static void check_success( std::function fn) { check_success(nullptr, fn); }; diff --git a/test/test_parser.cpp b/test/test_parser.cpp index 49b85bfb..1baba8f0 100644 --- a/test/test_parser.cpp +++ b/test/test_parser.cpp @@ -400,6 +400,23 @@ TEST(Parser, estimate_tree_capacity) )")); } +TEST(Parser, alloc_arena) +{ + Tree tree; + Parser::handler_type evt_handler = {}; + evt_handler.reset(&tree, tree.root_id()); + substr bufa = evt_handler.alloc_arena(64); + bufa.fill('a'); + csubstr prev = bufa; + csubstr prev_arena = tree.arena(); + substr bufb = evt_handler.alloc_arena(64, &bufa); + csubstr curr_arena = tree.arena(); + EXPECT_NE(prev_arena.str, curr_arena.str); + EXPECT_NE(prev.str, bufa.str); + EXPECT_EQ(bufa.first_not_of('a'), npos); + (void)bufb; +} + TEST(parse_in_place, overloads) { char src1_[] = "{a: b}"; diff --git a/test/test_scalar_squoted.cpp b/test/test_scalar_squoted.cpp index 6a329be5..3cd5b4bf 100644 --- a/test/test_scalar_squoted.cpp +++ b/test/test_scalar_squoted.cpp @@ -283,6 +283,39 @@ void verify_error_is_reported(csubstr case_name, csubstr yaml, Location loc={}) }, loc); } +void verify_filter_error_is_reported(csubstr case_name, csubstr scalar_, Location loc={}) +{ + SCOPED_TRACE(case_name); + SCOPED_TRACE(scalar_); + { + Tree t; + ExpectError::do_check(&t, [&](){ + Parser::handler_type evt_handler = {}; + Parser parser(&evt_handler); + evt_handler.reset(&t, t.root_id()); + std::string buf(scalar_.begin(), scalar_.end()); + buf.resize(scalar_.len * 2); + substr scalar = to_substr(buf).first(scalar_.len); + to_substr(buf).sub(scalar_.len).fill('_'); + FilterResult result = parser.filter_scalar_squoted_in_place(scalar, buf.size()); + return result; + }, loc); + } + { + Tree t; + ExpectError::do_check(&t, [&](){ + Parser::handler_type evt_handler = {}; + Parser parser(&evt_handler); + evt_handler.reset(&t, t.root_id()); + std::string buf; + buf.resize(scalar_.len * 2); + to_substr(buf).fill('_'); + FilterResult result = parser.filter_scalar_squoted(scalar_, to_substr(buf)); + return result; + }, loc); + } +} + TEST(single_quoted, error_on_unmatched_quotes) { verify_error_is_reported("map block", R"(foo: '" @@ -363,6 +396,11 @@ bar: ''')"); verify_error_is_reported("seq flow", R"([what, '''])"); } +TEST(single_quoted, error_on_isolated_quotes) +{ + verify_filter_error_is_reported("isolated quotes", "a'a'a'a"); +} + //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- diff --git a/test/test_seq.cpp b/test/test_seq.cpp index e092901b..3436c6bd 100644 --- a/test/test_seq.cpp +++ b/test/test_seq.cpp @@ -205,6 +205,25 @@ TEST(simple_seq, flow_tab_tokens__3_flow_tabs_everywhere) #endif // RYML_WITH_TAB_TOKENS +TEST(simple_seq, unterminated_seqimap) +{ + { + SCOPED_TRACE("space after"); + Tree t; + ExpectError::do_check(&t, [&]{ + parse_in_arena("[a: "); + }); + } + { + SCOPED_TRACE("no space after"); + Tree t; + ExpectError::do_check(&t, [&]{ + parse_in_arena("[a:"); + }); + } +} + + //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- @@ -1081,6 +1100,14 @@ R"([ , , ] LineCol(1, 3) ); +ADD_CASE_TO_GROUP("simple seq flow, seqimap, at line end", +R"([a: + +b] +)", +N(SFS, L{N(MFS, L{N(KP|VP, "a", "b")})}) +); + } } // namespace yml diff --git a/test/test_suite/test_suite_event_handler.hpp b/test/test_suite/test_suite_event_handler.hpp index 9addf034..c830e1d6 100644 --- a/test/test_suite/test_suite_event_handler.hpp +++ b/test/test_suite/test_suite_event_handler.hpp @@ -90,9 +90,9 @@ struct EventHandlerYamlStd : public EventHandlerStacklevel, tmp); + _disable_(_VALMASK|VAL_STYLE); // create the map. // this will push a new level, and tmp is one further begin_map_val_flow(); @@ -557,9 +567,26 @@ struct EventHandlerYamlStd : public EventHandlerStack