Add TagCache

Author: biojppm

changelog/current.md

@@ -13,6 +13,7 @@ This release focuses in compliance with the YAML standard, mostly by ensuring pa
    - `NodeType`: rename TAGD->TAGH and TAGV->TAGP
    - Internal changes to improve the design of event handlers, moving relocation and error checking logic to `ParseEngine`, where it is most suited.
    - Fix warnings with clang-tidy 22
+- Add `TagCache` accelerator structure (in `c4/yml/tag.hpp`), used by `Tree::resolve_tags()` and while parsing. This reduces significantly the arena requirements for heavily-tagged YAML by allowing reuse of resolved tags.
 
 
 ## YAML fixes: valid cases

samples/quickstart.cpp

@@ -5150,14 +5150,25 @@ void sample_tag_directives()
 %TAG !m! !meta-
 --- !m!light green
 )");
-    // tags are not resolved by default. Use .resolve_tags() to
-    // accomplish this:
-    tree.resolve_tags();
+    // tags are not resolved by default. Use Tree::resolve_tags()
+    // to accomplish this in an existing tree:
+    ryml::TagCache tag_cache; // reduces memory requirements by reusing resolved tags
+    tree.resolve_tags(tag_cache);
     CHECK(ryml::emitrs_yaml<std::string>(tree) == R"(%TAG !m! !my-
 --- !<!my-light> fluorescent
 ...
 %TAG !m! !meta-
 --- !<!meta-light> green
+)");
+    // You can also Use ParserOptions to force resolution of tags
+    // while parsing:
+    ryml::ParserOptions opts = ryml::ParserOptions{}.resolve_tags(true);
+    ryml::Tree resolved_tree = ryml::parse_in_arena(ryml::to_csubstr(yaml), opts);
+    CHECK(ryml::emitrs_yaml<std::string>(resolved_tree) == R"(%TAG !m! !my-
+--- !<!my-light> fluorescent
+...
+%TAG !m! !meta-
+--- !<!meta-light> green
 )");
     // see also tree.normalize_tags()
     // see also tree.normalize_tags_long()

src/c4/yml/event_handler_tree.hpp

@@ -37,6 +37,7 @@ struct EventHandlerTree : public EventHandlerStack<EventHandlerTree, EventHandle
      * @{ */
 
     using state = EventHandlerTreeState;
+    enum { requires_strings_on_buffers = false };
 
     /** @} */
 
@@ -45,6 +46,7 @@ struct EventHandlerTree : public EventHandlerStack<EventHandlerTree, EventHandle
     /** @cond dev */
     Tree *C4_RESTRICT m_tree;
     id_type m_curr_doc;
+    TagCache m_tag_cache;
 
     #ifdef RYML_DBG
     #define _enable_(bits) _enable__<bits>(); _c4dbgpf("node[{}]: enable {}", m_curr->node_id, #bits)
@@ -91,11 +93,13 @@ struct EventHandlerTree : public EventHandlerStack<EventHandlerTree, EventHandle
             _reset_parser_state(m_curr, id, id);
         }
         m_curr_doc = m_tree->ancestor_doc(id);
+        m_tag_cache.clear();
     }
 
     Callbacks const& callbacks() const { return m_stack.m_callbacks; }
 
     C4_ALWAYS_INLINE TagDirectives& tag_directives() { return m_tree->m_tag_directives; } // NOLINT(readability-make-member-function-const)
+    C4_ALWAYS_INLINE TagCache &tag_cache() { return m_tag_cache; }
 
     /** @} */
 

src/c4/yml/parse_engine.def.hpp

@@ -476,14 +476,30 @@ void ParseEngine<EventHandler>::_relocate_arena(csubstr prev_arena, substr next_
     }
     _ryml_relocate(m_evt_handler->m_src);
     for(size_t i = 0; i < m_pending_tags.num_entries; ++i)
-        _ryml_relocate(m_pending_tags.annotations[i].str); // LCOV_EXCL_LINE
+    {
+        _ryml_relocate(m_pending_tags.annotations[i].str);    // LCOV_EXCL_LINE
+        _ryml_relocate(m_pending_tags.annotations[i].orig);   // LCOV_EXCL_LINE
+    }
     for(size_t i = 0; i < m_pending_anchors.num_entries; ++i)
-        _ryml_relocate(m_pending_anchors.annotations[i].str); // LCOV_EXCL_LINE
-    TagDirectives &tds = m_evt_handler->tag_directives();
-    for(size_t i = 0, sz = tds.size(); i < sz; ++i)
     {
-        _ryml_relocate(tds.m_directives[i].handle); // LCOV_EXCL_LINE
-        _ryml_relocate(tds.m_directives[i].prefix); // LCOV_EXCL_LINE
+        _ryml_relocate(m_pending_anchors.annotations[i].str);  // LCOV_EXCL_LINE
+        _ryml_relocate(m_pending_anchors.annotations[i].orig); // LCOV_EXCL_LINE
+    }
+    {
+        TagDirectives &tds = m_evt_handler->tag_directives();
+        for(size_t i = 0, sz = tds.size(); i < sz; ++i)
+        {
+            _ryml_relocate(tds.m_directives[i].handle); // LCOV_EXCL_LINE
+            _ryml_relocate(tds.m_directives[i].prefix); // LCOV_EXCL_LINE
+        }
+    }
+    {
+        TagCache &tch = m_evt_handler->tag_cache();
+        for(id_type i = 0, sz = tch.m_entries.size(); i < sz; ++i)
+        {
+            _ryml_relocate(tch.m_entries[i].tag);      // LCOV_EXCL_LINE
+            _ryml_relocate(tch.m_entries[i].resolved); // LCOV_EXCL_LINE
+        }
     }
     if(other)
         _ryml_relocate(*other); // LCOV_EXCL_LINE
@@ -4608,27 +4624,63 @@ template<class EventHandler>
 csubstr ParseEngine<EventHandler>::_resolve_tag(csubstr tag)
 {
     _c4dbgpf("resolving tag: {} curr_doc={}", _prs(tag), m_evt_handler->m_curr_doc);
+    _c4assert(tag.is_sub(_buf()));
+    TagCache::LookupResult ret = m_evt_handler->tag_cache().find(tag, m_evt_handler->m_curr_doc);
+    if(ret)
+    {
+        _c4dbgpf("resolving tag: found in cache[{}]: {}", ret.pos, _prs(ret.resolved));
+        return ret.resolved;
+    }
+    _c4dbgpf("resolving tag: not in cache: {} curr_doc={}", _prs(tag), m_evt_handler->m_curr_doc);
     size_t bufsz = 0;
     substr buf = m_evt_handler->arena_rem();
     TagDirectives const& C4_RESTRICT tds = m_evt_handler->tag_directives();
     csubstr ttag = tds.resolve(buf, &bufsz, tag, m_evt_handler->m_curr_doc,
                                m_evt_handler->m_curr->pos,
                                m_evt_handler->m_stack.m_callbacks);
-    _c4dbgpf("resolving tag: bufsz={}", bufsz);
-    if(bufsz)
+    _c4dbgpf("resolving tag: bufsz={} ttag.len={} !!ttag.str={}", bufsz, ttag.len, !!ttag.str);
+    _c4assert((bufsz > buf.len) == (!ttag.str));
+    _c4assert(!!bufsz == (ttag.len == bufsz));
+    // try again if the arena size was not enough
+    if(!ttag.str)
+    {
+        _c4dbgpf("tag requires arena, but it was small. arena.len={} arena.slack={} tag.required={}", m_evt_handler->arena_rem().len, m_evt_handler->arena().len, ttag.len);
+        _c4assert(ttag.len == bufsz);
+        buf = _alloc_arena(bufsz, &tag);
+        if(buf.str) // the alloc may fail eg with the ints handler
+        {
+            ttag = tds.resolve(buf, &bufsz, tag, m_evt_handler->m_curr_doc,
+                               m_evt_handler->m_curr->pos,
+                               m_evt_handler->m_stack.m_callbacks);
+        }
+        _c4assert(ttag.len == bufsz);
+        _c4assert(!ttag.str || ttag.is_sub(m_evt_handler->arena()));
+    }
+    else if(bufsz) // if we succeeded writing into the arena, grow it as needed
     {
-        substr bufretry = _alloc_arena(bufsz, &tag);
-        if(C4_UNLIKELY(bufsz > buf.len))
+        _c4dbgp("tag required arena. update size");
+        _c4assert(ttag.len == bufsz);
+        _c4assert(ttag.is_sub(buf));
+        (void)_alloc_arena(bufsz);
+    }
+    if C4_IF_CONSTEXPR (EventHandler::requires_strings_on_buffers) // NOLINT
+    {
+        _c4dbgpf("handler requires tags in buffers. !!ttag.str={} in_arena={} in_src={}", !!ttag.str, ttag.is_sub(m_evt_handler->arena()), ttag.is_sub(_buf()));
+        // is the resolved tag not in any of those buffers?
+        if(ttag.str && !ttag.is_sub(m_evt_handler->arena()) && !ttag.is_sub(_buf()))
         {
-            if(bufretry.str) // some handlers may be just counting the required arena size
-            {
-                ttag = tds.resolve(bufretry, &bufsz, tag, m_evt_handler->m_curr_doc,
-                                   m_evt_handler->m_curr->pos,
-                                   m_evt_handler->m_stack.m_callbacks);
-            }
+            _c4dbgpf("copying resolved tag to arena: slack={} required={}", m_evt_handler->arena_rem().len, ttag.len);
+            buf = _alloc_arena(ttag.len, &tag);
+            if(buf.str) // the alloc may fail eg with the ints handler
+                memcpy(buf.str, ttag.str, ttag.len);
+            ttag.str = buf.str; // keep the current len!
+            _c4assert(!ttag.str || ttag.is_sub(m_evt_handler->arena()));
         }
     }
-    _c4dbgpf("resolving tag: {} -->  {}", _prs(tag), _prs(ttag.str ? ttag : csubstr("(arena full)")));
+    _c4dbgpf("resolved tag: {} -->  [{}]~~~{}~~~", _prs(tag), ttag.len, ttag.str ? ttag : csubstr("(out of size)"));
+    _c4assert(ttag.len > 0);
+    // cache the hard-earned result!
+    m_evt_handler->tag_cache().add(tag, ttag, m_evt_handler->m_curr_doc, ret.pos);
     return ttag;
 }
 
@@ -8248,18 +8300,19 @@ uint32_t ParseEngine<EventHandler>::_get_annotations_same_line(csubstr token_sou
             _c4dbgpf("second annotation: {} indent={} line={}", second->str, second->indentation, second->line);
         }
         auto extract_string = [&](EntryPtr e){
-            if(e->str.begins_with_any("!<"))
+            // tags can be null when the arena ran out of space
+            if(!e->str.str || e->str.begins_with_any("!<"))
             {
                 csubstr tag = e->orig;
                 _c4assert(tag.str);
                 _c4assert(tag.len);
                 _c4assert(tag.is_sub(token_soup));
-                _c4dbgpf("tag: {} -> {}", e->str, tag);
+                _c4dbgpf("tag: {} -> {}", e->str.str ? e->str : csubstr("(out of size)"), tag);
                 return tag;
             }
             csubstr anchor = e->str;
-            _c4assert(anchor.str);
             _c4assert(anchor.len);
+            _c4assert(anchor.str);
             _c4assert(anchor.is_sub(token_soup));
             _c4assert(!anchor.begins_with('&'));
             _c4assert(anchor.str - token_soup.str > 0);

src/c4/yml/tag.cpp

@@ -532,5 +532,79 @@ csubstr TagDirectives::resolve(substr buf, size_t *bufsz, csubstr tag, id_type i
     }
 }
 
+
+//-----------------------------------------------------------------------------
+TagCache::LookupResult TagCache::find(csubstr tag, id_type doc_id, id_type linear_threshold) const noexcept
+{
+    LookupResult ret = {};
+    id_type sz = m_entries.size();
+    if(sz < linear_threshold) // do a linear search on small size
+    {
+        for(size_t i = 0; i < sz; ++i)
+        {
+            Entry const& C4_RESTRICT e = m_entries[i];
+            if(e.tag == tag && e.doc_id == doc_id)
+            {
+                ret.resolved = e.resolved;
+                ret.pos = i;
+                return ret;
+            }
+            else if(e.tag > tag || ((e.tag == tag) && e.doc_id > doc_id))
+            {
+                ret.pos = i;
+                return ret;
+            }
+        }
+        ret.pos = sz;
+    }
+    else // do a binary search on larger size
+    {
+        id_type first = 0;
+        id_type count = sz;
+        while(count)
+        {
+            id_type halfsz = count / id_type(2);
+            id_type mid = first + halfsz;
+            _RYML_ASSERT_BASIC_(m_entries.m_callbacks, mid < sz);
+            Entry const& C4_RESTRICT e = m_entries[mid];
+            if(e.tag < tag || (e.tag == tag && e.doc_id < doc_id))
+            {
+                first = mid + 1;
+                _RYML_ASSERT_BASIC_(m_entries.m_callbacks, count >= halfsz + 1);
+                count -= halfsz + 1;
+            }
+            else
+            {
+                count = halfsz;
+            }
+        }
+        ret.pos = first;
+        if(first < sz)
+        {
+            Entry const& C4_RESTRICT e = m_entries[first];
+            if(e.tag == tag && e.doc_id == doc_id)
+            {
+                ret.resolved = m_entries[first].resolved;
+            }
+        }
+    }
+    return ret;
+}
+
+void TagCache::add(csubstr tag, csubstr resolved, id_type doc_id, const_iterator pos)
+{
+    id_type sz = m_entries.size();
+    _RYML_ASSERT_BASIC_(m_entries.m_callbacks, pos <= sz);
+    _RYML_ASSERT_BASIC_(m_entries.m_callbacks, pos == sz || tag < m_entries[pos].tag || (tag == m_entries[pos].tag && doc_id < m_entries[pos].doc_id));
+    m_entries.resize(sz + 1);
+    Entry *C4_RESTRICT ptr = m_entries.m_stack;
+    if(pos < sz)
+        memmove(ptr + pos + 1, ptr + pos, (sz - pos) * sizeof(Entry));
+    ptr[pos].tag = tag;
+    ptr[pos].resolved = resolved;
+    ptr[pos].doc_id = doc_id;
+    _c4dbgpf("tagcache: add entry @pos={}:  docid={}  {}->{}", pos, doc_id, tag, (resolved.str ? resolved : csubstr("(out of size)")));
+}
+
 } // namespace yml
 } // namespace c4

src/c4/yml/tag.hpp

@@ -1,7 +1,12 @@
 #ifndef _C4_YML_TAG_HPP_
 #define _C4_YML_TAG_HPP_
 
+#ifndef _C4_YML_COMMON_HPP_
 #include <c4/yml/common.hpp>
+#endif
+#ifndef _C4_YML_DETAIL_STACK_HPP_
+#include <c4/yml/detail/stack.hpp>
+#endif
 
 namespace c4 {
 namespace yml {
@@ -55,6 +60,47 @@ RYML_EXPORT csubstr normalize_tag_long(csubstr tag, substr output);
 RYML_EXPORT bool is_custom_tag(csubstr tag);
 RYML_EXPORT bool is_valid_tag_handle(csubstr handle);
 
+
+//-----------------------------------------------------------------------------
+
+/** Accelerator structure to reduce memory requirements by enabling
+ * reuse of resolved tags. */
+struct RYML_EXPORT TagCache
+{
+    struct Entry
+    {
+        csubstr tag;
+        csubstr resolved;
+        id_type doc_id;
+    };
+    using Entries = detail::stack<Entry>;
+    using const_iterator = id_type;
+    struct LookupResult
+    {
+        csubstr resolved;
+        const_iterator pos;
+        operator bool() const noexcept { return resolved.len > 0; }
+    };
+
+public:
+
+    TagCache() noexcept : m_entries() {}
+    LookupResult find(csubstr tag, id_type doc_id, id_type linear_threshold=Entries::sso_size) const noexcept;
+    void add(csubstr tag, csubstr resolved, id_type doc_id, const_iterator pos);
+
+    void clear() { m_entries.clear(); }
+
+public:
+
+    /** @cond dev */
+    Entries m_entries;
+    /** @endcond */
+
+};
+
+
+//-----------------------------------------------------------------------------
+
 struct RYML_EXPORT TagDirective
 {
     /** Eg <pre>!e!</pre> in <pre>%TAG !e! tag:example.com,2000:app/</pre> */