diff options
Diffstat (limited to 'immer/detail/rbts')
| -rw-r--r-- | immer/detail/rbts/bits.hpp | 33 | ||||
| -rw-r--r-- | immer/detail/rbts/node.hpp | 932 | ||||
| -rw-r--r-- | immer/detail/rbts/operations.hpp | 2461 | ||||
| -rw-r--r-- | immer/detail/rbts/position.hpp | 1977 | ||||
| -rw-r--r-- | immer/detail/rbts/rbtree.hpp | 509 | ||||
| -rw-r--r-- | immer/detail/rbts/rbtree_iterator.hpp | 99 | ||||
| -rw-r--r-- | immer/detail/rbts/rrbtree.hpp | 1396 | ||||
| -rw-r--r-- | immer/detail/rbts/rrbtree_iterator.hpp | 98 | ||||
| -rw-r--r-- | immer/detail/rbts/visitor.hpp | 56 |
9 files changed, 7561 insertions, 0 deletions
diff --git a/immer/detail/rbts/bits.hpp b/immer/detail/rbts/bits.hpp new file mode 100644 index 000000000000..58d4e3c9c93a --- /dev/null +++ b/immer/detail/rbts/bits.hpp @@ -0,0 +1,33 @@ +// +// immer: immutable data structures for C++ +// Copyright (C) 2016, 2017, 2018 Juan Pedro Bolivar Puente +// +// This software is distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE or copy at http://boost.org/LICENSE_1_0.txt +// + +#pragma once + +#include <cstdint> + +namespace immer { +namespace detail { +namespace rbts { + +using bits_t = std::uint32_t; +using shift_t = std::uint32_t; +using count_t = std::uint32_t; +using size_t = std::size_t; + +template <bits_t B, typename T = count_t> +constexpr T branches = T{1} << B; + +template <bits_t B, typename T = size_t> +constexpr T mask = branches<B, T> - 1; + +template <bits_t B, bits_t BL> +constexpr shift_t endshift = shift_t{BL} - shift_t{B}; + +} // namespace rbts +} // namespace detail +} // namespace immer diff --git a/immer/detail/rbts/node.hpp b/immer/detail/rbts/node.hpp new file mode 100644 index 000000000000..e54e569636ac --- /dev/null +++ b/immer/detail/rbts/node.hpp @@ -0,0 +1,932 @@ +// +// immer: immutable data structures for C++ +// Copyright (C) 2016, 2017, 2018 Juan Pedro Bolivar Puente +// +// This software is distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE or copy at http://boost.org/LICENSE_1_0.txt +// + +#pragma once + +#include <immer/detail/combine_standard_layout.hpp> +#include <immer/detail/rbts/bits.hpp> +#include <immer/detail/util.hpp> +#include <immer/heap/tags.hpp> + +#include <cassert> +#include <cstddef> +#include <memory> +#include <type_traits> + +namespace immer { +namespace detail { +namespace rbts { + +template <typename T, typename MemoryPolicy, bits_t B, bits_t BL> +struct node +{ + static constexpr auto bits = B; + static constexpr auto bits_leaf = BL; + + using node_t = node; + using memory = MemoryPolicy; + using heap_policy = typename memory::heap; + using transience = typename memory::transience_t; + using refs_t = typename memory::refcount; + using ownee_t = typename transience::ownee; + using edit_t = typename transience::edit; + using value_t = T; + + static constexpr bool embed_relaxed = memory::prefer_fewer_bigger_objects; + + enum class kind_t + { + leaf, + inner + }; + + struct relaxed_data_t + { + count_t count; + size_t sizes[branches<B>]; + }; + + using relaxed_data_with_meta_t = + combine_standard_layout_t<relaxed_data_t, refs_t, ownee_t>; + + using relaxed_data_no_meta_t = combine_standard_layout_t<relaxed_data_t>; + + using relaxed_t = std::conditional_t<embed_relaxed, + relaxed_data_no_meta_t, + relaxed_data_with_meta_t>; + + struct leaf_t + { + aligned_storage_for<T> buffer; + }; + + struct inner_t + { + relaxed_t* relaxed; + aligned_storage_for<node_t*> buffer; + }; + + union data_t + { + inner_t inner; + leaf_t leaf; + }; + + struct impl_data_t + { +#if IMMER_TAGGED_NODE + kind_t kind; +#endif + data_t data; + }; + + using impl_t = combine_standard_layout_t<impl_data_t, refs_t, ownee_t>; + + impl_t impl; + + // assume that we need to keep headroom space in the node when we + // are doing reference counting, since any node may become + // transient when it has only one reference + constexpr static bool keep_headroom = !std::is_empty<refs_t>{}; + + constexpr static std::size_t sizeof_packed_leaf_n(count_t count) + { + return immer_offsetof(impl_t, d.data.leaf.buffer) + + sizeof(leaf_t::buffer) * count; + } + + constexpr static std::size_t sizeof_packed_inner_n(count_t count) + { + return immer_offsetof(impl_t, d.data.inner.buffer) + + sizeof(inner_t::buffer) * count; + } + + constexpr static std::size_t sizeof_packed_relaxed_n(count_t count) + { + return immer_offsetof(relaxed_t, d.sizes) + sizeof(size_t) * count; + } + + constexpr static std::size_t sizeof_packed_inner_r_n(count_t count) + { + return embed_relaxed ? sizeof_packed_inner_n(count) + + sizeof_packed_relaxed_n(count) + : sizeof_packed_inner_n(count); + } + + constexpr static std::size_t max_sizeof_leaf = + sizeof_packed_leaf_n(branches<BL>); + + constexpr static std::size_t max_sizeof_inner = + sizeof_packed_inner_n(branches<B>); + + constexpr static std::size_t max_sizeof_relaxed = + sizeof_packed_relaxed_n(branches<B>); + + constexpr static std::size_t max_sizeof_inner_r = + sizeof_packed_inner_r_n(branches<B>); + + constexpr static std::size_t sizeof_inner_n(count_t n) + { + return keep_headroom ? max_sizeof_inner : sizeof_packed_inner_n(n); + } + + constexpr static std::size_t sizeof_inner_r_n(count_t n) + { + return keep_headroom ? max_sizeof_inner_r : sizeof_packed_inner_r_n(n); + } + + constexpr static std::size_t sizeof_relaxed_n(count_t n) + { + return keep_headroom ? max_sizeof_relaxed : sizeof_packed_relaxed_n(n); + } + + constexpr static std::size_t sizeof_leaf_n(count_t n) + { + return keep_headroom ? max_sizeof_leaf : sizeof_packed_leaf_n(n); + } + + using heap = + typename heap_policy::template optimized<max_sizeof_inner>::type; + +#if IMMER_TAGGED_NODE + kind_t kind() const { return impl.d.kind; } +#endif + + relaxed_t* relaxed() + { + IMMER_ASSERT_TAGGED(kind() == kind_t::inner); + return impl.d.data.inner.relaxed; + } + + const relaxed_t* relaxed() const + { + IMMER_ASSERT_TAGGED(kind() == kind_t::inner); + return impl.d.data.inner.relaxed; + } + + node_t** inner() + { + IMMER_ASSERT_TAGGED(kind() == kind_t::inner); + return reinterpret_cast<node_t**>(&impl.d.data.inner.buffer); + } + + T* leaf() + { + IMMER_ASSERT_TAGGED(kind() == kind_t::leaf); + return reinterpret_cast<T*>(&impl.d.data.leaf.buffer); + } + + static refs_t& refs(const relaxed_t* x) + { + return auto_const_cast(get<refs_t>(*x)); + } + static const ownee_t& ownee(const relaxed_t* x) { return get<ownee_t>(*x); } + static ownee_t& ownee(relaxed_t* x) { return get<ownee_t>(*x); } + + static refs_t& refs(const node_t* x) + { + return auto_const_cast(get<refs_t>(x->impl)); + } + static const ownee_t& ownee(const node_t* x) + { + return get<ownee_t>(x->impl); + } + static ownee_t& ownee(node_t* x) { return get<ownee_t>(x->impl); } + + static node_t* make_inner_n(count_t n) + { + assert(n <= branches<B>); + auto m = heap::allocate(sizeof_inner_n(n)); + auto p = new (m) node_t; + p->impl.d.data.inner.relaxed = nullptr; +#if IMMER_TAGGED_NODE + p->impl.d.kind = node_t::kind_t::inner; +#endif + return p; + } + + static node_t* make_inner_e(edit_t e) + { + auto m = heap::allocate(max_sizeof_inner); + auto p = new (m) node_t; + ownee(p) = e; + p->impl.d.data.inner.relaxed = nullptr; +#if IMMER_TAGGED_NODE + p->impl.d.kind = node_t::kind_t::inner; +#endif + return p; + } + + static node_t* make_inner_r_n(count_t n) + { + assert(n <= branches<B>); + auto mp = heap::allocate(sizeof_inner_r_n(n)); + auto mr = static_cast<void*>(nullptr); + if (embed_relaxed) { + mr = reinterpret_cast<unsigned char*>(mp) + sizeof_inner_n(n); + } else { + try { + mr = heap::allocate(sizeof_relaxed_n(n), norefs_tag{}); + } catch (...) { + heap::deallocate(sizeof_inner_r_n(n), mp); + throw; + } + } + auto p = new (mp) node_t; + auto r = new (mr) relaxed_t; + r->d.count = 0; + p->impl.d.data.inner.relaxed = r; +#if IMMER_TAGGED_NODE + p->impl.d.kind = node_t::kind_t::inner; +#endif + return p; + } + + static node_t* make_inner_sr_n(count_t n, relaxed_t* r) + { + return static_if<embed_relaxed, node_t*>( + [&](auto) { return node_t::make_inner_r_n(n); }, + [&](auto) { + auto p = + new (heap::allocate(node_t::sizeof_inner_r_n(n))) node_t; + assert(r->d.count >= n); + node_t::refs(r).inc(); + p->impl.d.data.inner.relaxed = r; +#if IMMER_TAGGED_NODE + p->impl.d.kind = node_t::kind_t::inner; +#endif + return p; + }); + } + + static node_t* make_inner_r_e(edit_t e) + { + auto mp = heap::allocate(max_sizeof_inner_r); + auto mr = static_cast<void*>(nullptr); + if (embed_relaxed) { + mr = reinterpret_cast<unsigned char*>(mp) + max_sizeof_inner; + } else { + try { + mr = heap::allocate(max_sizeof_relaxed, norefs_tag{}); + } catch (...) { + heap::deallocate(max_sizeof_inner_r, mp); + throw; + } + } + auto p = new (mp) node_t; + auto r = new (mr) relaxed_t; + ownee(p) = e; + static_if<!embed_relaxed>([&](auto) { node_t::ownee(r) = e; }); + r->d.count = 0; + p->impl.d.data.inner.relaxed = r; +#if IMMER_TAGGED_NODE + p->impl.d.kind = node_t::kind_t::inner; +#endif + return p; + } + + static node_t* make_inner_sr_e(edit_t e, relaxed_t* r) + { + return static_if<embed_relaxed, node_t*>( + [&](auto) { return node_t::make_inner_r_e(e); }, + [&](auto) { + auto p = + new (heap::allocate(node_t::max_sizeof_inner_r)) node_t; + node_t::refs(r).inc(); + p->impl.d.data.inner.relaxed = r; + node_t::ownee(p) = e; +#if IMMER_TAGGED_NODE + p->impl.d.kind = node_t::kind_t::inner; +#endif + return p; + }); + } + + static node_t* make_leaf_n(count_t n) + { + assert(n <= branches<BL>); + auto p = new (heap::allocate(sizeof_leaf_n(n))) node_t; +#if IMMER_TAGGED_NODE + p->impl.d.kind = node_t::kind_t::leaf; +#endif + return p; + } + + static node_t* make_leaf_e(edit_t e) + { + auto p = new (heap::allocate(max_sizeof_leaf)) node_t; + ownee(p) = e; +#if IMMER_TAGGED_NODE + p->impl.d.kind = node_t::kind_t::leaf; +#endif + return p; + } + + static node_t* make_inner_n(count_t n, node_t* x) + { + assert(n >= 1); + auto p = make_inner_n(n); + p->inner()[0] = x; + return p; + } + + static node_t* make_inner_n(edit_t n, node_t* x) + { + assert(n >= 1); + auto p = make_inner_n(n); + p->inner()[0] = x; + return p; + } + + static node_t* make_inner_n(count_t n, node_t* x, node_t* y) + { + assert(n >= 2); + auto p = make_inner_n(n); + p->inner()[0] = x; + p->inner()[1] = y; + return p; + } + + static node_t* make_inner_r_n(count_t n, node_t* x) + { + assert(n >= 1); + auto p = make_inner_r_n(n); + auto r = p->relaxed(); + p->inner()[0] = x; + r->d.count = 1; + return p; + } + + static node_t* make_inner_r_n(count_t n, node_t* x, size_t xs) + { + assert(n >= 1); + auto p = make_inner_r_n(n); + auto r = p->relaxed(); + p->inner()[0] = x; + r->d.sizes[0] = xs; + r->d.count = 1; + return p; + } + + static node_t* make_inner_r_n(count_t n, node_t* x, node_t* y) + { + assert(n >= 2); + auto p = make_inner_r_n(n); + auto r = p->relaxed(); + p->inner()[0] = x; + p->inner()[1] = y; + r->d.count = 2; + return p; + } + + static node_t* make_inner_r_n(count_t n, node_t* x, size_t xs, node_t* y) + { + assert(n >= 2); + auto p = make_inner_r_n(n); + auto r = p->relaxed(); + p->inner()[0] = x; + p->inner()[1] = y; + r->d.sizes[0] = xs; + r->d.count = 2; + return p; + } + + static node_t* + make_inner_r_n(count_t n, node_t* x, size_t xs, node_t* y, size_t ys) + { + assert(n >= 2); + auto p = make_inner_r_n(n); + auto r = p->relaxed(); + p->inner()[0] = x; + p->inner()[1] = y; + r->d.sizes[0] = xs; + r->d.sizes[1] = xs + ys; + r->d.count = 2; + return p; + } + + static node_t* make_inner_r_n(count_t n, + node_t* x, + size_t xs, + node_t* y, + size_t ys, + node_t* z, + size_t zs) + { + assert(n >= 3); + auto p = make_inner_r_n(n); + auto r = p->relaxed(); + p->inner()[0] = x; + p->inner()[1] = y; + p->inner()[2] = z; + r->d.sizes[0] = xs; + r->d.sizes[1] = xs + ys; + r->d.sizes[2] = xs + ys + zs; + r->d.count = 3; + return p; + } + + template <typename U> + static node_t* make_leaf_n(count_t n, U&& x) + { + assert(n >= 1); + auto p = make_leaf_n(n); + try { + new (p->leaf()) T{std::forward<U>(x)}; + } catch (...) { + heap::deallocate(node_t::sizeof_leaf_n(n), p); + throw; + } + return p; + } + + template <typename U> + static node_t* make_leaf_e(edit_t e, U&& x) + { + auto p = make_leaf_e(e); + try { + new (p->leaf()) T{std::forward<U>(x)}; + } catch (...) { + heap::deallocate(node_t::max_sizeof_leaf, p); + throw; + } + return p; + } + + static node_t* make_path(shift_t shift, node_t* node) + { + IMMER_ASSERT_TAGGED(node->kind() == kind_t::leaf); + if (shift == endshift<B, BL>) + return node; + else { + auto n = node_t::make_inner_n(1); + try { + n->inner()[0] = make_path(shift - B, node); + } catch (...) { + heap::deallocate(node_t::sizeof_inner_n(1), n); + throw; + } + return n; + } + } + + static node_t* make_path_e(edit_t e, shift_t shift, node_t* node) + { + IMMER_ASSERT_TAGGED(node->kind() == kind_t::leaf); + if (shift == endshift<B, BL>) + return node; + else { + auto n = node_t::make_inner_e(e); + try { + n->inner()[0] = make_path_e(e, shift - B, node); + } catch (...) { + heap::deallocate(node_t::max_sizeof_inner, n); + throw; + } + return n; + } + } + + static node_t* copy_inner(node_t* src, count_t n) + { + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + auto dst = make_inner_n(n); + inc_nodes(src->inner(), n); + std::uninitialized_copy(src->inner(), src->inner() + n, dst->inner()); + return dst; + } + + static node_t* copy_inner_n(count_t allocn, node_t* src, count_t n) + { + assert(allocn >= n); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + auto dst = make_inner_n(allocn); + return do_copy_inner(dst, src, n); + } + + static node_t* copy_inner_e(edit_t e, node_t* src, count_t n) + { + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + auto dst = make_inner_e(e); + return do_copy_inner(dst, src, n); + } + + static node_t* do_copy_inner(node_t* dst, node_t* src, count_t n) + { + IMMER_ASSERT_TAGGED(dst->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + auto p = src->inner(); + inc_nodes(p, n); + std::uninitialized_copy(p, p + n, dst->inner()); + return dst; + } + + static node_t* copy_inner_r(node_t* src, count_t n) + { + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + auto dst = make_inner_r_n(n); + return do_copy_inner_r(dst, src, n); + } + + static node_t* copy_inner_r_n(count_t allocn, node_t* src, count_t n) + { + assert(allocn >= n); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + auto dst = make_inner_r_n(allocn); + return do_copy_inner_r(dst, src, n); + } + + static node_t* copy_inner_r_e(edit_t e, node_t* src, count_t n) + { + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + auto dst = make_inner_r_e(e); + return do_copy_inner_r(dst, src, n); + } + + static node_t* copy_inner_sr_e(edit_t e, node_t* src, count_t n) + { + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + auto dst = make_inner_sr_e(e, src->relaxed()); + return do_copy_inner_sr(dst, src, n); + } + + static node_t* do_copy_inner_r(node_t* dst, node_t* src, count_t n) + { + IMMER_ASSERT_TAGGED(dst->kind() == kind_t::inner); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::inner); + auto src_r = src->relaxed(); + auto dst_r = dst->relaxed(); + inc_nodes(src->inner(), n); + std::copy(src->inner(), src->inner() + n, dst->inner()); + std::copy(src_r->d.sizes, src_r->d.sizes + n, dst_r->d.sizes); + dst_r->d.count = n; + return dst; + } + + static node_t* do_copy_inner_sr(node_t* dst, node_t* src, count_t n) + { + if (embed_relaxed) + return do_copy_inner_r(dst, src, n); + else { + inc_nodes(src->inner(), n); + std::copy(src->inner(), src->inner() + n, dst->inner()); + return dst; + } + } + + static node_t* copy_leaf(node_t* src, count_t n) + { + IMMER_ASSERT_TAGGED(src->kind() == kind_t::leaf); + auto dst = make_leaf_n(n); + try { + std::uninitialized_copy(src->leaf(), src->leaf() + n, dst->leaf()); + } catch (...) { + heap::deallocate(node_t::sizeof_leaf_n(n), dst); + throw; + } + return dst; + } + + static node_t* copy_leaf_e(edit_t e, node_t* src, count_t n) + { + IMMER_ASSERT_TAGGED(src->kind() == kind_t::leaf); + auto dst = make_leaf_e(e); + try { + std::uninitialized_copy(src->leaf(), src->leaf() + n, dst->leaf()); + } catch (...) { + heap::deallocate(node_t::max_sizeof_leaf, dst); + throw; + } + return dst; + } + + static node_t* copy_leaf_n(count_t allocn, node_t* src, count_t n) + { + assert(allocn >= n); + IMMER_ASSERT_TAGGED(src->kind() == kind_t::leaf); + auto dst = make_leaf_n(allocn); + try { + std::uninitialized_copy(src->leaf(), src->leaf() + n, dst->leaf()); + } catch (...) { + heap::deallocate(node_t::sizeof_leaf_n(allocn), dst); + throw; + } + return dst; + } + + static node_t* copy_leaf(node_t* src1, count_t n1, node_t* src2, count_t n2) + { + IMMER_ASSERT_TAGGED(src1->kind() == kind_t::leaf); + IMMER_ASSERT_TAGGED(src2->kind() == kind_t::leaf); + auto dst = make_leaf_n(n1 + n2); + try { + std::uninitialized_copy( + src1->leaf(), src1->leaf() + n1, dst->leaf()); + } catch (...) { + heap::deallocate(node_t::sizeof_leaf_n(n1 + n2), dst); + throw; + } + try { + std::uninitialized_copy( + src2->leaf(), src2->leaf() + n2, dst->leaf() + n1); + } catch (...) { + destroy_n(dst->leaf(), n1); + heap::deallocate(node_t::sizeof_leaf_n(n1 + n2), dst); + throw; + } + return dst; + } + + static node_t* + copy_leaf_e(edit_t e, node_t* src1, count_t n1, node_t* src2, count_t n2) + { + IMMER_ASSERT_TAGGED(src1->kind() == kind_t::leaf); + IMMER_ASSERT_TAGGED(src2->kind() == kind_t::leaf); + auto dst = make_leaf_e(e); + try { + std::uninitialized_copy( + src1->leaf(), src1->leaf() + n1, dst->leaf()); + } catch (...) { + heap::deallocate(max_sizeof_leaf, dst); + throw; + } + try { + std::uninitialized_copy( + src2->leaf(), src2->leaf() + n2, dst->leaf() + n1); + } catch (...) { + destroy_n(dst->leaf(), n1); + heap::deallocate(max_sizeof_leaf, dst); + throw; + } + return dst; + } + + static node_t* copy_leaf_e(edit_t e, node_t* src, count_t idx, count_t last) + { + IMMER_ASSERT_TAGGED(src->kind() == kind_t::leaf); + auto dst = make_leaf_e(e); + try { + std::uninitialized_copy( + src->leaf() + idx, src->leaf() + last, dst->leaf()); + } catch (...) { + heap::deallocate(max_sizeof_leaf, dst); + throw; + } + return dst; + } + + static node_t* copy_leaf(node_t* src, count_t idx, count_t last) + { + IMMER_ASSERT_TAGGED(src->kind() == kind_t::leaf); + auto dst = make_leaf_n(last - idx); + try { + std::uninitialized_copy( + src->leaf() + idx, src->leaf() + last, dst->leaf()); + } catch (...) { + heap::deallocate(node_t::sizeof_leaf_n(last - idx), dst); + throw; + } + return dst; + } + + template <typename U> + static node_t* copy_leaf_emplace(node_t* src, count_t n, U&& x) + { + auto dst = copy_leaf_n(n + 1, src, n); + try { + new (dst->leaf() + n) T{std::forward<U>(x)}; + } catch (...) { + destroy_n(dst->leaf(), n); + heap::deallocate(node_t::sizeof_leaf_n(n + 1), dst); + throw; + } + return dst; + } + + static void delete_inner(node_t* p, count_t n) + { + IMMER_ASSERT_TAGGED(p->kind() == kind_t::inner); + assert(!p->relaxed()); + heap::deallocate(ownee(p).owned() ? node_t::max_sizeof_inner + : node_t::sizeof_inner_n(n), + p); + } + + static void delete_inner_e(node_t* p) + { + IMMER_ASSERT_TAGGED(p->kind() == kind_t::inner); + assert(!p->relaxed()); + heap::deallocate(node_t::max_sizeof_inner, p); + } + + static void delete_inner_any(node_t* p, count_t n) + { + if (p->relaxed()) + delete_inner_r(p, n); + else + delete_inner(p, n); + } + + static void delete_inner_r(node_t* p, count_t n) + { + IMMER_ASSERT_TAGGED(p->kind() == kind_t::inner); + auto r = p->relaxed(); + assert(r); + static_if<!embed_relaxed>([&](auto) { + if (node_t::refs(r).dec()) + heap::deallocate(node_t::ownee(r).owned() + ? node_t::max_sizeof_relaxed + : node_t::sizeof_relaxed_n(n), + r); + }); + heap::deallocate(ownee(p).owned() ? node_t::max_sizeof_inner_r + : node_t::sizeof_inner_r_n(n), + p); + } + + static void delete_inner_r_e(node_t* p) + { + IMMER_ASSERT_TAGGED(p->kind() == kind_t::inner); + auto r = p->relaxed(); + assert(r); + static_if<!embed_relaxed>([&](auto) { + if (node_t::refs(r).dec()) + heap::deallocate(node_t::max_sizeof_relaxed, r); + }); + heap::deallocate(node_t::max_sizeof_inner_r, p); + } + + static void delete_leaf(node_t* p, count_t n) + { + IMMER_ASSERT_TAGGED(p->kind() == kind_t::leaf); + destroy_n(p->leaf(), n); + heap::deallocate(ownee(p).owned() ? node_t::max_sizeof_leaf + : node_t::sizeof_leaf_n(n), + p); + } + + bool can_mutate(edit_t e) const + { + return refs(this).unique() || ownee(this).can_mutate(e); + } + + bool can_relax() const { return !embed_relaxed || relaxed(); } + + relaxed_t* ensure_mutable_relaxed(edit_t e) + { + auto src_r = relaxed(); + return static_if<embed_relaxed, relaxed_t*>( + [&](auto) { return src_r; }, + [&](auto) { + if (node_t::refs(src_r).unique() || + node_t::ownee(src_r).can_mutate(e)) + return src_r; + else { + if (src_r) + node_t::refs(src_r).dec_unsafe(); + auto dst_r = impl.d.data.inner.relaxed = + new (heap::allocate(max_sizeof_relaxed)) relaxed_t; + node_t::ownee(dst_r) = e; + return dst_r; + } + }); + } + + relaxed_t* ensure_mutable_relaxed_e(edit_t e, edit_t ec) + { + auto src_r = relaxed(); + return static_if<embed_relaxed, relaxed_t*>( + [&](auto) { return src_r; }, + [&](auto) { + if (src_r && (node_t::refs(src_r).unique() || + node_t::ownee(src_r).can_mutate(e))) { + node_t::ownee(src_r) = ec; + return src_r; + } else { + if (src_r) + node_t::refs(src_r).dec_unsafe(); + auto dst_r = impl.d.data.inner.relaxed = + new (heap::allocate(max_sizeof_relaxed)) relaxed_t; + node_t::ownee(dst_r) = ec; + return dst_r; + } + }); + } + + relaxed_t* ensure_mutable_relaxed_n(edit_t e, count_t n) + { + auto src_r = relaxed(); + return static_if<embed_relaxed, relaxed_t*>( + [&](auto) { return src_r; }, + [&](auto) { + if (node_t::refs(src_r).unique() || + node_t::ownee(src_r).can_mutate(e)) + return src_r; + else { + if (src_r) + node_t::refs(src_r).dec_unsafe(); + auto dst_r = + new (heap::allocate(max_sizeof_relaxed)) relaxed_t; + std::copy( + src_r->d.sizes, src_r->d.sizes + n, dst_r->d.sizes); + node_t::ownee(dst_r) = e; + return impl.d.data.inner.relaxed = dst_r; + } + }); + } + + node_t* inc() + { + refs(this).inc(); + return this; + } + + const node_t* inc() const + { + refs(this).inc(); + return this; + } + + bool dec() const { return refs(this).dec(); } + void dec_unsafe() const { refs(this).dec_unsafe(); } + + static void inc_nodes(node_t** p, count_t n) + { + for (auto i = p, e = i + n; i != e; ++i) + refs(*i).inc(); + } + +#if IMMER_TAGGED_NODE + shift_t compute_shift() + { + if (kind() == kind_t::leaf) + return endshift<B, BL>; + else + return B + inner()[0]->compute_shift(); + } +#endif + + bool check(shift_t shift, size_t size) + { +#if IMMER_DEBUG_DEEP_CHECK + assert(size > 0); + if (shift == endshift<B, BL>) { + IMMER_ASSERT_TAGGED(kind() == kind_t::leaf); + assert(size <= branches<BL>); + } else if (auto r = relaxed()) { + auto count = r->d.count; + assert(count > 0); + assert(count <= branches<B>); + if (r->d.sizes[count - 1] != size) { + IMMER_TRACE_F("check"); + IMMER_TRACE_E(r->d.sizes[count - 1]); + IMMER_TRACE_E(size); + } + assert(r->d.sizes[count - 1] == size); + for (auto i = 1; i < count; ++i) + assert(r->d.sizes[i - 1] < r->d.sizes[i]); + auto last_size = size_t{}; + for (auto i = 0; i < count; ++i) { + assert(inner()[i]->check(shift - B, r->d.sizes[i] - last_size)); + last_size = r->d.sizes[i]; + } + } else { + assert(size <= branches<B> << shift); + auto count = + (size >> shift) + (size - ((size >> shift) << shift) > 0); + assert(count <= branches<B>); + if (count) { + for (auto i = 1; i < count - 1; ++i) + assert(inner()[i]->check(shift - B, 1 << shift)); + assert(inner()[count - 1]->check( + shift - B, size - ((count - 1) << shift))); + } + } +#endif // IMMER_DEBUG_DEEP_CHECK + return true; + } +}; + +template <typename T, typename MP, bits_t B> +constexpr bits_t derive_bits_leaf_aux() +{ + using node_t = node<T, MP, B, B>; + constexpr auto sizeof_elem = sizeof(T); + constexpr auto space = + node_t::max_sizeof_inner - node_t::sizeof_packed_leaf_n(0); + constexpr auto full_elems = space / sizeof_elem; + constexpr auto BL = log2(full_elems); + return BL; +} + +template <typename T, typename MP, bits_t B> +constexpr bits_t derive_bits_leaf = derive_bits_leaf_aux<T, MP, B>(); + +} // namespace rbts +} // namespace detail +} // namespace immer diff --git a/immer/detail/rbts/operations.hpp b/immer/detail/rbts/operations.hpp new file mode 100644 index 000000000000..ff703e892b42 --- /dev/null +++ b/immer/detail/rbts/operations.hpp @@ -0,0 +1,2461 @@ +// +// immer: immutable data structures for C++ +// Copyright (C) 2016, 2017, 2018 Juan Pedro Bolivar Puente +// +// This software is distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE or copy at http://boost.org/LICENSE_1_0.txt +// + +#pragma once + +#include <algorithm> +#include <memory> +#include <numeric> +#include <utility> + +#include <immer/config.hpp> +#include <immer/detail/rbts/position.hpp> +#include <immer/detail/rbts/visitor.hpp> +#include <immer/detail/util.hpp> +#include <immer/heap/tags.hpp> + +namespace immer { +namespace detail { +namespace rbts { + +template <typename T> +struct array_for_visitor : visitor_base<array_for_visitor<T>> +{ + using this_t = array_for_visitor; + + template <typename PosT> + static T* visit_inner(PosT&& pos, size_t idx) + { + return pos.descend(this_t{}, idx); + } + + template <typename PosT> + static T* visit_leaf(PosT&& pos, size_t) + { + return pos.node()->leaf(); + } +}; + +template <typename T> +struct region_for_visitor : visitor_base<region_for_visitor<T>> +{ + using this_t = region_for_visitor; + using result_t = std::tuple<T*, size_t, size_t>; + + template <typename PosT> + static result_t visit_inner(PosT&& pos, size_t idx) + { + return pos.towards(this_t{}, idx); + } + + template <typename PosT> + static result_t visit_leaf(PosT&& pos, size_t idx) + { + return std::make_tuple(pos.node()->leaf(), pos.index(idx), pos.count()); + } +}; + +template <typename T> +struct get_visitor : visitor_base<get_visitor<T>> +{ + using this_t = get_visitor; + + template <typename PosT> + static const T& visit_inner(PosT&& pos, size_t idx) + { + return pos.descend(this_t{}, idx); + } + + template <typename PosT> + static const T& visit_leaf(PosT&& pos, size_t idx) + { + return pos.node()->leaf()[pos.index(idx)]; + } +}; + +struct for_each_chunk_visitor : visitor_base<for_each_chunk_visitor> +{ + using this_t = for_each_chunk_visitor; + + template <typename Pos, typename Fn> + static void visit_inner(Pos&& pos, Fn&& fn) + { + pos.each(this_t{}, fn); + } + + template <typename Pos, typename Fn> + static void visit_leaf(Pos&& pos, Fn&& fn) + { + auto data = pos.node()->leaf(); + fn(data, data + pos.count()); + } +}; + +struct for_each_chunk_p_visitor : visitor_base<for_each_chunk_p_visitor> +{ + using this_t = for_each_chunk_p_visitor; + + template <typename Pos, typename Fn> + static bool visit_inner(Pos&& pos, Fn&& fn) + { + return pos.each_pred(this_t{}, fn); + } + + template <typename Pos, typename Fn> + static bool visit_leaf(Pos&& pos, Fn&& fn) + { + auto data = pos.node()->leaf(); + return fn(data, data + pos.count()); + } +}; + +struct for_each_chunk_left_visitor : visitor_base<for_each_chunk_left_visitor> +{ + using this_t = for_each_chunk_left_visitor; + + template <typename Pos, typename Fn> + static void visit_inner(Pos&& pos, size_t last, Fn&& fn) + { + auto l = pos.index(last); + pos.each_left(for_each_chunk_visitor{}, l, fn); + pos.towards_oh(this_t{}, last, l, fn); + } + + template <typename Pos, typename Fn> + static void visit_leaf(Pos&& pos, size_t last, Fn&& fn) + { + auto data = pos.node()->leaf(); + auto l = pos.index(last); + fn(data, data + l + 1); + } +}; + +struct for_each_chunk_right_visitor : visitor_base<for_each_chunk_right_visitor> +{ + using this_t = for_each_chunk_right_visitor; + + template <typename Pos, typename Fn> + static void visit_inner(Pos&& pos, size_t first, Fn&& fn) + { + auto f = pos.index(first); + pos.towards_oh(this_t{}, first, f, fn); + pos.each_right(for_each_chunk_visitor{}, f + 1, fn); + } + + template <typename Pos, typename Fn> + static void visit_leaf(Pos&& pos, size_t first, Fn&& fn) + { + auto data = pos.node()->leaf(); + auto f = pos.index(first); + fn(data + f, data + pos.count()); + } +}; + +struct for_each_chunk_i_visitor : visitor_base<for_each_chunk_i_visitor> +{ + using this_t = for_each_chunk_i_visitor; + + template <typename Pos, typename Fn> + static void visit_relaxed(Pos&& pos, size_t first, size_t last, Fn&& fn) + { + // we are going towards *two* indices, so we need to do the + // relaxed as a special case to correct the second index + if (first < last) { + auto f = pos.index(first); + auto l = pos.index(last - 1); + if (f == l) { + auto sbh = pos.size_before(f); + pos.towards_oh_sbh(this_t{}, first, f, sbh, last - sbh, fn); + } else { + assert(f < l); + pos.towards_oh(for_each_chunk_right_visitor{}, first, f, fn); + pos.each_i(for_each_chunk_visitor{}, f + 1, l, fn); + pos.towards_oh(for_each_chunk_left_visitor{}, last - 1, l, fn); + } + } + } + + template <typename Pos, typename Fn> + static void visit_regular(Pos&& pos, size_t first, size_t last, Fn&& fn) + { + if (first < last) { + auto f = pos.index(first); + auto l = pos.index(last - 1); + if (f == l) + pos.towards_oh(this_t{}, first, f, last, fn); + else { + assert(f < l); + pos.towards_oh(for_each_chunk_right_visitor{}, first, f, fn); + pos.each_i(for_each_chunk_visitor{}, f + 1, l, fn); + pos.towards_oh(for_each_chunk_left_visitor{}, last - 1, l, fn); + } + } + } + + template <typename Pos, typename Fn> + static void visit_leaf(Pos&& pos, size_t first, size_t last, Fn&& fn) + { + auto data = pos.node()->leaf(); + if (first < last) { + auto f = pos.index(first); + auto l = pos.index(last - 1); + fn(data + f, data + l + 1); + } + } +}; + +struct for_each_chunk_p_left_visitor + : visitor_base<for_each_chunk_p_left_visitor> +{ + using this_t = for_each_chunk_p_left_visitor; + + template <typename Pos, typename Fn> + static bool visit_inner(Pos&& pos, size_t last, Fn&& fn) + { + auto l = pos.index(last); + return pos.each_pred_left(for_each_chunk_p_visitor{}, l, fn) && + pos.towards_oh(this_t{}, last, l, fn); + } + + template <typename Pos, typename Fn> + static bool visit_leaf(Pos&& pos, size_t last, Fn&& fn) + { + auto data = pos.node()->leaf(); + auto l = pos.index(last); + return fn(data, data + l + 1); + } +}; + +struct for_each_chunk_p_right_visitor + : visitor_base<for_each_chunk_p_right_visitor> +{ + using this_t = for_each_chunk_p_right_visitor; + + template <typename Pos, typename Fn> + static bool visit_inner(Pos&& pos, size_t first, Fn&& fn) + { + auto f = pos.index(first); + return pos.towards_oh(this_t{}, first, f, fn) && + pos.each_pred_right(for_each_chunk_p_visitor{}, f + 1, fn); + } + + template <typename Pos, typename Fn> + static bool visit_leaf(Pos&& pos, size_t first, Fn&& fn) + { + auto data = pos.node()->leaf(); + auto f = pos.index(first); + return fn(data + f, data + pos.count()); + } +}; + +struct for_each_chunk_p_i_visitor : visitor_base<for_each_chunk_p_i_visitor> +{ + using this_t = for_each_chunk_p_i_visitor; + + template <typename Pos, typename Fn> + static bool visit_relaxed(Pos&& pos, size_t first, size_t last, Fn&& fn) + { + // we are going towards *two* indices, so we need to do the + // relaxed as a special case to correct the second index + if (first < last) { + auto f = pos.index(first); + auto l = pos.index(last - 1); + if (f == l) { + auto sbh = pos.size_before(f); + return pos.towards_oh_sbh( + this_t{}, first, f, sbh, last - sbh, fn); + } else { + assert(f < l); + return pos.towards_oh( + for_each_chunk_p_right_visitor{}, first, f, fn) && + pos.each_pred_i( + for_each_chunk_p_visitor{}, f + 1, l, fn) && + pos.towards_oh( + for_each_chunk_p_left_visitor{}, last - 1, l, fn); + } + } + return true; + } + + template <typename Pos, typename Fn> + static bool visit_regular(Pos&& pos, size_t first, size_t last, Fn&& fn) + { + if (first < last) { + auto f = pos.index(first); + auto l = pos.index(last - 1); + if (f == l) + return pos.towards_oh(this_t{}, first, f, last, fn); + else { + assert(f < l); + return pos.towards_oh( + for_each_chunk_p_right_visitor{}, first, f, fn) && + pos.each_pred_i( + for_each_chunk_p_visitor{}, f + 1, l, fn) && + pos.towards_oh( + for_each_chunk_p_left_visitor{}, last - 1, l, fn); + } + } + return true; + } + + template <typename Pos, typename Fn> + static bool visit_leaf(Pos&& pos, size_t first, size_t last, Fn&& fn) + { + auto data = pos.node()->leaf(); + if (first < last) { + auto f = pos.index(first); + auto l = pos.index(last - 1); + return fn(data + f, data + l + 1); + } + return true; + } +}; + +struct equals_visitor : visitor_base<equals_visitor> +{ + using this_t = equals_visitor; + + struct this_aux_t : visitor_base<this_aux_t> + { + template <typename PosR, typename PosL, typename Iter> + static bool visit_inner( + PosR&& posr, count_t i, PosL&& posl, Iter&& first, size_t idx) + { + return posl.nth_sub(i, this_t{}, posr, first, idx); + } + + template <typename PosR, typename PosL, typename Iter> + static bool visit_leaf( + PosR&& posr, count_t i, PosL&& posl, Iter&& first, size_t idx) + { + return posl.nth_sub_leaf(i, this_t{}, posr, first, idx); + } + }; + + struct rrb : visitor_base<rrb> + { + template <typename PosR, typename Iter, typename Node> + static bool visit_node(PosR&& posr, + Iter&& first, + Node* rootl, + shift_t shiftl, + size_t sizel) + { + assert(shiftl <= posr.shift()); + return shiftl == posr.shift() + ? visit_maybe_relaxed_sub(rootl, + shiftl, + sizel, + this_t{}, + posr, + first, + size_t{}) + : posr.first_sub_inner( + rrb{}, first, rootl, shiftl, sizel); + } + }; + + template <typename Iter> + static auto equal_chunk_p(Iter&& iter) + { + return [iter](auto f, auto e) mutable { + if (f == &*iter) { + iter += e - f; + return true; + } + for (; f != e; ++f, ++iter) + if (*f != *iter) + return false; + return true; + }; + } + + template <typename PosL, typename PosR, typename Iter> + static bool + visit_relaxed(PosL&& posl, PosR&& posr, Iter&& first, size_t idx) + { + auto nl = posl.node(); + auto nr = posr.node(); + if (nl == nr) + return true; + auto cl = posl.count(); + auto cr = posr.count(); + assert(cr > 0); + auto sbr = size_t{}; + auto i = count_t{}; + auto j = count_t{}; + for (; i < cl; ++i) { + auto sbl = posl.size_before(i); + for (; j + 1 < cr && (sbr = posr.size_before(j)) < sbl; ++j) + ; + auto res = + sbl == sbr + ? posr.nth_sub(j, this_aux_t{}, i, posl, first, idx + sbl) + : posl.nth_sub(i, + for_each_chunk_p_visitor{}, + this_t::equal_chunk_p(first + (idx + sbl))); + if (!res) + return false; + } + return true; + } + + template <typename PosL, typename PosR, typename Iter> + static std::enable_if_t<is_relaxed_v<PosR>, bool> + visit_regular(PosL&& posl, PosR&& posr, Iter&& first, size_t idx) + { + return this_t::visit_relaxed(posl, posr, first, idx); + } + + template <typename PosL, typename PosR, typename Iter> + static std::enable_if_t<!is_relaxed_v<PosR>, bool> + visit_regular(PosL&& posl, PosR&& posr, Iter&& first, size_t idx) + { + return posl.count() >= posr.count() + ? this_t::visit_regular(posl, posr.node()) + : this_t::visit_regular(posr, posl.node()); + } + + template <typename PosL, typename PosR, typename Iter> + static bool visit_leaf(PosL&& posl, PosR&& posr, Iter&& first, size_t idx) + { + if (posl.node() == posr.node()) + return true; + auto cl = posl.count(); + auto cr = posr.count(); + auto mp = std::min(cl, cr); + return std::equal(posl.node()->leaf(), + posl.node()->leaf() + mp, + posr.node()->leaf()) && + std::equal(posl.node()->leaf() + mp, + posl.node()->leaf() + posl.count(), + first + (idx + mp)); + } + + template <typename Pos, typename NodeT> + static bool visit_regular(Pos&& pos, NodeT* other) + { + auto node = pos.node(); + return node == other || pos.each_pred_zip(this_t{}, other); + } + + template <typename Pos, typename NodeT> + static bool visit_leaf(Pos&& pos, NodeT* other) + { + auto node = pos.node(); + return node == other || std::equal(node->leaf(), + node->leaf() + pos.count(), + other->leaf()); + } +}; + +template <typename NodeT> +struct update_visitor : visitor_base<update_visitor<NodeT>> +{ + using node_t = NodeT; + using this_t = update_visitor; + + template <typename Pos, typename Fn> + static node_t* visit_relaxed(Pos&& pos, size_t idx, Fn&& fn) + { + auto offset = pos.index(idx); + auto count = pos.count(); + auto node = node_t::make_inner_sr_n(count, pos.relaxed()); + try { + auto child = pos.towards_oh(this_t{}, idx, offset, fn); + node_t::do_copy_inner_sr(node, pos.node(), count); + node->inner()[offset]->dec_unsafe(); + node->inner()[offset] = child; + return node; + } catch (...) { + node_t::delete_inner_r(node, count); + throw; + } + } + + template <typename Pos, typename Fn> + static node_t* visit_regular(Pos&& pos, size_t idx, Fn&& fn) + { + auto offset = pos.index(idx); + auto count = pos.count(); + auto node = node_t::make_inner_n(count); + try { + auto child = pos.towards_oh_ch(this_t{}, idx, offset, count, fn); + node_t::do_copy_inner(node, pos.node(), count); + node->inner()[offset]->dec_unsafe(); + node->inner()[offset] = child; + return node; + } catch (...) { + node_t::delete_inner(node, count); + throw; + } + } + + template <typename Pos, typename Fn> + static node_t* visit_leaf(Pos&& pos, size_t idx, Fn&& fn) + { + auto offset = pos.index(idx); + auto node = node_t::copy_leaf(pos.node(), pos.count()); + try { + node->leaf()[offset] = + std::forward<Fn>(fn)(std::move(node->leaf()[offset])); + return node; + } catch (...) { + node_t::delete_leaf(node, pos.count()); + throw; + } + } +}; + +struct dec_visitor : visitor_base<dec_visitor> +{ + using this_t = dec_visitor; + + template <typename Pos> + static void visit_relaxed(Pos&& p) + { + using node_t = node_type<Pos>; + auto node = p.node(); + if (node->dec()) { + p.each(this_t{}); + node_t::delete_inner_r(node, p.count()); + } + } + + template <typename Pos> + static void visit_regular(Pos&& p) + { + using node_t = node_type<Pos>; + auto node = p.node(); + if (node->dec()) { + p.each(this_t{}); + node_t::delete_inner(node, p.count()); + } + } + + template <typename Pos> + static void visit_leaf(Pos&& p) + { + using node_t = node_type<Pos>; + auto node = p.node(); + if (node->dec()) { + node_t::delete_leaf(node, p.count()); + } + } +}; + +template <typename NodeT> +void dec_leaf(NodeT* node, count_t n) +{ + make_leaf_sub_pos(node, n).visit(dec_visitor{}); +} + +template <typename NodeT> +void dec_inner(NodeT* node, shift_t shift, size_t size) +{ + visit_maybe_relaxed_sub(node, shift, size, dec_visitor()); +} + +template <typename NodeT> +void dec_relaxed(NodeT* node, shift_t shift) +{ + make_relaxed_pos(node, shift, node->relaxed()).visit(dec_visitor()); +} + +template <typename NodeT> +void dec_regular(NodeT* node, shift_t shift, size_t size) +{ + make_regular_pos(node, shift, size).visit(dec_visitor()); +} + +template <typename NodeT> +void dec_empty_regular(NodeT* node) +{ + make_empty_regular_pos(node).visit(dec_visitor()); +} + +template <typename NodeT> +struct get_mut_visitor : visitor_base<get_mut_visitor<NodeT>> +{ + using node_t = NodeT; + using this_t = get_mut_visitor; + using value_t = typename NodeT::value_t; + using edit_t = typename NodeT::edit_t; + + template <typename Pos> + static value_t& + visit_relaxed(Pos&& pos, size_t idx, edit_t e, node_t** location) + { + auto offset = pos.index(idx); + auto count = pos.count(); + auto node = pos.node(); + if (node->can_mutate(e)) { + return pos.towards_oh( + this_t{}, idx, offset, e, &node->inner()[offset]); + } else { + auto new_node = node_t::copy_inner_sr_e(e, node, count); + try { + auto& res = pos.towards_oh( + this_t{}, idx, offset, e, &new_node->inner()[offset]); + pos.visit(dec_visitor{}); + *location = new_node; + return res; + } catch (...) { + dec_relaxed(new_node, pos.shift()); + throw; + } + } + } + + template <typename Pos> + static value_t& + visit_regular(Pos&& pos, size_t idx, edit_t e, node_t** location) + { + assert(pos.node() == *location); + auto offset = pos.index(idx); + auto count = pos.count(); + auto node = pos.node(); + if (node->can_mutate(e)) { + return pos.towards_oh_ch( + this_t{}, idx, offset, count, e, &node->inner()[offset]); + } else { + auto new_node = node_t::copy_inner_e(e, node, count); + try { + auto& res = pos.towards_oh_ch(this_t{}, + idx, + offset, + count, + e, + &new_node->inner()[offset]); + pos.visit(dec_visitor{}); + *location = new_node; + return res; + } catch (...) { + dec_regular(new_node, pos.shift(), pos.size()); + throw; + } + } + } + + template <typename Pos> + static value_t& + visit_leaf(Pos&& pos, size_t idx, edit_t e, node_t** location) + { + assert(pos.node() == *location); + auto node = pos.node(); + if (node->can_mutate(e)) { + return node->leaf()[pos.index(idx)]; + } else { + auto new_node = node_t::copy_leaf_e(e, pos.node(), pos.count()); + pos.visit(dec_visitor{}); + *location = new_node; + return new_node->leaf()[pos.index(idx)]; + } + } +}; + +template <typename NodeT, bool Mutating = true> +struct push_tail_mut_visitor + : visitor_base<push_tail_mut_visitor<NodeT, Mutating>> +{ + static constexpr auto B = NodeT::bits; + static constexpr auto BL = NodeT::bits_leaf; + + using this_t = push_tail_mut_visitor; + using this_no_mut_t = push_tail_mut_visitor<NodeT, false>; + using node_t = NodeT; + using edit_t = typename NodeT::edit_t; + + template <typename Pos> + static node_t* visit_relaxed(Pos&& pos, edit_t e, node_t* tail, count_t ts) + { + auto node = pos.node(); + auto level = pos.shift(); + auto idx = pos.count() - 1; + auto children = pos.size(idx); + auto new_idx = + children == size_t{1} << level || level == BL ? idx + 1 : idx; + auto new_child = static_cast<node_t*>(nullptr); + auto mutate = Mutating && node->can_mutate(e); + + if (new_idx >= branches<B>) + return nullptr; + else if (idx == new_idx) { + new_child = + mutate ? pos.last_oh_csh(this_t{}, idx, children, e, tail, ts) + : pos.last_oh_csh( + this_no_mut_t{}, idx, children, e, tail, ts); + if (!new_child) { + if (++new_idx < branches<B>) + new_child = node_t::make_path_e(e, level - B, tail); + else + return nullptr; + } + } else + new_child = node_t::make_path_e(e, level - B, tail); + + if (mutate) { + auto count = new_idx + 1; + auto relaxed = node->ensure_mutable_relaxed_n(e, new_idx); + node->inner()[new_idx] = new_child; + relaxed->d.sizes[new_idx] = pos.size() + ts; + relaxed->d.count = count; + return node; + } else { + try { + auto count = new_idx + 1; + auto new_node = node_t::copy_inner_r_e(e, pos.node(), new_idx); + auto relaxed = new_node->relaxed(); + new_node->inner()[new_idx] = new_child; + relaxed->d.sizes[new_idx] = pos.size() + ts; + relaxed->d.count = count; + if (Mutating) + pos.visit(dec_visitor{}); + return new_node; + } catch (...) { + auto shift = pos.shift(); + auto size = new_idx == idx ? children + ts : ts; + if (shift > BL) { + tail->inc(); + dec_inner(new_child, shift - B, size); + } + throw; + } + } + } + + template <typename Pos, typename... Args> + static node_t* visit_regular(Pos&& pos, edit_t e, node_t* tail, Args&&...) + { + assert((pos.size() & mask<BL>) == 0); + auto node = pos.node(); + auto idx = pos.index(pos.size() - 1); + auto new_idx = pos.index(pos.size() + branches<BL> - 1); + auto mutate = Mutating && node->can_mutate(e); + if (mutate) { + node->inner()[new_idx] = + idx == new_idx ? pos.last_oh(this_t{}, idx, e, tail) + /* otherwise */ + : node_t::make_path_e(e, pos.shift() - B, tail); + return node; + } else { + auto new_parent = node_t::make_inner_e(e); + try { + new_parent->inner()[new_idx] = + idx == new_idx + ? pos.last_oh(this_no_mut_t{}, idx, e, tail) + /* otherwise */ + : node_t::make_path_e(e, pos.shift() - B, tail); + node_t::do_copy_inner(new_parent, node, new_idx); + if (Mutating) + pos.visit(dec_visitor{}); + return new_parent; + } catch (...) { + node_t::delete_inner_e(new_parent); + throw; + } + } + } + + template <typename Pos, typename... Args> + static node_t* visit_leaf(Pos&& pos, edit_t e, node_t* tail, Args&&...) + { + IMMER_UNREACHABLE; + } +}; + +template <typename NodeT> +struct push_tail_visitor : visitor_base<push_tail_visitor<NodeT>> +{ + static constexpr auto B = NodeT::bits; + static constexpr auto BL = NodeT::bits_leaf; + + using this_t = push_tail_visitor; + using node_t = NodeT; + + template <typename Pos> + static node_t* visit_relaxed(Pos&& pos, node_t* tail, count_t ts) + { + auto level = pos.shift(); + auto idx = pos.count() - 1; + auto children = pos.size(idx); + auto new_idx = + children == size_t{1} << level || level == BL ? idx + 1 : idx; + auto new_child = static_cast<node_t*>(nullptr); + if (new_idx >= branches<B>) + return nullptr; + else if (idx == new_idx) { + new_child = pos.last_oh_csh(this_t{}, idx, children, tail, ts); + if (!new_child) { + if (++new_idx < branches<B>) + new_child = node_t::make_path(level - B, tail); + else + return nullptr; + } + } else + new_child = node_t::make_path(level - B, tail); + try { + auto count = new_idx + 1; + auto new_parent = + node_t::copy_inner_r_n(count, pos.node(), new_idx); + auto new_relaxed = new_parent->relaxed(); + new_parent->inner()[new_idx] = new_child; + new_relaxed->d.sizes[new_idx] = pos.size() + ts; + new_relaxed->d.count = count; + return new_parent; + } catch (...) { + auto shift = pos.shift(); + auto size = new_idx == idx ? children + ts : ts; + if (shift > BL) { + tail->inc(); + dec_inner(new_child, shift - B, size); + } + throw; + } + } + + template <typename Pos, typename... Args> + static node_t* visit_regular(Pos&& pos, node_t* tail, Args&&...) + { + assert((pos.size() & mask<BL>) == 0); + auto idx = pos.index(pos.size() - 1); + auto new_idx = pos.index(pos.size() + branches<BL> - 1); + auto count = new_idx + 1; + auto new_parent = node_t::make_inner_n(count); + try { + new_parent->inner()[new_idx] = + idx == new_idx ? pos.last_oh(this_t{}, idx, tail) + /* otherwise */ + : node_t::make_path(pos.shift() - B, tail); + } catch (...) { + node_t::delete_inner(new_parent, count); + throw; + } + return node_t::do_copy_inner(new_parent, pos.node(), new_idx); + } + + template <typename Pos, typename... Args> + static node_t* visit_leaf(Pos&& pos, node_t* tail, Args&&...) + { + IMMER_UNREACHABLE; + } +}; + +struct dec_right_visitor : visitor_base<dec_right_visitor> +{ + using this_t = dec_right_visitor; + using dec_t = dec_visitor; + + template <typename Pos> + static void visit_relaxed(Pos&& p, count_t idx) + { + using node_t = node_type<Pos>; + auto node = p.node(); + if (node->dec()) { + p.each_right(dec_t{}, idx); + node_t::delete_inner_r(node, p.count()); + } + } + + template <typename Pos> + static void visit_regular(Pos&& p, count_t idx) + { + using node_t = node_type<Pos>; + auto node = p.node(); + if (node->dec()) { + p.each_right(dec_t{}, idx); + node_t::delete_inner(node, p.count()); + } + } + + template <typename Pos> + static void visit_leaf(Pos&& p, count_t idx) + { + IMMER_UNREACHABLE; + } +}; + +template <typename NodeT, bool Collapse = true, bool Mutating = true> +struct slice_right_mut_visitor + : visitor_base<slice_right_mut_visitor<NodeT, Collapse, Mutating>> +{ + using node_t = NodeT; + using this_t = slice_right_mut_visitor; + using edit_t = typename NodeT::edit_t; + + // returns a new shift, new root, the new tail size and the new tail + using result_t = std::tuple<shift_t, NodeT*, count_t, NodeT*>; + using no_collapse_t = slice_right_mut_visitor<NodeT, false, true>; + using no_collapse_no_mut_t = slice_right_mut_visitor<NodeT, false, false>; + using no_mut_t = slice_right_mut_visitor<NodeT, Collapse, false>; + + static constexpr auto B = NodeT::bits; + static constexpr auto BL = NodeT::bits_leaf; + + template <typename PosT> + static result_t visit_relaxed(PosT&& pos, size_t last, edit_t e) + { + auto idx = pos.index(last); + auto node = pos.node(); + auto mutate = Mutating && node->can_mutate(e); + if (Collapse && idx == 0) { + auto res = mutate ? pos.towards_oh(this_t{}, last, idx, e) + : pos.towards_oh(no_mut_t{}, last, idx, e); + if (Mutating) + pos.visit(dec_right_visitor{}, count_t{1}); + return res; + } else { + using std::get; + auto subs = + mutate ? pos.towards_oh(no_collapse_t{}, last, idx, e) + : pos.towards_oh(no_collapse_no_mut_t{}, last, idx, e); + auto next = get<1>(subs); + auto ts = get<2>(subs); + auto tail = get<3>(subs); + try { + if (next) { + if (mutate) { + auto nodr = node->ensure_mutable_relaxed_n(e, idx); + pos.each_right(dec_visitor{}, idx + 1); + node->inner()[idx] = next; + nodr->d.sizes[idx] = last + 1 - ts; + nodr->d.count = idx + 1; + return std::make_tuple(pos.shift(), node, ts, tail); + } else { + auto newn = node_t::copy_inner_r_e(e, node, idx); + auto newr = newn->relaxed(); + newn->inner()[idx] = next; + newr->d.sizes[idx] = last + 1 - ts; + newr->d.count = idx + 1; + if (Mutating) + pos.visit(dec_visitor{}); + return std::make_tuple(pos.shift(), newn, ts, tail); + } + } else if (idx == 0) { + if (Mutating) + pos.visit(dec_right_visitor{}, count_t{1}); + return std::make_tuple(pos.shift(), nullptr, ts, tail); + } else if (Collapse && idx == 1 && pos.shift() > BL) { + auto newn = pos.node()->inner()[0]; + if (!mutate) + newn->inc(); + if (Mutating) + pos.visit(dec_right_visitor{}, count_t{2}); + return std::make_tuple(pos.shift() - B, newn, ts, tail); + } else { + if (mutate) { + pos.each_right(dec_visitor{}, idx + 1); + node->ensure_mutable_relaxed_n(e, idx)->d.count = idx; + return std::make_tuple(pos.shift(), node, ts, tail); + } else { + auto newn = node_t::copy_inner_r_e(e, node, idx); + if (Mutating) + pos.visit(dec_visitor{}); + return std::make_tuple(pos.shift(), newn, ts, tail); + } + } + } catch (...) { + assert(!mutate); + assert(!next || pos.shift() > BL); + if (next) + dec_inner(next, + pos.shift() - B, + last + 1 - ts - pos.size_before(idx)); + dec_leaf(tail, ts); + throw; + } + } + } + + template <typename PosT> + static result_t visit_regular(PosT&& pos, size_t last, edit_t e) + { + auto idx = pos.index(last); + auto node = pos.node(); + auto mutate = Mutating && node->can_mutate(e); + if (Collapse && idx == 0) { + auto res = mutate ? pos.towards_oh(this_t{}, last, idx, e) + : pos.towards_oh(no_mut_t{}, last, idx, e); + if (Mutating) + pos.visit(dec_right_visitor{}, count_t{1}); + return res; + } else { + using std::get; + auto subs = + mutate ? pos.towards_oh(no_collapse_t{}, last, idx, e) + : pos.towards_oh(no_collapse_no_mut_t{}, last, idx, e); + auto next = get<1>(subs); + auto ts = get<2>(subs); + auto tail = get<3>(subs); + try { + if (next) { + if (mutate) { + node->inner()[idx] = next; + pos.each_right(dec_visitor{}, idx + 1); + return std::make_tuple(pos.shift(), node, ts, tail); + } else { + auto newn = node_t::copy_inner_e(e, node, idx); + newn->inner()[idx] = next; + if (Mutating) + pos.visit(dec_visitor{}); + return std::make_tuple(pos.shift(), newn, ts, tail); + } + } else if (idx == 0) { + if (Mutating) + pos.visit(dec_right_visitor{}, count_t{1}); + return std::make_tuple(pos.shift(), nullptr, ts, tail); + } else if (Collapse && idx == 1 && pos.shift() > BL) { + auto newn = pos.node()->inner()[0]; + if (!mutate) + newn->inc(); + if (Mutating) + pos.visit(dec_right_visitor{}, count_t{2}); + return std::make_tuple(pos.shift() - B, newn, ts, tail); + } else { + if (mutate) { + pos.each_right(dec_visitor{}, idx + 1); + return std::make_tuple(pos.shift(), node, ts, tail); + } else { + auto newn = node_t::copy_inner_e(e, node, idx); + if (Mutating) + pos.visit(dec_visitor{}); + return std::make_tuple(pos.shift(), newn, ts, tail); + } + } + } catch (...) { + assert(!mutate); + assert(!next || pos.shift() > BL); + assert(tail); + if (next) + dec_regular(next, pos.shift() - B, last + 1 - ts); + dec_leaf(tail, ts); + throw; + } + } + } + + template <typename PosT> + static result_t visit_leaf(PosT&& pos, size_t last, edit_t e) + { + auto old_tail_size = pos.count(); + auto new_tail_size = pos.index(last) + 1; + auto node = pos.node(); + auto mutate = Mutating && node->can_mutate(e); + if (new_tail_size == old_tail_size) { + if (!Mutating) + node->inc(); + return std::make_tuple(0, nullptr, new_tail_size, node); + } else if (mutate) { + destroy_n(node->leaf() + new_tail_size, + old_tail_size - new_tail_size); + return std::make_tuple(0, nullptr, new_tail_size, node); + } else { + auto new_tail = node_t::copy_leaf_e(e, node, new_tail_size); + if (Mutating) + pos.visit(dec_visitor{}); + return std::make_tuple(0, nullptr, new_tail_size, new_tail); + } + } +}; + +template <typename NodeT, bool Collapse = true> +struct slice_right_visitor : visitor_base<slice_right_visitor<NodeT, Collapse>> +{ + using node_t = NodeT; + using this_t = slice_right_visitor; + + // returns a new shift, new root, the new tail size and the new tail + using result_t = std::tuple<shift_t, NodeT*, count_t, NodeT*>; + using no_collapse_t = slice_right_visitor<NodeT, false>; + + static constexpr auto B = NodeT::bits; + static constexpr auto BL = NodeT::bits_leaf; + + template <typename PosT> + static result_t visit_relaxed(PosT&& pos, size_t last) + { + auto idx = pos.index(last); + if (Collapse && idx == 0) { + return pos.towards_oh(this_t{}, last, idx); + } else { + using std::get; + auto subs = pos.towards_oh(no_collapse_t{}, last, idx); + auto next = get<1>(subs); + auto ts = get<2>(subs); + auto tail = get<3>(subs); + try { + if (next) { + auto count = idx + 1; + auto newn = node_t::copy_inner_r_n(count, pos.node(), idx); + auto newr = newn->relaxed(); + newn->inner()[idx] = next; + newr->d.sizes[idx] = last + 1 - ts; + newr->d.count = count; + return std::make_tuple(pos.shift(), newn, ts, tail); + } else if (idx == 0) { + return std::make_tuple(pos.shift(), nullptr, ts, tail); + } else if (Collapse && idx == 1 && pos.shift() > BL) { + auto newn = pos.node()->inner()[0]; + return std::make_tuple( + pos.shift() - B, newn->inc(), ts, tail); + } else { + auto newn = node_t::copy_inner_r(pos.node(), idx); + return std::make_tuple(pos.shift(), newn, ts, tail); + } + } catch (...) { + assert(!next || pos.shift() > BL); + if (next) + dec_inner(next, + pos.shift() - B, + last + 1 - ts - pos.size_before(idx)); + if (tail) + dec_leaf(tail, ts); + throw; + } + } + } + + template <typename PosT> + static result_t visit_regular(PosT&& pos, size_t last) + { + auto idx = pos.index(last); + if (Collapse && idx == 0) { + return pos.towards_oh(this_t{}, last, idx); + } else { + using std::get; + auto subs = pos.towards_oh(no_collapse_t{}, last, idx); + auto next = get<1>(subs); + auto ts = get<2>(subs); + auto tail = get<3>(subs); + try { + if (next) { + auto newn = node_t::copy_inner_n(idx + 1, pos.node(), idx); + newn->inner()[idx] = next; + return std::make_tuple(pos.shift(), newn, ts, tail); + } else if (idx == 0) { + return std::make_tuple(pos.shift(), nullptr, ts, tail); + } else if (Collapse && idx == 1 && pos.shift() > BL) { + auto newn = pos.node()->inner()[0]; + return std::make_tuple( + pos.shift() - B, newn->inc(), ts, tail); + } else { + auto newn = node_t::copy_inner_n(idx, pos.node(), idx); + return std::make_tuple(pos.shift(), newn, ts, tail); + } + } catch (...) { + assert(!next || pos.shift() > BL); + assert(tail); + if (next) + dec_regular(next, pos.shift() - B, last + 1 - ts); + dec_leaf(tail, ts); + throw; + } + } + } + + template <typename PosT> + static result_t visit_leaf(PosT&& pos, size_t last) + { + auto old_tail_size = pos.count(); + auto new_tail_size = pos.index(last) + 1; + auto new_tail = new_tail_size == old_tail_size + ? pos.node()->inc() + : node_t::copy_leaf(pos.node(), new_tail_size); + return std::make_tuple(0, nullptr, new_tail_size, new_tail); + } +}; + +struct dec_left_visitor : visitor_base<dec_left_visitor> +{ + using this_t = dec_left_visitor; + using dec_t = dec_visitor; + + template <typename Pos> + static void visit_relaxed(Pos&& p, count_t idx) + { + using node_t = node_type<Pos>; + auto node = p.node(); + if (node->dec()) { + p.each_left(dec_t{}, idx); + node_t::delete_inner_r(node, p.count()); + } + } + + template <typename Pos> + static void visit_regular(Pos&& p, count_t idx) + { + using node_t = node_type<Pos>; + auto node = p.node(); + if (node->dec()) { + p.each_left(dec_t{}, idx); + node_t::delete_inner(node, p.count()); + } + } + + template <typename Pos> + static void visit_leaf(Pos&& p, count_t idx) + { + IMMER_UNREACHABLE; + } +}; + +template <typename NodeT, bool Collapse = true, bool Mutating = true> +struct slice_left_mut_visitor + : visitor_base<slice_left_mut_visitor<NodeT, Collapse, Mutating>> +{ + using node_t = NodeT; + using this_t = slice_left_mut_visitor; + using edit_t = typename NodeT::edit_t; + using value_t = typename NodeT::value_t; + using relaxed_t = typename NodeT::relaxed_t; + // returns a new shift and new root + using result_t = std::tuple<shift_t, NodeT*>; + + using no_collapse_t = slice_left_mut_visitor<NodeT, false, true>; + using no_collapse_no_mut_t = slice_left_mut_visitor<NodeT, false, false>; + using no_mut_t = slice_left_mut_visitor<NodeT, Collapse, false>; + + static constexpr auto B = NodeT::bits; + static constexpr auto BL = NodeT::bits_leaf; + + template <typename PosT> + static result_t visit_relaxed(PosT&& pos, size_t first, edit_t e) + { + auto idx = pos.subindex(first); + auto count = pos.count(); + auto node = pos.node(); + auto mutate = Mutating && node->can_mutate(e); + auto left_size = pos.size_before(idx); + auto child_size = pos.size_sbh(idx, left_size); + auto dropped_size = first; + auto child_dropped_size = dropped_size - left_size; + if (Collapse && pos.shift() > BL && idx == pos.count() - 1) { + auto r = mutate ? pos.towards_sub_oh(this_t{}, first, idx, e) + : pos.towards_sub_oh(no_mut_t{}, first, idx, e); + if (Mutating) + pos.visit(dec_left_visitor{}, idx); + return r; + } else { + using std::get; + auto newn = mutate ? (node->ensure_mutable_relaxed(e), node) + : node_t::make_inner_r_e(e); + auto newr = newn->relaxed(); + auto newcount = count - idx; + auto new_child_size = child_size - child_dropped_size; + try { + auto subs = + mutate ? pos.towards_sub_oh(no_collapse_t{}, first, idx, e) + : pos.towards_sub_oh( + no_collapse_no_mut_t{}, first, idx, e); + if (mutate) + pos.each_left(dec_visitor{}, idx); + pos.copy_sizes( + idx + 1, newcount - 1, new_child_size, newr->d.sizes + 1); + std::uninitialized_copy(node->inner() + idx + 1, + node->inner() + count, + newn->inner() + 1); + newn->inner()[0] = get<1>(subs); + newr->d.sizes[0] = new_child_size; + newr->d.count = newcount; + if (!mutate) { + node_t::inc_nodes(newn->inner() + 1, newcount - 1); + if (Mutating) + pos.visit(dec_visitor{}); + } + return std::make_tuple(pos.shift(), newn); + } catch (...) { + if (!mutate) + node_t::delete_inner_r_e(newn); + throw; + } + } + } + + template <typename PosT> + static result_t visit_regular(PosT&& pos, size_t first, edit_t e) + { + auto idx = pos.subindex(first); + auto count = pos.count(); + auto node = pos.node(); + auto mutate = Mutating + // this is more restrictive than actually needed because + // it causes the algorithm to also avoid mutating the leaf + // in place + && !node_t::embed_relaxed && node->can_mutate(e); + auto left_size = pos.size_before(idx); + auto child_size = pos.size_sbh(idx, left_size); + auto dropped_size = first; + auto child_dropped_size = dropped_size - left_size; + if (Collapse && pos.shift() > BL && idx == pos.count() - 1) { + auto r = mutate ? pos.towards_sub_oh(this_t{}, first, idx, e) + : pos.towards_sub_oh(no_mut_t{}, first, idx, e); + if (Mutating) + pos.visit(dec_left_visitor{}, idx); + return r; + } else { + using std::get; + // if possible, we convert the node to a relaxed one + // simply by allocating a `relaxed_t` size table for + // it... maybe some of this magic should be moved as a + // `node<...>` static method... + auto newcount = count - idx; + auto newn = + mutate ? (node->impl.d.data.inner.relaxed = new ( + node_t::heap::allocate(node_t::max_sizeof_relaxed, + norefs_tag{})) relaxed_t, + node) + : node_t::make_inner_r_e(e); + auto newr = newn->relaxed(); + try { + auto subs = + mutate ? pos.towards_sub_oh(no_collapse_t{}, first, idx, e) + : pos.towards_sub_oh( + no_collapse_no_mut_t{}, first, idx, e); + if (mutate) + pos.each_left(dec_visitor{}, idx); + newr->d.sizes[0] = child_size - child_dropped_size; + pos.copy_sizes( + idx + 1, newcount - 1, newr->d.sizes[0], newr->d.sizes + 1); + newr->d.count = newcount; + newn->inner()[0] = get<1>(subs); + std::uninitialized_copy(node->inner() + idx + 1, + node->inner() + count, + newn->inner() + 1); + if (!mutate) { + node_t::inc_nodes(newn->inner() + 1, newcount - 1); + if (Mutating) + pos.visit(dec_visitor{}); + } + return std::make_tuple(pos.shift(), newn); + } catch (...) { + if (!mutate) + node_t::delete_inner_r_e(newn); + else { + // restore the regular node that we were + // attempting to relax... + node_t::heap::deallocate(node_t::max_sizeof_relaxed, + node->impl.d.data.inner.relaxed); + node->impl.d.data.inner.relaxed = nullptr; + } + throw; + } + } + } + + template <typename PosT> + static result_t visit_leaf(PosT&& pos, size_t first, edit_t e) + { + auto node = pos.node(); + auto idx = pos.index(first); + auto count = pos.count(); + auto mutate = Mutating && + std::is_nothrow_move_constructible<value_t>::value && + node->can_mutate(e); + if (mutate) { + auto data = node->leaf(); + auto newcount = count - idx; + std::move(data + idx, data + count, data); + destroy_n(data + newcount, idx); + return std::make_tuple(0, node); + } else { + auto newn = node_t::copy_leaf_e(e, node, idx, count); + if (Mutating) + pos.visit(dec_visitor{}); + return std::make_tuple(0, newn); + } + } +}; + +template <typename NodeT, bool Collapse = true> +struct slice_left_visitor : visitor_base<slice_left_visitor<NodeT, Collapse>> +{ + using node_t = NodeT; + using this_t = slice_left_visitor; + + // returns a new shift and new root + using result_t = std::tuple<shift_t, NodeT*>; + using no_collapse_t = slice_left_visitor<NodeT, false>; + + static constexpr auto B = NodeT::bits; + static constexpr auto BL = NodeT::bits_leaf; + + template <typename PosT> + static result_t visit_inner(PosT&& pos, size_t first) + { + auto idx = pos.subindex(first); + auto count = pos.count(); + auto left_size = pos.size_before(idx); + auto child_size = pos.size_sbh(idx, left_size); + auto dropped_size = first; + auto child_dropped_size = dropped_size - left_size; + if (Collapse && pos.shift() > BL && idx == pos.count() - 1) { + return pos.towards_sub_oh(this_t{}, first, idx); + } else { + using std::get; + auto n = pos.node(); + auto newc = count - idx; + auto newn = node_t::make_inner_r_n(newc); + try { + auto subs = pos.towards_sub_oh(no_collapse_t{}, first, idx); + auto newr = newn->relaxed(); + newr->d.count = count - idx; + newr->d.sizes[0] = child_size - child_dropped_size; + pos.copy_sizes(idx + 1, + newr->d.count - 1, + newr->d.sizes[0], + newr->d.sizes + 1); + assert(newr->d.sizes[newr->d.count - 1] == + pos.size() - dropped_size); + newn->inner()[0] = get<1>(subs); + std::uninitialized_copy(n->inner() + idx + 1, + n->inner() + count, + newn->inner() + 1); + node_t::inc_nodes(newn->inner() + 1, newr->d.count - 1); + return std::make_tuple(pos.shift(), newn); + } catch (...) { + node_t::delete_inner_r(newn, newc); + throw; + } + } + } + + template <typename PosT> + static result_t visit_leaf(PosT&& pos, size_t first) + { + auto n = node_t::copy_leaf(pos.node(), pos.index(first), pos.count()); + return std::make_tuple(0, n); + } +}; + +template <typename Node> +struct concat_center_pos +{ + static constexpr auto B = Node::bits; + static constexpr auto BL = Node::bits_leaf; + + static constexpr count_t max_children = 3; + + using node_t = Node; + using edit_t = typename Node::edit_t; + + shift_t shift_ = 0u; + count_t count_ = 0u; + node_t* nodes_[max_children]; + size_t sizes_[max_children]; + + auto shift() const { return shift_; } + + concat_center_pos(shift_t s, Node* n0, size_t s0) + : shift_{s} + , count_{1} + , nodes_{n0} + , sizes_{s0} + {} + + concat_center_pos(shift_t s, Node* n0, size_t s0, Node* n1, size_t s1) + : shift_{s} + , count_{2} + , nodes_{n0, n1} + , sizes_{s0, s1} + {} + + concat_center_pos(shift_t s, + Node* n0, + size_t s0, + Node* n1, + size_t s1, + Node* n2, + size_t s2) + : shift_{s} + , count_{3} + , nodes_{n0, n1, n2} + , sizes_{s0, s1, s2} + {} + + template <typename Visitor, typename... Args> + void each_sub(Visitor v, Args&&... args) + { + if (shift_ == BL) { + for (auto i = count_t{0}; i < count_; ++i) + make_leaf_sub_pos(nodes_[i], sizes_[i]).visit(v, args...); + } else { + for (auto i = count_t{0}; i < count_; ++i) + make_relaxed_pos(nodes_[i], shift_ - B, nodes_[i]->relaxed()) + .visit(v, args...); + } + } + + relaxed_pos<Node> realize() && + { + if (count_ > 1) { + try { + auto result = node_t::make_inner_r_n(count_); + auto r = result->relaxed(); + r->d.count = count_; + std::copy(nodes_, nodes_ + count_, result->inner()); + std::copy(sizes_, sizes_ + count_, r->d.sizes); + return {result, shift_, r}; + } catch (...) { + each_sub(dec_visitor{}); + throw; + } + } else { + assert(shift_ >= B + BL); + return {nodes_[0], shift_ - B, nodes_[0]->relaxed()}; + } + } + + relaxed_pos<Node> realize_e(edit_t e) + { + if (count_ > 1) { + auto result = node_t::make_inner_r_e(e); + auto r = result->relaxed(); + r->d.count = count_; + std::copy(nodes_, nodes_ + count_, result->inner()); + std::copy(sizes_, sizes_ + count_, r->d.sizes); + return {result, shift_, r}; + } else { + assert(shift_ >= B + BL); + return {nodes_[0], shift_ - B, nodes_[0]->relaxed()}; + } + } +}; + +template <typename Node> +struct concat_merger +{ + using node_t = Node; + static constexpr auto B = Node::bits; + static constexpr auto BL = Node::bits_leaf; + + using result_t = concat_center_pos<Node>; + + count_t* curr_; + count_t n_; + result_t result_; + + concat_merger(shift_t shift, count_t* counts, count_t n) + : curr_{counts} + , n_{n} + , result_{ + shift + B, node_t::make_inner_r_n(std::min(n_, branches<B>)), 0} + {} + + node_t* to_ = {}; + count_t to_offset_ = {}; + size_t to_size_ = {}; + + void add_child(node_t* p, size_t size) + { + ++curr_; + auto parent = result_.nodes_[result_.count_ - 1]; + auto relaxed = parent->relaxed(); + if (relaxed->d.count == branches<B>) { + assert(result_.count_ < result_t::max_children); + n_ -= branches<B>; + parent = node_t::make_inner_r_n(std::min(n_, branches<B>)); + relaxed = parent->relaxed(); + result_.nodes_[result_.count_] = parent; + result_.sizes_[result_.count_] = result_.sizes_[result_.count_ - 1]; + ++result_.count_; + } + auto idx = relaxed->d.count++; + result_.sizes_[result_.count_ - 1] += size; + relaxed->d.sizes[idx] = size + (idx ? relaxed->d.sizes[idx - 1] : 0); + parent->inner()[idx] = p; + }; + + template <typename Pos> + void merge_leaf(Pos&& p) + { + auto from = p.node(); + auto from_size = p.size(); + auto from_count = p.count(); + assert(from_size); + if (!to_ && *curr_ == from_count) { + add_child(from, from_size); + from->inc(); + } else { + auto from_offset = count_t{}; + auto from_data = from->leaf(); + do { + if (!to_) { + to_ = node_t::make_leaf_n(*curr_); + to_offset_ = 0; + } + auto data = to_->leaf(); + auto to_copy = + std::min(from_count - from_offset, *curr_ - to_offset_); + std::uninitialized_copy(from_data + from_offset, + from_data + from_offset + to_copy, + data + to_offset_); + to_offset_ += to_copy; + from_offset += to_copy; + if (*curr_ == to_offset_) { + add_child(to_, to_offset_); + to_ = nullptr; + } + } while (from_offset != from_count); + } + } + + template <typename Pos> + void merge_inner(Pos&& p) + { + auto from = p.node(); + auto from_size = p.size(); + auto from_count = p.count(); + assert(from_size); + if (!to_ && *curr_ == from_count) { + add_child(from, from_size); + from->inc(); + } else { + auto from_offset = count_t{}; + auto from_data = from->inner(); + do { + if (!to_) { + to_ = node_t::make_inner_r_n(*curr_); + to_offset_ = 0; + to_size_ = 0; + } + auto data = to_->inner(); + auto to_copy = + std::min(from_count - from_offset, *curr_ - to_offset_); + std::uninitialized_copy(from_data + from_offset, + from_data + from_offset + to_copy, + data + to_offset_); + node_t::inc_nodes(from_data + from_offset, to_copy); + auto sizes = to_->relaxed()->d.sizes; + p.copy_sizes( + from_offset, to_copy, to_size_, sizes + to_offset_); + to_offset_ += to_copy; + from_offset += to_copy; + to_size_ = sizes[to_offset_ - 1]; + if (*curr_ == to_offset_) { + to_->relaxed()->d.count = to_offset_; + add_child(to_, to_size_); + to_ = nullptr; + } + } while (from_offset != from_count); + } + } + + concat_center_pos<Node> finish() const + { + assert(!to_); + return result_; + } + + void abort() + { + auto shift = result_.shift_ - B; + if (to_) { + if (shift == BL) + node_t::delete_leaf(to_, to_offset_); + else { + to_->relaxed()->d.count = to_offset_; + dec_relaxed(to_, shift - B); + } + } + result_.each_sub(dec_visitor()); + } +}; + +struct concat_merger_visitor : visitor_base<concat_merger_visitor> +{ + using this_t = concat_merger_visitor; + + template <typename Pos, typename Merger> + static void visit_inner(Pos&& p, Merger& merger) + { + merger.merge_inner(p); + } + + template <typename Pos, typename Merger> + static void visit_leaf(Pos&& p, Merger& merger) + { + merger.merge_leaf(p); + } +}; + +struct concat_rebalance_plan_fill_visitor + : visitor_base<concat_rebalance_plan_fill_visitor> +{ + using this_t = concat_rebalance_plan_fill_visitor; + + template <typename Pos, typename Plan> + static void visit_node(Pos&& p, Plan& plan) + { + auto count = p.count(); + assert(plan.n < Plan::max_children); + plan.counts[plan.n++] = count; + plan.total += count; + } +}; + +template <bits_t B, bits_t BL> +struct concat_rebalance_plan +{ + static constexpr auto max_children = 2 * branches<B> + 1; + + count_t counts[max_children]; + count_t n = 0u; + count_t total = 0u; + + template <typename LPos, typename CPos, typename RPos> + void fill(LPos&& lpos, CPos&& cpos, RPos&& rpos) + { + assert(n == 0u); + assert(total == 0u); + using visitor_t = concat_rebalance_plan_fill_visitor; + lpos.each_left_sub(visitor_t{}, *this); + cpos.each_sub(visitor_t{}, *this); + rpos.each_right_sub(visitor_t{}, *this); + } + + void shuffle(shift_t shift) + { + // gcc seems to not really understand this code... :( +#if !defined(_MSC_VER) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Warray-bounds" +#endif + constexpr count_t rrb_extras = 2; + constexpr count_t rrb_invariant = 1; + const auto bits = shift == BL ? BL : B; + const auto branches = count_t{1} << bits; + const auto optimal = ((total - 1) >> bits) + 1; + count_t i = 0; + while (n >= optimal + rrb_extras) { + // skip ok nodes + while (counts[i] > branches - rrb_invariant) + i++; + // short node, redistribute + auto remaining = counts[i]; + do { + auto count = std::min(remaining + counts[i + 1], branches); + counts[i] = count; + remaining += counts[i + 1] - count; + ++i; + } while (remaining > 0); + // remove node + std::move(counts + i + 1, counts + n, counts + i); + --n; + --i; + } +#if !defined(_MSC_VER) +#pragma GCC diagnostic pop +#endif + } + + template <typename LPos, typename CPos, typename RPos> + concat_center_pos<node_type<CPos>> + merge(LPos&& lpos, CPos&& cpos, RPos&& rpos) + { + using node_t = node_type<CPos>; + using merger_t = concat_merger<node_t>; + using visitor_t = concat_merger_visitor; + auto merger = merger_t{cpos.shift(), counts, n}; + try { + lpos.each_left_sub(visitor_t{}, merger); + cpos.each_sub(visitor_t{}, merger); + rpos.each_right_sub(visitor_t{}, merger); + cpos.each_sub(dec_visitor{}); + return merger.finish(); + } catch (...) { + merger.abort(); + throw; + } + } +}; + +template <typename Node, typename LPos, typename CPos, typename RPos> +concat_center_pos<Node> concat_rebalance(LPos&& lpos, CPos&& cpos, RPos&& rpos) +{ + auto plan = concat_rebalance_plan<Node::bits, Node::bits_leaf>{}; + plan.fill(lpos, cpos, rpos); + plan.shuffle(cpos.shift()); + try { + return plan.merge(lpos, cpos, rpos); + } catch (...) { + cpos.each_sub(dec_visitor{}); + throw; + } +} + +template <typename Node, typename LPos, typename TPos, typename RPos> +concat_center_pos<Node> concat_leafs(LPos&& lpos, TPos&& tpos, RPos&& rpos) +{ + static_assert(Node::bits >= 2, ""); + assert(lpos.shift() == tpos.shift()); + assert(lpos.shift() == rpos.shift()); + assert(lpos.shift() == 0); + if (tpos.count() > 0) + return { + Node::bits_leaf, + lpos.node()->inc(), + lpos.count(), + tpos.node()->inc(), + tpos.count(), + rpos.node()->inc(), + rpos.count(), + }; + else + return { + Node::bits_leaf, + lpos.node()->inc(), + lpos.count(), + rpos.node()->inc(), + rpos.count(), + }; +} + +template <typename Node> +struct concat_left_visitor; +template <typename Node> +struct concat_right_visitor; +template <typename Node> +struct concat_both_visitor; + +template <typename Node, typename LPos, typename TPos, typename RPos> +concat_center_pos<Node> concat_inners(LPos&& lpos, TPos&& tpos, RPos&& rpos) +{ + auto lshift = lpos.shift(); + auto rshift = rpos.shift(); + if (lshift > rshift) { + auto cpos = lpos.last_sub(concat_left_visitor<Node>{}, tpos, rpos); + return concat_rebalance<Node>(lpos, cpos, null_sub_pos{}); + } else if (lshift < rshift) { + auto cpos = rpos.first_sub(concat_right_visitor<Node>{}, lpos, tpos); + return concat_rebalance<Node>(null_sub_pos{}, cpos, rpos); + } else { + assert(lshift == rshift); + assert(Node::bits_leaf == 0u || lshift > 0); + auto cpos = lpos.last_sub(concat_both_visitor<Node>{}, tpos, rpos); + return concat_rebalance<Node>(lpos, cpos, rpos); + } +} + +template <typename Node> +struct concat_left_visitor : visitor_base<concat_left_visitor<Node>> +{ + using this_t = concat_left_visitor; + + template <typename LPos, typename TPos, typename RPos> + static concat_center_pos<Node> + visit_inner(LPos&& lpos, TPos&& tpos, RPos&& rpos) + { + return concat_inners<Node>(lpos, tpos, rpos); + } + + template <typename LPos, typename TPos, typename RPos> + static concat_center_pos<Node> + visit_leaf(LPos&& lpos, TPos&& tpos, RPos&& rpos) + { + IMMER_UNREACHABLE; + } +}; + +template <typename Node> +struct concat_right_visitor : visitor_base<concat_right_visitor<Node>> +{ + using this_t = concat_right_visitor; + + template <typename RPos, typename LPos, typename TPos> + static concat_center_pos<Node> + visit_inner(RPos&& rpos, LPos&& lpos, TPos&& tpos) + { + return concat_inners<Node>(lpos, tpos, rpos); + } + + template <typename RPos, typename LPos, typename TPos> + static concat_center_pos<Node> + visit_leaf(RPos&& rpos, LPos&& lpos, TPos&& tpos) + { + return concat_leafs<Node>(lpos, tpos, rpos); + } +}; + +template <typename Node> +struct concat_both_visitor : visitor_base<concat_both_visitor<Node>> +{ + using this_t = concat_both_visitor; + + template <typename LPos, typename TPos, typename RPos> + static concat_center_pos<Node> + visit_inner(LPos&& lpos, TPos&& tpos, RPos&& rpos) + { + return rpos.first_sub(concat_right_visitor<Node>{}, lpos, tpos); + } + + template <typename LPos, typename TPos, typename RPos> + static concat_center_pos<Node> + visit_leaf(LPos&& lpos, TPos&& tpos, RPos&& rpos) + { + return rpos.first_sub_leaf(concat_right_visitor<Node>{}, lpos, tpos); + } +}; + +template <typename Node> +struct concat_trees_right_visitor + : visitor_base<concat_trees_right_visitor<Node>> +{ + using this_t = concat_trees_right_visitor; + + template <typename RPos, typename LPos, typename TPos> + static concat_center_pos<Node> + visit_node(RPos&& rpos, LPos&& lpos, TPos&& tpos) + { + return concat_inners<Node>(lpos, tpos, rpos); + } +}; + +template <typename Node> +struct concat_trees_left_visitor : visitor_base<concat_trees_left_visitor<Node>> +{ + using this_t = concat_trees_left_visitor; + + template <typename LPos, typename TPos, typename... Args> + static concat_center_pos<Node> + visit_node(LPos&& lpos, TPos&& tpos, Args&&... args) + { + return visit_maybe_relaxed_sub( + args..., concat_trees_right_visitor<Node>{}, lpos, tpos); + } +}; + +template <typename Node> +relaxed_pos<Node> concat_trees(Node* lroot, + shift_t lshift, + size_t lsize, + Node* ltail, + count_t ltcount, + Node* rroot, + shift_t rshift, + size_t rsize) +{ + return visit_maybe_relaxed_sub(lroot, + lshift, + lsize, + concat_trees_left_visitor<Node>{}, + make_leaf_pos(ltail, ltcount), + rroot, + rshift, + rsize) + .realize(); +} + +template <typename Node> +relaxed_pos<Node> concat_trees( + Node* ltail, count_t ltcount, Node* rroot, shift_t rshift, size_t rsize) +{ + return make_singleton_regular_sub_pos(ltail, ltcount) + .visit(concat_trees_left_visitor<Node>{}, + empty_leaf_pos<Node>{}, + rroot, + rshift, + rsize) + .realize(); +} + +template <typename Node> +using concat_center_mut_pos = concat_center_pos<Node>; + +template <typename Node> +struct concat_merger_mut +{ + using node_t = Node; + using edit_t = typename Node::edit_t; + + static constexpr auto B = Node::bits; + static constexpr auto BL = Node::bits_leaf; + + using result_t = concat_center_pos<Node>; + + edit_t ec_ = {}; + + count_t* curr_; + count_t n_; + result_t result_; + count_t count_ = 0; + node_t* candidate_ = nullptr; + edit_t candidate_e_ = Node::memory::transience_t::noone; + + concat_merger_mut(edit_t ec, + shift_t shift, + count_t* counts, + count_t n, + edit_t candidate_e, + node_t* candidate) + : ec_{ec} + , curr_{counts} + , n_{n} + , result_{shift + B, nullptr, 0} + { + if (candidate) { + candidate->ensure_mutable_relaxed_e(candidate_e, ec); + result_.nodes_[0] = candidate; + } else { + result_.nodes_[0] = node_t::make_inner_r_e(ec); + } + } + + node_t* to_ = {}; + count_t to_offset_ = {}; + size_t to_size_ = {}; + + void set_candidate(edit_t candidate_e, node_t* candidate) + { + candidate_ = candidate; + candidate_e_ = candidate_e; + } + + void add_child(node_t* p, size_t size) + { + ++curr_; + auto parent = result_.nodes_[result_.count_ - 1]; + auto relaxed = parent->relaxed(); + if (count_ == branches<B>) { + parent->relaxed()->d.count = count_; + assert(result_.count_ < result_t::max_children); + n_ -= branches<B>; + if (candidate_) { + parent = candidate_; + parent->ensure_mutable_relaxed_e(candidate_e_, ec_); + candidate_ = nullptr; + } else + parent = node_t::make_inner_r_e(ec_); + count_ = 0; + relaxed = parent->relaxed(); + result_.nodes_[result_.count_] = parent; + result_.sizes_[result_.count_] = result_.sizes_[result_.count_ - 1]; + ++result_.count_; + } + auto idx = count_++; + result_.sizes_[result_.count_ - 1] += size; + relaxed->d.sizes[idx] = size + (idx ? relaxed->d.sizes[idx - 1] : 0); + parent->inner()[idx] = p; + }; + + template <typename Pos> + void merge_leaf(Pos&& p, edit_t e) + { + auto from = p.node(); + auto from_size = p.size(); + auto from_count = p.count(); + assert(from_size); + if (!to_ && *curr_ == from_count) { + add_child(from, from_size); + } else { + auto from_offset = count_t{}; + auto from_data = from->leaf(); + auto from_mutate = from->can_mutate(e); + do { + if (!to_) { + if (from_mutate) { + node_t::ownee(from) = ec_; + to_ = from->inc(); + assert(from_count); + } else { + to_ = node_t::make_leaf_e(ec_); + } + to_offset_ = 0; + } + auto data = to_->leaf(); + auto to_copy = + std::min(from_count - from_offset, *curr_ - to_offset_); + if (from == to_) { + if (from_offset != to_offset_) + std::move(from_data + from_offset, + from_data + from_offset + to_copy, + data + to_offset_); + } else { + if (!from_mutate) + std::uninitialized_copy(from_data + from_offset, + from_data + from_offset + + to_copy, + data + to_offset_); + else + detail::uninitialized_move(from_data + from_offset, + from_data + from_offset + + to_copy, + data + to_offset_); + } + to_offset_ += to_copy; + from_offset += to_copy; + if (*curr_ == to_offset_) { + add_child(to_, to_offset_); + to_ = nullptr; + } + } while (from_offset != from_count); + } + } + + template <typename Pos> + void merge_inner(Pos&& p, edit_t e) + { + auto from = p.node(); + auto from_size = p.size(); + auto from_count = p.count(); + assert(from_size); + if (!to_ && *curr_ == from_count) { + add_child(from, from_size); + } else { + auto from_offset = count_t{}; + auto from_data = from->inner(); + auto from_mutate = from->can_relax() && from->can_mutate(e); + do { + if (!to_) { + if (from_mutate) { + node_t::ownee(from) = ec_; + from->ensure_mutable_relaxed_e(e, ec_); + to_ = from; + } else { + to_ = node_t::make_inner_r_e(ec_); + } + to_offset_ = 0; + to_size_ = 0; + } + auto data = to_->inner(); + auto to_copy = + std::min(from_count - from_offset, *curr_ - to_offset_); + auto sizes = to_->relaxed()->d.sizes; + if (from != to_ || from_offset != to_offset_) { + std::copy(from_data + from_offset, + from_data + from_offset + to_copy, + data + to_offset_); + p.copy_sizes( + from_offset, to_copy, to_size_, sizes + to_offset_); + } + to_offset_ += to_copy; + from_offset += to_copy; + to_size_ = sizes[to_offset_ - 1]; + if (*curr_ == to_offset_) { + to_->relaxed()->d.count = to_offset_; + add_child(to_, to_size_); + to_ = nullptr; + } + } while (from_offset != from_count); + } + } + + concat_center_pos<Node> finish() const + { + assert(!to_); + result_.nodes_[result_.count_ - 1]->relaxed()->d.count = count_; + return result_; + } + + void abort() + { + // We may have mutated stuff the tree in place, leaving + // everything in a corrupted state... It should be possible + // to define cleanup properly, but that is a task for some + // other day... ;) + std::terminate(); + } +}; + +struct concat_merger_mut_visitor : visitor_base<concat_merger_mut_visitor> +{ + using this_t = concat_merger_mut_visitor; + + template <typename Pos, typename Merger> + static void visit_inner(Pos&& p, Merger& merger, edit_type<Pos> e) + { + merger.merge_inner(p, e); + } + + template <typename Pos, typename Merger> + static void visit_leaf(Pos&& p, Merger& merger, edit_type<Pos> e) + { + merger.merge_leaf(p, e); + } +}; + +template <bits_t B, bits_t BL> +struct concat_rebalance_plan_mut : concat_rebalance_plan<B, BL> +{ + using this_t = concat_rebalance_plan_mut; + + template <typename LPos, typename CPos, typename RPos> + concat_center_mut_pos<node_type<CPos>> merge(edit_type<CPos> ec, + edit_type<CPos> el, + LPos&& lpos, + CPos&& cpos, + edit_type<CPos> er, + RPos&& rpos) + { + using node_t = node_type<CPos>; + using merger_t = concat_merger_mut<node_t>; + using visitor_t = concat_merger_mut_visitor; + auto lnode = ((node_t*) lpos.node()); + auto rnode = ((node_t*) rpos.node()); + auto lmut2 = lnode && lnode->can_relax() && lnode->can_mutate(el); + auto rmut2 = rnode && rnode->can_relax() && rnode->can_mutate(er); + auto merger = merger_t{ec, + cpos.shift(), + this->counts, + this->n, + el, + lmut2 ? lnode : nullptr}; + try { + lpos.each_left_sub(visitor_t{}, merger, el); + cpos.each_sub(visitor_t{}, merger, ec); + if (rmut2) + merger.set_candidate(er, rnode); + rpos.each_right_sub(visitor_t{}, merger, er); + return merger.finish(); + } catch (...) { + merger.abort(); + throw; + } + } +}; + +template <typename Node, typename LPos, typename CPos, typename RPos> +concat_center_pos<Node> concat_rebalance_mut(edit_type<Node> ec, + edit_type<Node> el, + LPos&& lpos, + CPos&& cpos, + edit_type<Node> er, + RPos&& rpos) +{ + auto plan = concat_rebalance_plan_mut<Node::bits, Node::bits_leaf>{}; + plan.fill(lpos, cpos, rpos); + plan.shuffle(cpos.shift()); + return plan.merge(ec, el, lpos, cpos, er, rpos); +} + +template <typename Node, typename LPos, typename TPos, typename RPos> +concat_center_mut_pos<Node> concat_leafs_mut(edit_type<Node> ec, + edit_type<Node> el, + LPos&& lpos, + TPos&& tpos, + edit_type<Node> er, + RPos&& rpos) +{ + static_assert(Node::bits >= 2, ""); + assert(lpos.shift() == tpos.shift()); + assert(lpos.shift() == rpos.shift()); + assert(lpos.shift() == 0); + if (tpos.count() > 0) + return { + Node::bits_leaf, + lpos.node(), + lpos.count(), + tpos.node(), + tpos.count(), + rpos.node(), + rpos.count(), + }; + else + return { + Node::bits_leaf, + lpos.node(), + lpos.count(), + rpos.node(), + rpos.count(), + }; +} + +template <typename Node> +struct concat_left_mut_visitor; +template <typename Node> +struct concat_right_mut_visitor; +template <typename Node> +struct concat_both_mut_visitor; + +template <typename Node, typename LPos, typename TPos, typename RPos> +concat_center_mut_pos<Node> concat_inners_mut(edit_type<Node> ec, + edit_type<Node> el, + LPos&& lpos, + TPos&& tpos, + edit_type<Node> er, + RPos&& rpos) +{ + auto lshift = lpos.shift(); + auto rshift = rpos.shift(); + // lpos.node() can be null it is a singleton_regular_sub_pos<...>, + // this is, when the tree is just a tail... + if (lshift > rshift) { + auto cpos = lpos.last_sub( + concat_left_mut_visitor<Node>{}, ec, el, tpos, er, rpos); + return concat_rebalance_mut<Node>( + ec, el, lpos, cpos, er, null_sub_pos{}); + } else if (lshift < rshift) { + auto cpos = rpos.first_sub( + concat_right_mut_visitor<Node>{}, ec, el, lpos, tpos, er); + return concat_rebalance_mut<Node>( + ec, el, null_sub_pos{}, cpos, er, rpos); + } else { + assert(lshift == rshift); + assert(Node::bits_leaf == 0u || lshift > 0); + auto cpos = lpos.last_sub( + concat_both_mut_visitor<Node>{}, ec, el, tpos, er, rpos); + return concat_rebalance_mut<Node>(ec, el, lpos, cpos, er, rpos); + } +} + +template <typename Node> +struct concat_left_mut_visitor : visitor_base<concat_left_mut_visitor<Node>> +{ + using this_t = concat_left_mut_visitor; + using edit_t = typename Node::edit_t; + + template <typename LPos, typename TPos, typename RPos> + static concat_center_mut_pos<Node> visit_inner( + LPos&& lpos, edit_t ec, edit_t el, TPos&& tpos, edit_t er, RPos&& rpos) + { + return concat_inners_mut<Node>(ec, el, lpos, tpos, er, rpos); + } + + template <typename LPos, typename TPos, typename RPos> + static concat_center_mut_pos<Node> visit_leaf( + LPos&& lpos, edit_t ec, edit_t el, TPos&& tpos, edit_t er, RPos&& rpos) + { + IMMER_UNREACHABLE; + } +}; + +template <typename Node> +struct concat_right_mut_visitor : visitor_base<concat_right_mut_visitor<Node>> +{ + using this_t = concat_right_mut_visitor; + using edit_t = typename Node::edit_t; + + template <typename RPos, typename LPos, typename TPos> + static concat_center_mut_pos<Node> visit_inner( + RPos&& rpos, edit_t ec, edit_t el, LPos&& lpos, TPos&& tpos, edit_t er) + { + return concat_inners_mut<Node>(ec, el, lpos, tpos, er, rpos); + } + + template <typename RPos, typename LPos, typename TPos> + static concat_center_mut_pos<Node> visit_leaf( + RPos&& rpos, edit_t ec, edit_t el, LPos&& lpos, TPos&& tpos, edit_t er) + { + return concat_leafs_mut<Node>(ec, el, lpos, tpos, er, rpos); + } +}; + +template <typename Node> +struct concat_both_mut_visitor : visitor_base<concat_both_mut_visitor<Node>> +{ + using this_t = concat_both_mut_visitor; + using edit_t = typename Node::edit_t; + + template <typename LPos, typename TPos, typename RPos> + static concat_center_mut_pos<Node> visit_inner( + LPos&& lpos, edit_t ec, edit_t el, TPos&& tpos, edit_t er, RPos&& rpos) + { + return rpos.first_sub( + concat_right_mut_visitor<Node>{}, ec, el, lpos, tpos, er); + } + + template <typename LPos, typename TPos, typename RPos> + static concat_center_mut_pos<Node> visit_leaf( + LPos&& lpos, edit_t ec, edit_t el, TPos&& tpos, edit_t er, RPos&& rpos) + { + return rpos.first_sub_leaf( + concat_right_mut_visitor<Node>{}, ec, el, lpos, tpos, er); + } +}; + +template <typename Node> +struct concat_trees_right_mut_visitor + : visitor_base<concat_trees_right_mut_visitor<Node>> +{ + using this_t = concat_trees_right_mut_visitor; + using edit_t = typename Node::edit_t; + + template <typename RPos, typename LPos, typename TPos> + static concat_center_mut_pos<Node> visit_node( + RPos&& rpos, edit_t ec, edit_t el, LPos&& lpos, TPos&& tpos, edit_t er) + { + return concat_inners_mut<Node>(ec, el, lpos, tpos, er, rpos); + } +}; + +template <typename Node> +struct concat_trees_left_mut_visitor + : visitor_base<concat_trees_left_mut_visitor<Node>> +{ + using this_t = concat_trees_left_mut_visitor; + using edit_t = typename Node::edit_t; + + template <typename LPos, typename TPos, typename... Args> + static concat_center_mut_pos<Node> visit_node(LPos&& lpos, + edit_t ec, + edit_t el, + TPos&& tpos, + edit_t er, + Args&&... args) + { + return visit_maybe_relaxed_sub(args..., + concat_trees_right_mut_visitor<Node>{}, + ec, + el, + lpos, + tpos, + er); + } +}; + +template <typename Node> +relaxed_pos<Node> concat_trees_mut(edit_type<Node> ec, + edit_type<Node> el, + Node* lroot, + shift_t lshift, + size_t lsize, + Node* ltail, + count_t ltcount, + edit_type<Node> er, + Node* rroot, + shift_t rshift, + size_t rsize) +{ + return visit_maybe_relaxed_sub(lroot, + lshift, + lsize, + concat_trees_left_mut_visitor<Node>{}, + ec, + el, + make_leaf_pos(ltail, ltcount), + er, + rroot, + rshift, + rsize) + .realize_e(ec); +} + +template <typename Node> +relaxed_pos<Node> concat_trees_mut(edit_type<Node> ec, + edit_type<Node> el, + Node* ltail, + count_t ltcount, + edit_type<Node> er, + Node* rroot, + shift_t rshift, + size_t rsize) +{ + return make_singleton_regular_sub_pos(ltail, ltcount) + .visit(concat_trees_left_mut_visitor<Node>{}, + ec, + el, + empty_leaf_pos<Node>{}, + er, + rroot, + rshift, + rsize) + .realize_e(ec); +} + +} // namespace rbts +} // namespace detail +} // namespace immer diff --git a/immer/detail/rbts/position.hpp b/immer/detail/rbts/position.hpp new file mode 100644 index 000000000000..e9472b294088 --- /dev/null +++ b/immer/detail/rbts/position.hpp @@ -0,0 +1,1977 @@ +// +// immer: immutable data structures for C++ +// Copyright (C) 2016, 2017, 2018 Juan Pedro Bolivar Puente +// +// This software is distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE or copy at http://boost.org/LICENSE_1_0.txt +// + +#pragma once + +#include <immer/config.hpp> +#include <immer/detail/rbts/bits.hpp> + +#include <cassert> +#include <type_traits> +#include <utility> + +namespace immer { +namespace detail { +namespace rbts { + +template <typename Pos> +constexpr auto bits = std::decay_t<Pos>::node_t::bits; + +template <typename Pos> +constexpr auto bits_leaf = std::decay_t<Pos>::node_t::bits_leaf; + +template <typename Pos> +using node_type = typename std::decay<Pos>::type::node_t; + +template <typename Pos> +using edit_type = typename std::decay<Pos>::type::node_t::edit_t; + +template <typename NodeT> +struct empty_regular_pos +{ + using node_t = NodeT; + node_t* node_; + + count_t count() const { return 0; } + node_t* node() const { return node_; } + shift_t shift() const { return 0; } + size_t size() const { return 0; } + + template <typename Visitor, typename... Args> + void each(Visitor, Args&&...) + {} + template <typename Visitor, typename... Args> + bool each_pred(Visitor, Args&&...) + { + return true; + } + + template <typename Visitor, typename... Args> + decltype(auto) visit(Visitor v, Args&&... args) + { + return Visitor::visit_regular(*this, std::forward<Args>(args)...); + } +}; + +template <typename NodeT> +empty_regular_pos<NodeT> make_empty_regular_pos(NodeT* node) +{ + return {node}; +} + +template <typename NodeT> +struct empty_leaf_pos +{ + using node_t = NodeT; + node_t* node_; + + count_t count() const { return 0; } + node_t* node() const { return node_; } + shift_t shift() const { return 0; } + size_t size() const { return 0; } + + template <typename Visitor, typename... Args> + decltype(auto) visit(Visitor v, Args&&... args) + { + return Visitor::visit_leaf(*this, std::forward<Args>(args)...); + } +}; + +template <typename NodeT> +empty_leaf_pos<NodeT> make_empty_leaf_pos(NodeT* node) +{ + assert(node); + return {node}; +} + +template <typename NodeT> +struct leaf_pos +{ + static constexpr auto B = NodeT::bits; + static constexpr auto BL = NodeT::bits_leaf; + + using node_t = NodeT; + node_t* node_; + size_t size_; + + count_t count() const { return index(size_ - 1) + 1; } + node_t* node() const { return node_; } + size_t size() const { return size_; } + shift_t shift() const { return 0; } + count_t index(size_t idx) const { return idx & mask<BL>; } + count_t subindex(size_t idx) const { return idx; } + + template <typename Visitor, typename... Args> + decltype(auto) visit(Visitor v, Args&&... args) + { + return Visitor::visit_leaf(*this, std::forward<Args>(args)...); + } +}; + +template <typename NodeT> +leaf_pos<NodeT> make_leaf_pos(NodeT* node, size_t size) +{ + assert(node); + assert(size > 0); + return {node, size}; +} + +template <typename NodeT> +struct leaf_sub_pos +{ + static constexpr auto B = NodeT::bits; + static constexpr auto BL = NodeT::bits_leaf; + + using node_t = NodeT; + node_t* node_; + count_t count_; + + count_t count() const { return count_; } + node_t* node() const { return node_; } + size_t size() const { return count_; } + shift_t shift() const { return 0; } + count_t index(size_t idx) const { return idx & mask<BL>; } + count_t subindex(size_t idx) const { return idx; } + + template <typename Visitor, typename... Args> + decltype(auto) visit(Visitor v, Args&&... args) + { + return Visitor::visit_leaf(*this, std::forward<Args>(args)...); + } +}; + +template <typename NodeT> +leaf_sub_pos<NodeT> make_leaf_sub_pos(NodeT* node, count_t count) +{ + assert(node); + assert(count <= branches<NodeT::bits_leaf>); + return {node, count}; +} + +template <typename NodeT> +struct leaf_descent_pos +{ + static constexpr auto B = NodeT::bits; + static constexpr auto BL = NodeT::bits_leaf; + + using node_t = NodeT; + node_t* node_; + + node_t* node() const { return node_; } + shift_t shift() const { return 0; } + count_t index(size_t idx) const { return idx & mask<BL>; } + + template <typename... Args> + decltype(auto) descend(Args&&...) + {} + + template <typename Visitor, typename... Args> + decltype(auto) visit(Visitor v, Args&&... args) + { + return Visitor::visit_leaf(*this, std::forward<Args>(args)...); + } +}; + +template <typename NodeT> +leaf_descent_pos<NodeT> make_leaf_descent_pos(NodeT* node) +{ + assert(node); + return {node}; +} + +template <typename NodeT> +struct full_leaf_pos +{ + static constexpr auto B = NodeT::bits; + static constexpr auto BL = NodeT::bits_leaf; + + using node_t = NodeT; + node_t* node_; + + count_t count() const { return branches<BL>; } + node_t* node() const { return node_; } + size_t size() const { return branches<BL>; } + shift_t shift() const { return 0; } + count_t index(size_t idx) const { return idx & mask<BL>; } + count_t subindex(size_t idx) const { return idx; } + + template <typename Visitor, typename... Args> + decltype(auto) visit(Visitor v, Args&&... args) + { + return Visitor::visit_leaf(*this, std::forward<Args>(args)...); + } +}; + +template <typename NodeT> +full_leaf_pos<NodeT> make_full_leaf_pos(NodeT* node) +{ + assert(node); + return {node}; +} + +template <typename NodeT> +struct regular_pos +{ + static constexpr auto B = NodeT::bits; + static constexpr auto BL = NodeT::bits_leaf; + + using node_t = NodeT; + node_t* node_; + shift_t shift_; + size_t size_; + + count_t count() const { return index(size_ - 1) + 1; } + node_t* node() const { return node_; } + size_t size() const { return size_; } + shift_t shift() const { return shift_; } + count_t index(size_t idx) const { return (idx >> shift_) & mask<B>; } + count_t subindex(size_t idx) const { return idx >> shift_; } + size_t this_size() const + { + return ((size_ - 1) & ~(~size_t{} << (shift_ + B))) + 1; + } + + template <typename Visitor, typename... Args> + void each(Visitor v, Args&&... args) + { + return each_regular(*this, v, args...); + } + + template <typename Visitor, typename... Args> + bool each_pred(Visitor v, Args&&... args) + { + return each_pred_regular(*this, v, args...); + } + + template <typename Visitor, typename... Args> + bool each_pred_zip(Visitor v, node_t* other, Args&&... args) + { + return each_pred_zip_regular(*this, v, other, args...); + } + + template <typename Visitor, typename... Args> + bool each_pred_i(Visitor v, count_t i, count_t n, Args&&... args) + { + return each_pred_i_regular(*this, v, i, n, args...); + } + + template <typename Visitor, typename... Args> + bool each_pred_right(Visitor v, count_t start, Args&&... args) + { + return each_pred_right_regular(*this, v, start, args...); + } + + template <typename Visitor, typename... Args> + bool each_pred_left(Visitor v, count_t n, Args&&... args) + { + return each_pred_left_regular(*this, v, n, args...); + } + + template <typename Visitor, typename... Args> + void each_i(Visitor v, count_t i, count_t n, Args&&... args) + { + return each_i_regular(*this, v, i, n, args...); + } + + template <typename Visitor, typename... Args> + void each_right(Visitor v, count_t start, Args&&... args) + { + return each_right_regular(*this, v, start, args...); + } + + template <typename Visitor, typename... Args> + void each_left(Visitor v, count_t n, Args&&... args) + { + return each_left_regular(*this, v, n, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) towards(Visitor v, size_t idx, Args&&... args) + { + return towards_oh_ch_regular( + *this, v, idx, index(idx), count(), args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) + towards_oh(Visitor v, size_t idx, count_t offset_hint, Args&&... args) + { + return towards_oh_ch_regular( + *this, v, idx, offset_hint, count(), args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) towards_oh_ch(Visitor v, + size_t idx, + count_t offset_hint, + count_t count_hint, + Args&&... args) + { + return towards_oh_ch_regular( + *this, v, idx, offset_hint, count(), args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) + towards_sub_oh(Visitor v, size_t idx, count_t offset_hint, Args&&... args) + { + return towards_sub_oh_regular(*this, v, idx, offset_hint, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) last_oh(Visitor v, count_t offset_hint, Args&&... args) + { + return last_oh_regular(*this, v, offset_hint, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) visit(Visitor v, Args&&... args) + { + return Visitor::visit_regular(*this, std::forward<Args>(args)...); + } +}; + +template <typename Pos, typename Visitor, typename... Args> +void each_regular(Pos&& p, Visitor v, Args&&... args) +{ + constexpr auto B = bits<Pos>; + constexpr auto BL = bits_leaf<Pos>; + auto n = p.node()->inner(); + auto last = p.count() - 1; + auto e = n + last; + if (p.shift() == BL) { + for (; n != e; ++n) { + IMMER_PREFETCH(n + 1); + make_full_leaf_pos(*n).visit(v, args...); + } + make_leaf_pos(*n, p.size()).visit(v, args...); + } else { + auto ss = p.shift() - B; + for (; n != e; ++n) + make_full_pos(*n, ss).visit(v, args...); + make_regular_pos(*n, ss, p.size()).visit(v, args...); + } +} + +template <typename Pos, typename Visitor, typename... Args> +bool each_pred_regular(Pos&& p, Visitor v, Args&&... args) +{ + constexpr auto B = bits<Pos>; + constexpr auto BL = bits_leaf<Pos>; + auto n = p.node()->inner(); + auto last = p.count() - 1; + auto e = n + last; + if (p.shift() == BL) { + for (; n != e; ++n) { + IMMER_PREFETCH(n + 1); + if (!make_full_leaf_pos(*n).visit(v, args...)) + return false; + } + return make_leaf_pos(*n, p.size()).visit(v, args...); + } else { + auto ss = p.shift() - B; + for (; n != e; ++n) + if (!make_full_pos(*n, ss).visit(v, args...)) + return false; + return make_regular_pos(*n, ss, p.size()).visit(v, args...); + } +} + +template <typename Pos, typename Visitor, typename... Args> +bool each_pred_zip_regular(Pos&& p, + Visitor v, + node_type<Pos>* other, + Args&&... args) +{ + constexpr auto B = bits<Pos>; + constexpr auto BL = bits_leaf<Pos>; + + auto n = p.node()->inner(); + auto n2 = other->inner(); + auto last = p.count() - 1; + auto e = n + last; + if (p.shift() == BL) { + for (; n != e; ++n, ++n2) { + IMMER_PREFETCH(n + 1); + IMMER_PREFETCH(n2 + 1); + if (!make_full_leaf_pos(*n).visit(v, *n2, args...)) + return false; + } + return make_leaf_pos(*n, p.size()).visit(v, *n2, args...); + } else { + auto ss = p.shift() - B; + for (; n != e; ++n, ++n2) + if (!make_full_pos(*n, ss).visit(v, *n2, args...)) + return false; + return make_regular_pos(*n, ss, p.size()).visit(v, *n2, args...); + } +} + +template <typename Pos, typename Visitor, typename... Args> +bool each_pred_i_regular( + Pos&& p, Visitor v, count_t f, count_t l, Args&&... args) +{ + constexpr auto B = bits<Pos>; + constexpr auto BL = bits_leaf<Pos>; + + if (p.shift() == BL) { + if (l > f) { + if (l < p.count()) { + auto n = p.node()->inner() + f; + auto e = p.node()->inner() + l; + for (; n < e; ++n) { + IMMER_PREFETCH(n + 1); + if (!make_full_leaf_pos(*n).visit(v, args...)) + return false; + } + } else { + auto n = p.node()->inner() + f; + auto e = p.node()->inner() + l - 1; + for (; n < e; ++n) { + IMMER_PREFETCH(n + 1); + if (!make_full_leaf_pos(*n).visit(v, args...)) + return false; + } + if (!make_leaf_pos(*n, p.size()).visit(v, args...)) + return false; + } + } + } else { + if (l > f) { + auto ss = p.shift() - B; + if (l < p.count()) { + auto n = p.node()->inner() + f; + auto e = p.node()->inner() + l; + for (; n < e; ++n) + if (!make_full_pos(*n, ss).visit(v, args...)) + return false; + } else { + auto n = p.node()->inner() + f; + auto e = p.node()->inner() + l - 1; + for (; n < e; ++n) + if (!make_full_pos(*n, ss).visit(v, args...)) + return false; + if (!make_regular_pos(*n, ss, p.size()).visit(v, args...)) + return false; + } + } + } + return true; +} + +template <typename Pos, typename Visitor, typename... Args> +bool each_pred_left_regular(Pos&& p, Visitor v, count_t last, Args&&... args) +{ + constexpr auto B = bits<Pos>; + constexpr auto BL = bits_leaf<Pos>; + assert(last < p.count()); + if (p.shift() == BL) { + auto n = p.node()->inner(); + auto e = n + last; + for (; n != e; ++n) { + IMMER_PREFETCH(n + 1); + if (!make_full_leaf_pos(*n).visit(v, args...)) + return false; + } + } else { + auto n = p.node()->inner(); + auto e = n + last; + auto ss = p.shift() - B; + for (; n != e; ++n) + if (!make_full_pos(*n, ss).visit(v, args...)) + return false; + } + return true; +} + +template <typename Pos, typename Visitor, typename... Args> +bool each_pred_right_regular(Pos&& p, Visitor v, count_t start, Args&&... args) +{ + constexpr auto B = bits<Pos>; + constexpr auto BL = bits_leaf<Pos>; + + if (p.shift() == BL) { + auto n = p.node()->inner() + start; + auto last = p.count() - 1; + auto e = p.node()->inner() + last; + if (n <= e) { + for (; n != e; ++n) { + IMMER_PREFETCH(n + 1); + if (!make_full_leaf_pos(*n).visit(v, args...)) + return false; + } + if (!make_leaf_pos(*n, p.size()).visit(v, args...)) + return false; + } + } else { + auto n = p.node()->inner() + start; + auto last = p.count() - 1; + auto e = p.node()->inner() + last; + auto ss = p.shift() - B; + if (n <= e) { + for (; n != e; ++n) + if (!make_full_pos(*n, ss).visit(v, args...)) + return false; + if (!make_regular_pos(*n, ss, p.size()).visit(v, args...)) + return false; + } + } + return true; +} + +template <typename Pos, typename Visitor, typename... Args> +void each_i_regular(Pos&& p, Visitor v, count_t f, count_t l, Args&&... args) +{ + constexpr auto B = bits<Pos>; + constexpr auto BL = bits_leaf<Pos>; + + if (p.shift() == BL) { + if (l > f) { + if (l < p.count()) { + auto n = p.node()->inner() + f; + auto e = p.node()->inner() + l; + for (; n < e; ++n) { + IMMER_PREFETCH(n + 1); + make_full_leaf_pos(*n).visit(v, args...); + } + } else { + auto n = p.node()->inner() + f; + auto e = p.node()->inner() + l - 1; + for (; n < e; ++n) { + IMMER_PREFETCH(n + 1); + make_full_leaf_pos(*n).visit(v, args...); + } + make_leaf_pos(*n, p.size()).visit(v, args...); + } + } + } else { + if (l > f) { + auto ss = p.shift() - B; + if (l < p.count()) { + auto n = p.node()->inner() + f; + auto e = p.node()->inner() + l; + for (; n < e; ++n) + make_full_pos(*n, ss).visit(v, args...); + } else { + auto n = p.node()->inner() + f; + auto e = p.node()->inner() + l - 1; + for (; n < e; ++n) + make_full_pos(*n, ss).visit(v, args...); + make_regular_pos(*n, ss, p.size()).visit(v, args...); + } + } + } +} + +template <typename Pos, typename Visitor, typename... Args> +void each_left_regular(Pos&& p, Visitor v, count_t last, Args&&... args) +{ + constexpr auto B = bits<Pos>; + constexpr auto BL = bits_leaf<Pos>; + assert(last < p.count()); + if (p.shift() == BL) { + auto n = p.node()->inner(); + auto e = n + last; + for (; n != e; ++n) { + IMMER_PREFETCH(n + 1); + make_full_leaf_pos(*n).visit(v, args...); + } + } else { + auto n = p.node()->inner(); + auto e = n + last; + auto ss = p.shift() - B; + for (; n != e; ++n) + make_full_pos(*n, ss).visit(v, args...); + } +} + +template <typename Pos, typename Visitor, typename... Args> +void each_right_regular(Pos&& p, Visitor v, count_t start, Args&&... args) +{ + constexpr auto B = bits<Pos>; + constexpr auto BL = bits_leaf<Pos>; + + if (p.shift() == BL) { + auto n = p.node()->inner() + start; + auto last = p.count() - 1; + auto e = p.node()->inner() + last; + if (n <= e) { + for (; n != e; ++n) { + IMMER_PREFETCH(n + 1); + make_full_leaf_pos(*n).visit(v, args...); + } + make_leaf_pos(*n, p.size()).visit(v, args...); + } + } else { + auto n = p.node()->inner() + start; + auto last = p.count() - 1; + auto e = p.node()->inner() + last; + auto ss = p.shift() - B; + if (n <= e) { + for (; n != e; ++n) + make_full_pos(*n, ss).visit(v, args...); + make_regular_pos(*n, ss, p.size()).visit(v, args...); + } + } +} + +template <typename Pos, typename Visitor, typename... Args> +decltype(auto) towards_oh_ch_regular(Pos&& p, + Visitor v, + size_t idx, + count_t offset_hint, + count_t count_hint, + Args&&... args) +{ + constexpr auto B = bits<Pos>; + constexpr auto BL = bits_leaf<Pos>; + assert(offset_hint == p.index(idx)); + assert(count_hint == p.count()); + auto is_leaf = p.shift() == BL; + auto child = p.node()->inner()[offset_hint]; + auto is_full = offset_hint + 1 != count_hint; + return is_full + ? (is_leaf ? make_full_leaf_pos(child).visit(v, idx, args...) + : make_full_pos(child, p.shift() - B) + .visit(v, idx, args...)) + : (is_leaf + ? make_leaf_pos(child, p.size()).visit(v, idx, args...) + : make_regular_pos(child, p.shift() - B, p.size()) + .visit(v, idx, args...)); +} + +template <typename Pos, typename Visitor, typename... Args> +decltype(auto) towards_sub_oh_regular( + Pos&& p, Visitor v, size_t idx, count_t offset_hint, Args&&... args) +{ + constexpr auto B = bits<Pos>; + constexpr auto BL = bits_leaf<Pos>; + assert(offset_hint == p.index(idx)); + auto is_leaf = p.shift() == BL; + auto child = p.node()->inner()[offset_hint]; + auto lsize = offset_hint << p.shift(); + auto size = p.this_size(); + auto is_full = (size - lsize) >= (size_t{1} << p.shift()); + return is_full + ? (is_leaf + ? make_full_leaf_pos(child).visit(v, idx - lsize, args...) + : make_full_pos(child, p.shift() - B) + .visit(v, idx - lsize, args...)) + : (is_leaf + ? make_leaf_sub_pos(child, size - lsize) + .visit(v, idx - lsize, args...) + : make_regular_sub_pos(child, p.shift() - B, size - lsize) + .visit(v, idx - lsize, args...)); +} + +template <typename Pos, typename Visitor, typename... Args> +decltype(auto) +last_oh_regular(Pos&& p, Visitor v, count_t offset_hint, Args&&... args) +{ + assert(offset_hint == p.count() - 1); + constexpr auto B = bits<Pos>; + constexpr auto BL = bits_leaf<Pos>; + auto child = p.node()->inner()[offset_hint]; + auto is_leaf = p.shift() == BL; + return is_leaf ? make_leaf_pos(child, p.size()).visit(v, args...) + : make_regular_pos(child, p.shift() - B, p.size()) + .visit(v, args...); +} + +template <typename NodeT> +regular_pos<NodeT> make_regular_pos(NodeT* node, shift_t shift, size_t size) +{ + assert(node); + assert(shift >= NodeT::bits_leaf); + assert(size > 0); + return {node, shift, size}; +} + +struct null_sub_pos +{ + auto node() const { return nullptr; } + + template <typename Visitor, typename... Args> + void each_sub(Visitor, Args&&...) + {} + template <typename Visitor, typename... Args> + void each_right_sub(Visitor, Args&&...) + {} + template <typename Visitor, typename... Args> + void each_left_sub(Visitor, Args&&...) + {} + template <typename Visitor, typename... Args> + void visit(Visitor, Args&&...) + {} +}; + +template <typename NodeT> +struct singleton_regular_sub_pos +{ + // this is a fake regular pos made out of a single child... useful + // to treat a single leaf node as a whole tree + + static constexpr auto B = NodeT::bits; + static constexpr auto BL = NodeT::bits_leaf; + + using node_t = NodeT; + node_t* leaf_; + count_t count_; + + count_t count() const { return 1; } + node_t* node() const { return nullptr; } + size_t size() const { return count_; } + shift_t shift() const { return BL; } + count_t index(size_t idx) const { return 0; } + count_t subindex(size_t idx) const { return 0; } + size_t size_before(count_t offset) const { return 0; } + size_t this_size() const { return count_; } + size_t size(count_t offset) { return count_; } + + template <typename Visitor, typename... Args> + void each_left_sub(Visitor v, Args&&... args) + {} + template <typename Visitor, typename... Args> + void each(Visitor v, Args&&... args) + {} + + template <typename Visitor, typename... Args> + decltype(auto) last_sub(Visitor v, Args&&... args) + { + return make_leaf_sub_pos(leaf_, count_).visit(v, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) visit(Visitor v, Args&&... args) + { + return Visitor::visit_regular(*this, std::forward<Args>(args)...); + } +}; + +template <typename NodeT> +auto make_singleton_regular_sub_pos(NodeT* leaf, count_t count) +{ + assert(leaf); + IMMER_ASSERT_TAGGED(leaf->kind() == NodeT::kind_t::leaf); + assert(count > 0); + return singleton_regular_sub_pos<NodeT>{leaf, count}; +} + +template <typename NodeT> +struct regular_sub_pos +{ + static constexpr auto B = NodeT::bits; + static constexpr auto BL = NodeT::bits_leaf; + + using node_t = NodeT; + node_t* node_; + shift_t shift_; + size_t size_; + + count_t count() const { return subindex(size_ - 1) + 1; } + node_t* node() const { return node_; } + size_t size() const { return size_; } + shift_t shift() const { return shift_; } + count_t index(size_t idx) const { return (idx >> shift_) & mask<B>; } + count_t subindex(size_t idx) const { return idx >> shift_; } + size_t size_before(count_t offset) const { return offset << shift_; } + size_t this_size() const { return size_; } + + auto size(count_t offset) + { + return offset == subindex(size_ - 1) ? size_ - size_before(offset) + : size_t{1} << shift_; + } + + auto size_sbh(count_t offset, size_t size_before_hint) + { + assert(size_before_hint == size_before(offset)); + return offset == subindex(size_ - 1) ? size_ - size_before_hint + : size_t{1} << shift_; + } + + void copy_sizes(count_t offset, count_t n, size_t init, size_t* sizes) + { + if (n) { + auto last = offset + n - 1; + auto e = sizes + n - 1; + for (; sizes != e; ++sizes) + init = *sizes = init + (size_t{1} << shift_); + *sizes = init + size(last); + } + } + + template <typename Visitor, typename... Args> + void each(Visitor v, Args&&... args) + { + return each_regular(*this, v, args...); + } + + template <typename Visitor, typename... Args> + bool each_pred(Visitor v, Args&&... args) + { + return each_pred_regular(*this, v, args...); + } + + template <typename Visitor, typename... Args> + bool each_pred_zip(Visitor v, node_t* other, Args&&... args) + { + return each_pred_zip_regular(*this, v, other, args...); + } + + template <typename Visitor, typename... Args> + bool each_pred_i(Visitor v, count_t i, count_t n, Args&&... args) + { + return each_pred_i_regular(*this, v, i, n, args...); + } + + template <typename Visitor, typename... Args> + bool each_pred_right(Visitor v, count_t start, Args&&... args) + { + return each_pred_right_regular(*this, v, start, args...); + } + + template <typename Visitor, typename... Args> + bool each_pred_left(Visitor v, count_t last, Args&&... args) + { + return each_pred_left_regular(*this, v, last, args...); + } + + template <typename Visitor, typename... Args> + void each_i(Visitor v, count_t i, count_t n, Args&&... args) + { + return each_i_regular(*this, v, i, n, args...); + } + + template <typename Visitor, typename... Args> + void each_right(Visitor v, count_t start, Args&&... args) + { + return each_right_regular(*this, v, start, args...); + } + + template <typename Visitor, typename... Args> + void each_left(Visitor v, count_t last, Args&&... args) + { + return each_left_regular(*this, v, last, args...); + } + + template <typename Visitor, typename... Args> + void each_right_sub_(Visitor v, count_t i, Args&&... args) + { + auto last = count() - 1; + auto lsize = size_ - (last << shift_); + auto n = node()->inner() + i; + auto e = node()->inner() + last; + if (shift() == BL) { + for (; n != e; ++n) { + IMMER_PREFETCH(n + 1); + make_full_leaf_pos(*n).visit(v, args...); + } + make_leaf_sub_pos(*n, lsize).visit(v, args...); + } else { + auto ss = shift_ - B; + for (; n != e; ++n) + make_full_pos(*n, ss).visit(v, args...); + make_regular_sub_pos(*n, ss, lsize).visit(v, args...); + } + } + + template <typename Visitor, typename... Args> + void each_sub(Visitor v, Args&&... args) + { + each_right_sub_(v, 0, args...); + } + + template <typename Visitor, typename... Args> + void each_right_sub(Visitor v, Args&&... args) + { + if (count() > 1) + each_right_sub_(v, 1, args...); + } + + template <typename Visitor, typename... Args> + void each_left_sub(Visitor v, Args&&... args) + { + each_left(v, count() - 1, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) towards(Visitor v, size_t idx, Args&&... args) + { + return towards_oh_ch_regular( + *this, v, idx, index(idx), count(), args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) + towards_oh(Visitor v, size_t idx, count_t offset_hint, Args&&... args) + { + return towards_oh_ch_regular( + *this, v, idx, offset_hint, count(), args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) towards_oh_ch(Visitor v, + size_t idx, + count_t offset_hint, + count_t count_hint, + Args&&... args) + { + return towards_oh_ch_regular( + *this, v, idx, offset_hint, count(), args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) + towards_sub_oh(Visitor v, size_t idx, count_t offset_hint, Args&&... args) + { + return towards_sub_oh_regular(*this, v, idx, offset_hint, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) last_oh(Visitor v, count_t offset_hint, Args&&... args) + { + return last_oh_regular(*this, v, offset_hint, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) last_sub(Visitor v, Args&&... args) + { + auto offset = count() - 1; + auto child = node_->inner()[offset]; + auto is_leaf = shift_ == BL; + auto lsize = size_ - (offset << shift_); + return is_leaf ? make_leaf_sub_pos(child, lsize).visit(v, args...) + : make_regular_sub_pos(child, shift_ - B, lsize) + .visit(v, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) first_sub(Visitor v, Args&&... args) + { + auto is_leaf = shift_ == BL; + auto child = node_->inner()[0]; + auto is_full = size_ >= (size_t{1} << shift_); + return is_full + ? (is_leaf + ? make_full_leaf_pos(child).visit(v, args...) + : make_full_pos(child, shift_ - B).visit(v, args...)) + : (is_leaf + ? make_leaf_sub_pos(child, size_).visit(v, args...) + : make_regular_sub_pos(child, shift_ - B, size_) + .visit(v, args...)); + } + + template <typename Visitor, typename... Args> + decltype(auto) first_sub_leaf(Visitor v, Args&&... args) + { + assert(shift_ == BL); + auto child = node_->inner()[0]; + auto is_full = size_ >= branches<BL>; + return is_full ? make_full_leaf_pos(child).visit(v, args...) + : make_leaf_sub_pos(child, size_).visit(v, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) first_sub_inner(Visitor v, Args&&... args) + { + assert(shift_ >= BL); + auto child = node_->inner()[0]; + auto is_full = size_ >= branches<BL>; + return is_full ? make_full_pos(child, shift_ - B).visit(v, args...) + : make_regular_sub_pos(child, shift_ - B, size_) + .visit(v, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) nth_sub(count_t idx, Visitor v, Args&&... args) + { + assert(idx < count()); + auto is_leaf = shift_ == BL; + auto child = node_->inner()[idx]; + auto lsize = size(idx); + auto is_full = idx + 1 < count(); + return is_full + ? (is_leaf + ? make_full_leaf_pos(child).visit(v, args...) + : make_full_pos(child, shift_ - B).visit(v, args...)) + : (is_leaf + ? make_leaf_sub_pos(child, lsize).visit(v, args...) + : make_regular_sub_pos(child, shift_ - B, lsize) + .visit(v, args...)); + } + + template <typename Visitor, typename... Args> + decltype(auto) nth_sub_leaf(count_t idx, Visitor v, Args&&... args) + { + assert(shift_ == BL); + auto child = node_->inner()[idx]; + auto lsize = size(idx); + auto is_full = idx + 1 < count(); + return is_full ? make_full_leaf_pos(child).visit(v, args...) + : make_leaf_sub_pos(child, lsize).visit(v, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) visit(Visitor v, Args&&... args) + { + return Visitor::visit_regular(*this, std::forward<Args>(args)...); + } +}; + +template <typename NodeT> +regular_sub_pos<NodeT> +make_regular_sub_pos(NodeT* node, shift_t shift, size_t size) +{ + assert(node); + assert(shift >= NodeT::bits_leaf); + assert(size > 0); + assert(size <= (branches<NodeT::bits, size_t> << shift)); + return {node, shift, size}; +} + +template <typename NodeT, + shift_t Shift, + bits_t B = NodeT::bits, + bits_t BL = NodeT::bits_leaf> +struct regular_descent_pos +{ + static_assert(Shift > 0, "not leaf..."); + + using node_t = NodeT; + node_t* node_; + + node_t* node() const { return node_; } + shift_t shift() const { return Shift; } + count_t index(size_t idx) const + { +#if !defined(_MSC_VER) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wshift-count-overflow" +#endif + return (idx >> Shift) & mask<B>; +#if !defined(_MSC_VER) +#pragma GCC diagnostic pop +#endif + } + + template <typename Visitor> + decltype(auto) descend(Visitor v, size_t idx) + { + auto offset = index(idx); + auto child = node_->inner()[offset]; + return regular_descent_pos<NodeT, Shift - B>{child}.visit(v, idx); + } + + template <typename Visitor, typename... Args> + decltype(auto) visit(Visitor v, Args&&... args) + { + return Visitor::visit_regular(*this, std::forward<Args>(args)...); + } +}; + +template <typename NodeT, bits_t B, bits_t BL> +struct regular_descent_pos<NodeT, BL, B, BL> +{ + using node_t = NodeT; + node_t* node_; + + node_t* node() const { return node_; } + shift_t shift() const { return BL; } + count_t index(size_t idx) const { return (idx >> BL) & mask<B>; } + + template <typename Visitor> + decltype(auto) descend(Visitor v, size_t idx) + { + auto offset = index(idx); + auto child = node_->inner()[offset]; + return make_leaf_descent_pos(child).visit(v, idx); + } + + template <typename Visitor, typename... Args> + decltype(auto) visit(Visitor v, Args&&... args) + { + return Visitor::visit_regular(*this, std::forward<Args>(args)...); + } +}; + +template <typename NodeT, typename Visitor> +decltype(auto) +visit_regular_descent(NodeT* node, shift_t shift, Visitor v, size_t idx) +{ + constexpr auto B = NodeT::bits; + constexpr auto BL = NodeT::bits_leaf; + assert(node); + assert(shift >= BL); + switch (shift) { + case BL + B * 0: + return regular_descent_pos<NodeT, BL + B * 0>{node}.visit(v, idx); + case BL + B * 1: + return regular_descent_pos<NodeT, BL + B * 1>{node}.visit(v, idx); + case BL + B * 2: + return regular_descent_pos<NodeT, BL + B * 2>{node}.visit(v, idx); + case BL + B * 3: + return regular_descent_pos<NodeT, BL + B * 3>{node}.visit(v, idx); + case BL + B * 4: + return regular_descent_pos<NodeT, BL + B * 4>{node}.visit(v, idx); + case BL + B * 5: + return regular_descent_pos<NodeT, BL + B * 5>{node}.visit(v, idx); +#if IMMER_DESCENT_DEEP + default: + for (auto level = shift; level != endshift<B, BL>; level -= B) + node = node->inner()[(idx >> level) & mask<B>]; + return make_leaf_descent_pos(node).visit(v, idx); +#endif // IMMER_DEEP_DESCENT + } + IMMER_UNREACHABLE; +} + +template <typename NodeT> +struct full_pos +{ + static constexpr auto B = NodeT::bits; + static constexpr auto BL = NodeT::bits_leaf; + + using node_t = NodeT; + node_t* node_; + shift_t shift_; + + count_t count() const { return branches<B>; } + node_t* node() const { return node_; } + size_t size() const { return branches<B> << shift_; } + shift_t shift() const { return shift_; } + count_t index(size_t idx) const { return (idx >> shift_) & mask<B>; } + count_t subindex(size_t idx) const { return idx >> shift_; } + size_t size(count_t offset) const { return size_t{1} << shift_; } + size_t size_sbh(count_t offset, size_t) const + { + return size_t{1} << shift_; + } + size_t size_before(count_t offset) const { return offset << shift_; } + + void copy_sizes(count_t offset, count_t n, size_t init, size_t* sizes) + { + auto e = sizes + n; + for (; sizes != e; ++sizes) + init = *sizes = init + (size_t{1} << shift_); + } + + template <typename Visitor, typename... Args> + void each(Visitor v, Args&&... args) + { + auto p = node_->inner(); + auto e = p + branches<B>; + if (shift_ == BL) { + for (; p != e; ++p) { + IMMER_PREFETCH(p + 1); + make_full_leaf_pos(*p).visit(v, args...); + } + } else { + auto ss = shift_ - B; + for (; p != e; ++p) + make_full_pos(*p, ss).visit(v, args...); + } + } + + template <typename Visitor, typename... Args> + bool each_pred(Visitor v, Args&&... args) + { + auto p = node_->inner(); + auto e = p + branches<B>; + if (shift_ == BL) { + for (; p != e; ++p) { + IMMER_PREFETCH(p + 1); + if (!make_full_leaf_pos(*p).visit(v, args...)) + return false; + } + } else { + auto ss = shift_ - B; + for (; p != e; ++p) + if (!make_full_pos(*p, ss).visit(v, args...)) + return false; + } + return true; + } + + template <typename Visitor, typename... Args> + bool each_pred_zip(Visitor v, node_t* other, Args&&... args) + { + auto p = node_->inner(); + auto p2 = other->inner(); + auto e = p + branches<B>; + if (shift_ == BL) { + for (; p != e; ++p, ++p2) { + IMMER_PREFETCH(p + 1); + if (!make_full_leaf_pos(*p).visit(v, *p2, args...)) + return false; + } + } else { + auto ss = shift_ - B; + for (; p != e; ++p, ++p2) + if (!make_full_pos(*p, ss).visit(v, *p2, args...)) + return false; + } + return true; + } + + template <typename Visitor, typename... Args> + bool each_pred_i(Visitor v, count_t i, count_t n, Args&&... args) + { + auto p = node_->inner() + i; + auto e = node_->inner() + n; + if (shift_ == BL) { + for (; p != e; ++p) { + IMMER_PREFETCH(p + 1); + if (!make_full_leaf_pos(*p).visit(v, args...)) + return false; + } + } else { + auto ss = shift_ - B; + for (; p != e; ++p) + if (!make_full_pos(*p, ss).visit(v, args...)) + return false; + } + return true; + } + + template <typename Visitor, typename... Args> + void each_i(Visitor v, count_t i, count_t n, Args&&... args) + { + auto p = node_->inner() + i; + auto e = node_->inner() + n; + if (shift_ == BL) { + for (; p != e; ++p) { + IMMER_PREFETCH(p + 1); + make_full_leaf_pos(*p).visit(v, args...); + } + } else { + auto ss = shift_ - B; + for (; p != e; ++p) + make_full_pos(*p, ss).visit(v, args...); + } + } + + template <typename Visitor, typename... Args> + bool each_pred_right(Visitor v, count_t start, Args&&... args) + { + return each_pred_i(v, start, branches<B>, args...); + } + + template <typename Visitor, typename... Args> + bool each_pred_left(Visitor v, count_t last, Args&&... args) + { + return each_pred_i(v, 0, last, args...); + } + + template <typename Visitor, typename... Args> + void each_sub(Visitor v, Args&&... args) + { + each(v, args...); + } + + template <typename Visitor, typename... Args> + void each_left_sub(Visitor v, Args&&... args) + { + each_i(v, 0, branches<B> - 1, args...); + } + + template <typename Visitor, typename... Args> + void each_right_sub(Visitor v, Args&&... args) + { + each_i(v, 1, branches<B>, args...); + } + + template <typename Visitor, typename... Args> + void each_right(Visitor v, count_t start, Args&&... args) + { + each_i(v, start, branches<B>, args...); + } + + template <typename Visitor, typename... Args> + void each_left(Visitor v, count_t last, Args&&... args) + { + each_i(v, 0, last, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) towards(Visitor v, size_t idx, Args&&... args) + { + return towards_oh(v, idx, index(idx), args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) towards_oh_ch( + Visitor v, size_t idx, count_t offset_hint, count_t, Args&&... args) + { + return towards_oh(v, idx, offset_hint, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) + towards_oh(Visitor v, size_t idx, count_t offset_hint, Args&&... args) + { + assert(offset_hint == index(idx)); + auto is_leaf = shift_ == BL; + auto child = node_->inner()[offset_hint]; + return is_leaf + ? make_full_leaf_pos(child).visit(v, idx, args...) + : make_full_pos(child, shift_ - B).visit(v, idx, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) + towards_sub_oh(Visitor v, size_t idx, count_t offset_hint, Args&&... args) + { + assert(offset_hint == index(idx)); + auto is_leaf = shift_ == BL; + auto child = node_->inner()[offset_hint]; + auto lsize = offset_hint << shift_; + return is_leaf + ? make_full_leaf_pos(child).visit(v, idx - lsize, args...) + : make_full_pos(child, shift_ - B) + .visit(v, idx - lsize, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) first_sub(Visitor v, Args&&... args) + { + auto is_leaf = shift_ == BL; + auto child = node_->inner()[0]; + return is_leaf ? make_full_leaf_pos(child).visit(v, args...) + : make_full_pos(child, shift_ - B).visit(v, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) first_sub_leaf(Visitor v, Args&&... args) + { + assert(shift_ == BL); + auto child = node_->inner()[0]; + return make_full_leaf_pos(child).visit(v, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) first_sub_inner(Visitor v, Args&&... args) + { + assert(shift_ >= BL); + auto child = node_->inner()[0]; + return make_full_pos(child, shift_ - B).visit(v, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) nth_sub(count_t idx, Visitor v, Args&&... args) + { + assert(idx < count()); + auto is_leaf = shift_ == BL; + auto child = node_->inner()[idx]; + return is_leaf ? make_full_leaf_pos(child).visit(v, args...) + : make_full_pos(child, shift_ - B).visit(v, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) nth_sub_leaf(count_t idx, Visitor v, Args&&... args) + { + assert(shift_ == BL); + assert(idx < count()); + auto child = node_->inner()[idx]; + return make_full_leaf_pos(child).visit(v, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) visit(Visitor v, Args&&... args) + { + return Visitor::visit_regular(*this, std::forward<Args>(args)...); + } +}; + +template <typename NodeT> +full_pos<NodeT> make_full_pos(NodeT* node, shift_t shift) +{ + assert(node); + assert(shift >= NodeT::bits_leaf); + return {node, shift}; +} + +template <typename NodeT> +struct relaxed_pos +{ + static constexpr auto B = NodeT::bits; + static constexpr auto BL = NodeT::bits_leaf; + + using node_t = NodeT; + using relaxed_t = typename NodeT::relaxed_t; + node_t* node_; + shift_t shift_; + relaxed_t* relaxed_; + + count_t count() const { return relaxed_->d.count; } + node_t* node() const { return node_; } + size_t size() const { return relaxed_->d.sizes[relaxed_->d.count - 1]; } + shift_t shift() const { return shift_; } + count_t subindex(size_t idx) const { return index(idx); } + relaxed_t* relaxed() const { return relaxed_; } + + size_t size_before(count_t offset) const + { + return offset ? relaxed_->d.sizes[offset - 1] : 0; + } + + size_t size(count_t offset) const + { + return size_sbh(offset, size_before(offset)); + } + + size_t size_sbh(count_t offset, size_t size_before_hint) const + { + assert(size_before_hint == size_before(offset)); + return relaxed_->d.sizes[offset] - size_before_hint; + } + + count_t index(size_t idx) const + { + auto offset = idx >> shift_; + while (relaxed_->d.sizes[offset] <= idx) + ++offset; + return offset; + } + + void copy_sizes(count_t offset, count_t n, size_t init, size_t* sizes) + { + auto e = sizes + n; + auto prev = size_before(offset); + auto these = relaxed_->d.sizes + offset; + for (; sizes != e; ++sizes, ++these) { + auto this_size = *these; + init = *sizes = init + (this_size - prev); + prev = this_size; + } + } + + template <typename Visitor, typename... Args> + void each(Visitor v, Args&&... args) + { + each_left(v, relaxed_->d.count, args...); + } + + template <typename Visitor, typename... Args> + bool each_pred(Visitor v, Args&&... args) + { + auto p = node_->inner(); + auto s = size_t{}; + auto n = count(); + if (shift_ == BL) { + for (auto i = count_t{0}; i < n; ++i) { + IMMER_PREFETCH(p + i + 1); + if (!make_leaf_sub_pos(p[i], relaxed_->d.sizes[i] - s) + .visit(v, args...)) + return false; + s = relaxed_->d.sizes[i]; + } + } else { + auto ss = shift_ - B; + for (auto i = count_t{0}; i < n; ++i) { + if (!visit_maybe_relaxed_sub( + p[i], ss, relaxed_->d.sizes[i] - s, v, args...)) + return false; + s = relaxed_->d.sizes[i]; + } + } + return true; + } + + template <typename Visitor, typename... Args> + bool each_pred_i(Visitor v, count_t i, count_t n, Args&&... args) + { + if (shift_ == BL) { + auto p = node_->inner(); + auto s = i > 0 ? relaxed_->d.sizes[i - 1] : 0; + for (; i < n; ++i) { + IMMER_PREFETCH(p + i + 1); + if (!make_leaf_sub_pos(p[i], relaxed_->d.sizes[i] - s) + .visit(v, args...)) + return false; + s = relaxed_->d.sizes[i]; + } + } else { + auto p = node_->inner(); + auto s = i > 0 ? relaxed_->d.sizes[i - 1] : 0; + auto ss = shift_ - B; + for (; i < n; ++i) { + if (!visit_maybe_relaxed_sub( + p[i], ss, relaxed_->d.sizes[i] - s, v, args...)) + return false; + s = relaxed_->d.sizes[i]; + } + } + return true; + } + + template <typename Visitor, typename... Args> + bool each_pred_left(Visitor v, count_t n, Args&&... args) + { + auto p = node_->inner(); + auto s = size_t{}; + if (shift_ == BL) { + for (auto i = count_t{0}; i < n; ++i) { + IMMER_PREFETCH(p + i + 1); + if (!make_leaf_sub_pos(p[i], relaxed_->d.sizes[i] - s) + .visit(v, args...)) + return false; + s = relaxed_->d.sizes[i]; + } + } else { + auto ss = shift_ - B; + for (auto i = count_t{0}; i < n; ++i) { + if (!visit_maybe_relaxed_sub( + p[i], ss, relaxed_->d.sizes[i] - s, v, args...)) + return false; + s = relaxed_->d.sizes[i]; + } + } + return true; + } + + template <typename Visitor, typename... Args> + bool each_pred_right(Visitor v, count_t start, Args&&... args) + { + assert(start > 0); + assert(start <= relaxed_->d.count); + auto s = relaxed_->d.sizes[start - 1]; + auto p = node_->inner(); + if (shift_ == BL) { + for (auto i = start; i < relaxed_->d.count; ++i) { + IMMER_PREFETCH(p + i + 1); + if (!make_leaf_sub_pos(p[i], relaxed_->d.sizes[i] - s) + .visit(v, args...)) + return false; + s = relaxed_->d.sizes[i]; + } + } else { + auto ss = shift_ - B; + for (auto i = start; i < relaxed_->d.count; ++i) { + if (!visit_maybe_relaxed_sub( + p[i], ss, relaxed_->d.sizes[i] - s, v, args...)) + return false; + s = relaxed_->d.sizes[i]; + } + } + return true; + } + + template <typename Visitor, typename... Args> + void each_i(Visitor v, count_t i, count_t n, Args&&... args) + { + if (shift_ == BL) { + auto p = node_->inner(); + auto s = i > 0 ? relaxed_->d.sizes[i - 1] : 0; + for (; i < n; ++i) { + IMMER_PREFETCH(p + i + 1); + make_leaf_sub_pos(p[i], relaxed_->d.sizes[i] - s) + .visit(v, args...); + s = relaxed_->d.sizes[i]; + } + } else { + auto p = node_->inner(); + auto s = i > 0 ? relaxed_->d.sizes[i - 1] : 0; + auto ss = shift_ - B; + for (; i < n; ++i) { + visit_maybe_relaxed_sub( + p[i], ss, relaxed_->d.sizes[i] - s, v, args...); + s = relaxed_->d.sizes[i]; + } + } + } + + template <typename Visitor, typename... Args> + void each_sub(Visitor v, Args&&... args) + { + each_left(v, relaxed_->d.count, args...); + } + + template <typename Visitor, typename... Args> + void each_left_sub(Visitor v, Args&&... args) + { + each_left(v, relaxed_->d.count - 1, args...); + } + + template <typename Visitor, typename... Args> + void each_left(Visitor v, count_t n, Args&&... args) + { + auto p = node_->inner(); + auto s = size_t{}; + if (shift_ == BL) { + for (auto i = count_t{0}; i < n; ++i) { + IMMER_PREFETCH(p + i + 1); + make_leaf_sub_pos(p[i], relaxed_->d.sizes[i] - s) + .visit(v, args...); + s = relaxed_->d.sizes[i]; + } + } else { + auto ss = shift_ - B; + for (auto i = count_t{0}; i < n; ++i) { + visit_maybe_relaxed_sub( + p[i], ss, relaxed_->d.sizes[i] - s, v, args...); + s = relaxed_->d.sizes[i]; + } + } + } + + template <typename Visitor, typename... Args> + void each_right_sub(Visitor v, Args&&... args) + { + each_right(v, 1, std::forward<Args>(args)...); + } + + template <typename Visitor, typename... Args> + void each_right(Visitor v, count_t start, Args&&... args) + { + assert(start > 0); + assert(start <= relaxed_->d.count); + auto s = relaxed_->d.sizes[start - 1]; + auto p = node_->inner(); + if (shift_ == BL) { + for (auto i = start; i < relaxed_->d.count; ++i) { + IMMER_PREFETCH(p + i + 1); + make_leaf_sub_pos(p[i], relaxed_->d.sizes[i] - s) + .visit(v, args...); + s = relaxed_->d.sizes[i]; + } + } else { + auto ss = shift_ - B; + for (auto i = start; i < relaxed_->d.count; ++i) { + visit_maybe_relaxed_sub( + p[i], ss, relaxed_->d.sizes[i] - s, v, args...); + s = relaxed_->d.sizes[i]; + } + } + } + + template <typename Visitor, typename... Args> + decltype(auto) towards(Visitor v, size_t idx, Args&&... args) + { + return towards_oh(v, idx, subindex(idx), args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) + towards_oh(Visitor v, size_t idx, count_t offset_hint, Args&&... args) + { + assert(offset_hint == index(idx)); + auto left_size = offset_hint ? relaxed_->d.sizes[offset_hint - 1] : 0; + return towards_oh_sbh(v, idx, offset_hint, left_size, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) towards_oh_sbh(Visitor v, + size_t idx, + count_t offset_hint, + size_t left_size_hint, + Args&&... args) + { + return towards_sub_oh_sbh(v, idx, offset_hint, left_size_hint, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) + towards_sub_oh(Visitor v, size_t idx, count_t offset_hint, Args&&... args) + { + assert(offset_hint == index(idx)); + auto left_size = offset_hint ? relaxed_->d.sizes[offset_hint - 1] : 0; + return towards_sub_oh_sbh(v, idx, offset_hint, left_size, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) towards_sub_oh_sbh(Visitor v, + size_t idx, + count_t offset_hint, + size_t left_size_hint, + Args&&... args) + { + assert(offset_hint == index(idx)); + assert(left_size_hint == + (offset_hint ? relaxed_->d.sizes[offset_hint - 1] : 0)); + auto child = node_->inner()[offset_hint]; + auto is_leaf = shift_ == BL; + auto next_size = relaxed_->d.sizes[offset_hint] - left_size_hint; + auto next_idx = idx - left_size_hint; + return is_leaf + ? make_leaf_sub_pos(child, next_size) + .visit(v, next_idx, args...) + : visit_maybe_relaxed_sub( + child, shift_ - B, next_size, v, next_idx, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) last_oh_csh(Visitor v, + count_t offset_hint, + size_t child_size_hint, + Args&&... args) + { + assert(offset_hint == count() - 1); + assert(child_size_hint == size(offset_hint)); + auto child = node_->inner()[offset_hint]; + auto is_leaf = shift_ == BL; + return is_leaf + ? make_leaf_sub_pos(child, child_size_hint).visit(v, args...) + : visit_maybe_relaxed_sub( + child, shift_ - B, child_size_hint, v, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) last_sub(Visitor v, Args&&... args) + { + auto offset = relaxed_->d.count - 1; + auto child = node_->inner()[offset]; + auto child_size = size(offset); + auto is_leaf = shift_ == BL; + return is_leaf ? make_leaf_sub_pos(child, child_size).visit(v, args...) + : visit_maybe_relaxed_sub( + child, shift_ - B, child_size, v, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) first_sub(Visitor v, Args&&... args) + { + auto child = node_->inner()[0]; + auto child_size = relaxed_->d.sizes[0]; + auto is_leaf = shift_ == BL; + return is_leaf ? make_leaf_sub_pos(child, child_size).visit(v, args...) + : visit_maybe_relaxed_sub( + child, shift_ - B, child_size, v, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) first_sub_leaf(Visitor v, Args&&... args) + { + assert(shift_ == BL); + auto child = node_->inner()[0]; + auto child_size = relaxed_->d.sizes[0]; + return make_leaf_sub_pos(child, child_size).visit(v, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) first_sub_inner(Visitor v, Args&&... args) + { + assert(shift_ > BL); + auto child = node_->inner()[0]; + auto child_size = relaxed_->d.sizes[0]; + return visit_maybe_relaxed_sub( + child, shift_ - B, child_size, v, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) nth_sub(count_t offset, Visitor v, Args&&... args) + { + auto child = node_->inner()[offset]; + auto child_size = size(offset); + auto is_leaf = shift_ == BL; + return is_leaf ? make_leaf_sub_pos(child, child_size).visit(v, args...) + : visit_maybe_relaxed_sub( + child, shift_ - B, child_size, v, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) nth_sub_leaf(count_t offset, Visitor v, Args&&... args) + { + assert(shift_ == BL); + auto child = node_->inner()[offset]; + auto child_size = size(offset); + return make_leaf_sub_pos(child, child_size).visit(v, args...); + } + + template <typename Visitor, typename... Args> + decltype(auto) visit(Visitor v, Args&&... args) + { + return Visitor::visit_relaxed(*this, std::forward<Args>(args)...); + } +}; + +template <typename Pos> +using is_relaxed = std::is_same<relaxed_pos<typename std::decay_t<Pos>::node_t>, + std::decay_t<Pos>>; + +template <typename Pos> +constexpr auto is_relaxed_v = is_relaxed<Pos>::value; + +template <typename NodeT> +relaxed_pos<NodeT> +make_relaxed_pos(NodeT* node, shift_t shift, typename NodeT::relaxed_t* relaxed) +{ + assert(node); + assert(relaxed); + assert(shift >= NodeT::bits_leaf); + return {node, shift, relaxed}; +} + +template <typename NodeT, typename Visitor, typename... Args> +decltype(auto) visit_maybe_relaxed_sub( + NodeT* node, shift_t shift, size_t size, Visitor v, Args&&... args) +{ + assert(node); + auto relaxed = node->relaxed(); + if (relaxed) { + assert(size == relaxed->d.sizes[relaxed->d.count - 1]); + return make_relaxed_pos(node, shift, relaxed) + .visit(v, std::forward<Args>(args)...); + } else { + return make_regular_sub_pos(node, shift, size) + .visit(v, std::forward<Args>(args)...); + } +} + +template <typename NodeT, + shift_t Shift, + bits_t B = NodeT::bits, + bits_t BL = NodeT::bits_leaf> +struct relaxed_descent_pos +{ + static_assert(Shift > 0, "not leaf..."); + + using node_t = NodeT; + using relaxed_t = typename NodeT::relaxed_t; + node_t* node_; + relaxed_t* relaxed_; + + count_t count() const { return relaxed_->d.count; } + node_t* node() const { return node_; } + shift_t shift() const { return Shift; } + size_t size() const { return relaxed_->d.sizes[relaxed_->d.count - 1]; } + + count_t index(size_t idx) const + { + // make gcc happy +#if !defined(_MSC_VER) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wshift-count-overflow" +#endif + auto offset = idx >> Shift; +#if !defined(_MSC_VER) +#pragma GCC diagnostic pop +#endif + while (relaxed_->d.sizes[offset] <= idx) + ++offset; + return offset; + } + + template <typename Visitor> + decltype(auto) descend(Visitor v, size_t idx) + { + auto offset = index(idx); + auto child = node_->inner()[offset]; + auto left_size = offset ? relaxed_->d.sizes[offset - 1] : 0; + auto next_idx = idx - left_size; + auto r = child->relaxed(); + return r ? relaxed_descent_pos<NodeT, Shift - B>{child, r}.visit( + v, next_idx) + : regular_descent_pos<NodeT, Shift - B>{child}.visit(v, + next_idx); + } + + template <typename Visitor, typename... Args> + decltype(auto) visit(Visitor v, Args&&... args) + { + return Visitor::visit_relaxed(*this, std::forward<Args>(args)...); + } +}; + +template <typename NodeT, bits_t B, bits_t BL> +struct relaxed_descent_pos<NodeT, BL, B, BL> +{ + using node_t = NodeT; + using relaxed_t = typename NodeT::relaxed_t; + node_t* node_; + relaxed_t* relaxed_; + + count_t count() const { return relaxed_->d.count; } + node_t* node() const { return node_; } + shift_t shift() const { return BL; } + size_t size() const { return relaxed_->d.sizes[relaxed_->d.count - 1]; } + + count_t index(size_t idx) const + { + auto offset = (idx >> BL) & mask<B>; + while (relaxed_->d.sizes[offset] <= idx) + ++offset; + return offset; + } + + template <typename Visitor> + decltype(auto) descend(Visitor v, size_t idx) + { + auto offset = index(idx); + auto child = node_->inner()[offset]; + auto left_size = offset ? relaxed_->d.sizes[offset - 1] : 0; + auto next_idx = idx - left_size; + return leaf_descent_pos<NodeT>{child}.visit(v, next_idx); + } + + template <typename Visitor, typename... Args> + decltype(auto) visit(Visitor v, Args&&... args) + { + return Visitor::visit_relaxed(*this, std::forward<Args>(args)...); + } +}; + +template <typename NodeT, typename Visitor, typename... Args> +decltype(auto) +visit_maybe_relaxed_descent(NodeT* node, shift_t shift, Visitor v, size_t idx) +{ + constexpr auto B = NodeT::bits; + constexpr auto BL = NodeT::bits_leaf; + assert(node); + assert(shift >= BL); + auto r = node->relaxed(); + if (r) { + switch (shift) { + case BL + B * 0: + return relaxed_descent_pos<NodeT, BL + B * 0>{node, r}.visit(v, + idx); + case BL + B * 1: + return relaxed_descent_pos<NodeT, BL + B * 1>{node, r}.visit(v, + idx); + case BL + B * 2: + return relaxed_descent_pos<NodeT, BL + B * 2>{node, r}.visit(v, + idx); + case BL + B * 3: + return relaxed_descent_pos<NodeT, BL + B * 3>{node, r}.visit(v, + idx); + case BL + B * 4: + return relaxed_descent_pos<NodeT, BL + B * 4>{node, r}.visit(v, + idx); + case BL + B * 5: + return relaxed_descent_pos<NodeT, BL + B * 5>{node, r}.visit(v, + idx); +#if IMMER_DESCENT_DEEP + default: + for (auto level = shift; level != endshift<B, BL>; level -= B) { + auto r = node->relaxed(); + if (r) { + auto node_idx = (idx >> level) & mask<B>; + while (r->d.sizes[node_idx] <= idx) + ++node_idx; + if (node_idx) + idx -= r->d.sizes[node_idx - 1]; + node = node->inner()[node_idx]; + } else { + do { + node = node->inner()[(idx >> level) & mask<B>]; + } while ((level -= B) != endshift<B, BL>); + return make_leaf_descent_pos(node).visit(v, idx); + } + } + return make_leaf_descent_pos(node).visit(v, idx); +#endif // IMMER_DESCENT_DEEP + } + IMMER_UNREACHABLE; + } else { + return visit_regular_descent(node, shift, v, idx); + } +} + +} // namespace rbts +} // namespace detail +} // namespace immer diff --git a/immer/detail/rbts/rbtree.hpp b/immer/detail/rbts/rbtree.hpp new file mode 100644 index 000000000000..b31f5e9b0788 --- /dev/null +++ b/immer/detail/rbts/rbtree.hpp @@ -0,0 +1,509 @@ +// +// immer: immutable data structures for C++ +// Copyright (C) 2016, 2017, 2018 Juan Pedro Bolivar Puente +// +// This software is distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE or copy at http://boost.org/LICENSE_1_0.txt +// + +#pragma once + +#include <immer/detail/rbts/node.hpp> +#include <immer/detail/rbts/operations.hpp> +#include <immer/detail/rbts/position.hpp> + +#include <immer/detail/type_traits.hpp> + +#include <cassert> +#include <memory> +#include <numeric> + +namespace immer { +namespace detail { +namespace rbts { + +template <typename T, typename MemoryPolicy, bits_t B, bits_t BL> +struct rbtree +{ + using node_t = node<T, MemoryPolicy, B, BL>; + using edit_t = typename node_t::edit_t; + using owner_t = typename MemoryPolicy::transience_t::owner; + + size_t size; + shift_t shift; + node_t* root; + node_t* tail; + + static const rbtree& empty() + { + static const rbtree empty_{ + 0, BL, node_t::make_inner_n(0u), node_t::make_leaf_n(0u)}; + return empty_; + } + + template <typename U> + static auto from_initializer_list(std::initializer_list<U> values) + { + auto e = owner_t{}; + auto result = rbtree{empty()}; + for (auto&& v : values) + result.push_back_mut(e, v); + return result; + } + + template <typename Iter, + typename Sent, + std::enable_if_t<compatible_sentinel_v<Iter, Sent>, bool> = true> + static auto from_range(Iter first, Sent last) + { + auto e = owner_t{}; + auto result = rbtree{empty()}; + for (; first != last; ++first) + result.push_back_mut(e, *first); + return result; + } + + static auto from_fill(size_t n, T v) + { + auto e = owner_t{}; + auto result = rbtree{empty()}; + while (n-- > 0) + result.push_back_mut(e, v); + return result; + } + + rbtree(size_t sz, shift_t sh, node_t* r, node_t* t) + : size{sz} + , shift{sh} + , root{r} + , tail{t} + { + assert(check_tree()); + } + + rbtree(const rbtree& other) + : rbtree{other.size, other.shift, other.root, other.tail} + { + inc(); + } + + rbtree(rbtree&& other) + : rbtree{empty()} + { + swap(*this, other); + } + + rbtree& operator=(const rbtree& other) + { + auto next = other; + swap(*this, next); + return *this; + } + + rbtree& operator=(rbtree&& other) + { + swap(*this, other); + return *this; + } + + friend void swap(rbtree& x, rbtree& y) + { + using std::swap; + swap(x.size, y.size); + swap(x.shift, y.shift); + swap(x.root, y.root); + swap(x.tail, y.tail); + } + + ~rbtree() { dec(); } + + void inc() const + { + root->inc(); + tail->inc(); + } + + void dec() const { traverse(dec_visitor()); } + + auto tail_size() const { return size ? ((size - 1) & mask<BL>) +1 : 0; } + + auto tail_offset() const { return size ? (size - 1) & ~mask<BL> : 0; } + + template <typename Visitor, typename... Args> + void traverse(Visitor v, Args&&... args) const + { + auto tail_off = tail_offset(); + auto tail_size = size - tail_off; + + if (tail_off) + make_regular_sub_pos(root, shift, tail_off).visit(v, args...); + else + make_empty_regular_pos(root).visit(v, args...); + + make_leaf_sub_pos(tail, tail_size).visit(v, args...); + } + + template <typename Visitor, typename... Args> + void traverse(Visitor v, size_t first, size_t last, Args&&... args) const + { + auto tail_off = tail_offset(); + auto tail_size = size - tail_off; + + if (first < tail_off) + make_regular_sub_pos(root, shift, tail_off) + .visit(v, first, last < tail_off ? last : tail_off, args...); + if (last > tail_off) + make_leaf_sub_pos(tail, tail_size) + .visit(v, + first > tail_off ? first - tail_off : 0, + last - tail_off, + args...); + } + + template <typename Visitor, typename... Args> + bool traverse_p(Visitor v, Args&&... args) const + { + auto tail_off = tail_offset(); + auto tail_size = size - tail_off; + return (tail_off ? make_regular_sub_pos(root, shift, tail_off) + .visit(v, args...) + : make_empty_regular_pos(root).visit(v, args...)) && + make_leaf_sub_pos(tail, tail_size).visit(v, args...); + } + + template <typename Visitor, typename... Args> + bool traverse_p(Visitor v, size_t first, size_t last, Args&&... args) const + { + auto tail_off = tail_offset(); + auto tail_size = size - tail_off; + + return (first < tail_off ? make_regular_sub_pos(root, shift, tail_off) + .visit(v, + first, + last < tail_off ? last : tail_off, + args...) + : true) && + (last > tail_off + ? make_leaf_sub_pos(tail, tail_size) + .visit(v, + first > tail_off ? first - tail_off : 0, + last - tail_off, + args...) + : true); + } + + template <typename Visitor> + decltype(auto) descend(Visitor v, size_t idx) const + { + auto tail_off = tail_offset(); + return idx >= tail_off ? make_leaf_descent_pos(tail).visit(v, idx) + : visit_regular_descent(root, shift, v, idx); + } + + template <typename Fn> + void for_each_chunk(Fn&& fn) const + { + traverse(for_each_chunk_visitor{}, std::forward<Fn>(fn)); + } + + template <typename Fn> + void for_each_chunk(size_t first, size_t last, Fn&& fn) const + { + traverse(for_each_chunk_i_visitor{}, first, last, std::forward<Fn>(fn)); + } + + template <typename Fn> + bool for_each_chunk_p(Fn&& fn) const + { + return traverse_p(for_each_chunk_p_visitor{}, std::forward<Fn>(fn)); + } + + template <typename Fn> + bool for_each_chunk_p(size_t first, size_t last, Fn&& fn) const + { + return traverse_p( + for_each_chunk_p_i_visitor{}, first, last, std::forward<Fn>(fn)); + } + + bool equals(const rbtree& other) const + { + if (size != other.size) + return false; + if (size == 0) + return true; + return (size <= branches<BL> || + make_regular_sub_pos(root, shift, tail_offset()) + .visit(equals_visitor{}, other.root)) && + make_leaf_sub_pos(tail, tail_size()) + .visit(equals_visitor{}, other.tail); + } + + void ensure_mutable_tail(edit_t e, count_t n) + { + if (!tail->can_mutate(e)) { + auto new_tail = node_t::copy_leaf_e(e, tail, n); + dec_leaf(tail, n); + tail = new_tail; + } + } + + void push_back_mut(edit_t e, T value) + { + auto tail_off = tail_offset(); + auto ts = size - tail_off; + if (ts < branches<BL>) { + ensure_mutable_tail(e, ts); + new (&tail->leaf()[ts]) T{std::move(value)}; + } else { + auto new_tail = node_t::make_leaf_e(e, std::move(value)); + try { + if (tail_off == size_t{branches<B>} << shift) { + auto new_root = node_t::make_inner_e(e); + try { + auto path = node_t::make_path_e(e, shift, tail); + new_root->inner()[0] = root; + new_root->inner()[1] = path; + root = new_root; + tail = new_tail; + shift += B; + } catch (...) { + node_t::delete_inner_e(new_root); + throw; + } + } else if (tail_off) { + auto new_root = + make_regular_sub_pos(root, shift, tail_off) + .visit(push_tail_mut_visitor<node_t>{}, e, tail); + root = new_root; + tail = new_tail; + } else { + auto new_root = node_t::make_path_e(e, shift, tail); + assert(tail_off == 0); + dec_empty_regular(root); + root = new_root; + tail = new_tail; + } + } catch (...) { + node_t::delete_leaf(new_tail, 1); + throw; + } + } + ++size; + } + + rbtree push_back(T value) const + { + auto tail_off = tail_offset(); + auto ts = size - tail_off; + if (ts < branches<BL>) { + auto new_tail = + node_t::copy_leaf_emplace(tail, ts, std::move(value)); + return {size + 1, shift, root->inc(), new_tail}; + } else { + auto new_tail = node_t::make_leaf_n(1, std::move(value)); + try { + if (tail_off == size_t{branches<B>} << shift) { + auto new_root = node_t::make_inner_n(2); + try { + auto path = node_t::make_path(shift, tail); + new_root->inner()[0] = root; + new_root->inner()[1] = path; + root->inc(); + tail->inc(); + return {size + 1, shift + B, new_root, new_tail}; + } catch (...) { + node_t::delete_inner(new_root, 2); + throw; + } + } else if (tail_off) { + auto new_root = + make_regular_sub_pos(root, shift, tail_off) + .visit(push_tail_visitor<node_t>{}, tail); + tail->inc(); + return {size + 1, shift, new_root, new_tail}; + } else { + auto new_root = node_t::make_path(shift, tail); + tail->inc(); + return {size + 1, shift, new_root, new_tail}; + } + } catch (...) { + node_t::delete_leaf(new_tail, 1); + throw; + } + } + } + + const T* array_for(size_t index) const + { + return descend(array_for_visitor<T>(), index); + } + + T& get_mut(edit_t e, size_t idx) + { + auto tail_off = tail_offset(); + if (idx >= tail_off) { + ensure_mutable_tail(e, size - tail_off); + return tail->leaf()[idx & mask<BL>]; + } else { + return make_regular_sub_pos(root, shift, tail_off) + .visit(get_mut_visitor<node_t>{}, idx, e, &root); + } + } + + const T& get(size_t index) const + { + return descend(get_visitor<T>(), index); + } + + const T& get_check(size_t index) const + { + if (index >= size) + throw std::out_of_range{"index out of range"}; + return descend(get_visitor<T>(), index); + } + + const T& front() const { return get(0); } + + const T& back() const { return tail->leaf()[(size - 1) & mask<BL>]; } + + template <typename FnT> + void update_mut(edit_t e, size_t idx, FnT&& fn) + { + auto& elem = get_mut(e, idx); + elem = std::forward<FnT>(fn)(std::move(elem)); + } + + template <typename FnT> + rbtree update(size_t idx, FnT&& fn) const + { + auto tail_off = tail_offset(); + if (idx >= tail_off) { + auto tail_size = size - tail_off; + auto new_tail = + make_leaf_sub_pos(tail, tail_size) + .visit(update_visitor<node_t>{}, idx - tail_off, fn); + return {size, shift, root->inc(), new_tail}; + } else { + auto new_root = make_regular_sub_pos(root, shift, tail_off) + .visit(update_visitor<node_t>{}, idx, fn); + return {size, shift, new_root, tail->inc()}; + } + } + + void assoc_mut(edit_t e, size_t idx, T value) + { + update_mut(e, idx, [&](auto&&) { return std::move(value); }); + } + + rbtree assoc(size_t idx, T value) const + { + return update(idx, [&](auto&&) { return std::move(value); }); + } + + rbtree take(size_t new_size) const + { + auto tail_off = tail_offset(); + if (new_size == 0) { + return empty(); + } else if (new_size >= size) { + return *this; + } else if (new_size > tail_off) { + auto new_tail = node_t::copy_leaf(tail, new_size - tail_off); + return {new_size, shift, root->inc(), new_tail}; + } else { + using std::get; + auto l = new_size - 1; + auto v = slice_right_visitor<node_t>(); + auto r = make_regular_sub_pos(root, shift, tail_off).visit(v, l); + auto new_shift = get<0>(r); + auto new_root = get<1>(r); + auto new_tail = get<3>(r); + if (new_root) { + IMMER_ASSERT_TAGGED(new_root->compute_shift() == get<0>(r)); + assert(new_root->check(new_shift, new_size - get<2>(r))); + return {new_size, new_shift, new_root, new_tail}; + } else { + return {new_size, BL, empty().root->inc(), new_tail}; + } + } + } + + void take_mut(edit_t e, size_t new_size) + { + auto tail_off = tail_offset(); + if (new_size == 0) { + // todo: more efficient? + *this = empty(); + } else if (new_size >= size) { + return; + } else if (new_size > tail_off) { + auto ts = size - tail_off; + auto newts = new_size - tail_off; + if (tail->can_mutate(e)) { + destroy_n(tail->leaf() + newts, ts - newts); + } else { + auto new_tail = node_t::copy_leaf_e(e, tail, newts); + dec_leaf(tail, ts); + tail = new_tail; + } + size = new_size; + return; + } else { + using std::get; + auto l = new_size - 1; + auto v = slice_right_mut_visitor<node_t>(); + auto r = make_regular_sub_pos(root, shift, tail_off).visit(v, l, e); + auto new_shift = get<0>(r); + auto new_root = get<1>(r); + auto new_tail = get<3>(r); + if (new_root) { + root = new_root; + shift = new_shift; + } else { + root = empty().root->inc(); + shift = BL; + } + dec_leaf(tail, size - tail_off); + size = new_size; + tail = new_tail; + return; + } + } + + bool check_tree() const + { +#if IMMER_DEBUG_DEEP_CHECK + assert(shift >= BL); + assert(tail_offset() <= size); + assert(check_root()); + assert(check_tail()); +#endif + return true; + } + + bool check_tail() const + { +#if IMMER_DEBUG_DEEP_CHECK + if (tail_size() > 0) + assert(tail->check(0, tail_size())); +#endif + return true; + } + + bool check_root() const + { +#if IMMER_DEBUG_DEEP_CHECK + if (tail_offset() > 0) + assert(root->check(shift, tail_offset())); + else { + IMMER_ASSERT_TAGGED(root->kind() == node_t::kind_t::inner); + assert(shift == BL); + } +#endif + return true; + } +}; + +} // namespace rbts +} // namespace detail +} // namespace immer diff --git a/immer/detail/rbts/rbtree_iterator.hpp b/immer/detail/rbts/rbtree_iterator.hpp new file mode 100644 index 000000000000..90613b10b98e --- /dev/null +++ b/immer/detail/rbts/rbtree_iterator.hpp @@ -0,0 +1,99 @@ +// +// immer: immutable data structures for C++ +// Copyright (C) 2016, 2017, 2018 Juan Pedro Bolivar Puente +// +// This software is distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE or copy at http://boost.org/LICENSE_1_0.txt +// + +#pragma once + +#include <immer/detail/iterator_facade.hpp> +#include <immer/detail/rbts/rbtree.hpp> + +namespace immer { +namespace detail { +namespace rbts { + +template <typename T, typename MP, bits_t B, bits_t BL> +struct rbtree_iterator + : iterator_facade<rbtree_iterator<T, MP, B, BL>, + std::random_access_iterator_tag, + T, + const T&, + std::ptrdiff_t, + const T*> +{ + using tree_t = rbtree<T, MP, B, BL>; + + struct end_t + {}; + + rbtree_iterator() = default; + + rbtree_iterator(const tree_t& v) + : v_{&v} + , i_{0} + , base_{~size_t{}} + , curr_{nullptr} + {} + + rbtree_iterator(const tree_t& v, end_t) + : v_{&v} + , i_{v.size} + , base_{~size_t{}} + , curr_{nullptr} + {} + + const tree_t& impl() const { return *v_; } + size_t index() const { return i_; } + +private: + friend iterator_core_access; + + const tree_t* v_; + size_t i_; + mutable size_t base_; + mutable const T* curr_ = nullptr; + + void increment() + { + assert(i_ < v_->size); + ++i_; + } + + void decrement() + { + assert(i_ > 0); + --i_; + } + + void advance(std::ptrdiff_t n) + { + assert(n <= 0 || i_ + static_cast<size_t>(n) <= v_->size); + assert(n >= 0 || static_cast<size_t>(-n) <= i_); + i_ += n; + } + + bool equal(const rbtree_iterator& other) const { return i_ == other.i_; } + + std::ptrdiff_t distance_to(const rbtree_iterator& other) const + { + return other.i_ > i_ ? static_cast<std::ptrdiff_t>(other.i_ - i_) + : -static_cast<std::ptrdiff_t>(i_ - other.i_); + } + + const T& dereference() const + { + auto base = i_ & ~mask<BL>; + if (base_ != base) { + base_ = base; + curr_ = v_->array_for(i_); + } + return curr_[i_ & mask<BL>]; + } +}; + +} // namespace rbts +} // namespace detail +} // namespace immer diff --git a/immer/detail/rbts/rrbtree.hpp b/immer/detail/rbts/rrbtree.hpp new file mode 100644 index 000000000000..7bf59e6e92b1 --- /dev/null +++ b/immer/detail/rbts/rrbtree.hpp @@ -0,0 +1,1396 @@ +// +// immer: immutable data structures for C++ +// Copyright (C) 2016, 2017, 2018 Juan Pedro Bolivar Puente +// +// This software is distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE or copy at http://boost.org/LICENSE_1_0.txt +// + +#pragma once + +#include <immer/config.hpp> +#include <immer/detail/rbts/node.hpp> +#include <immer/detail/rbts/operations.hpp> +#include <immer/detail/rbts/position.hpp> + +#include <immer/detail/type_traits.hpp> + +#include <cassert> +#include <memory> +#include <numeric> + +namespace immer { +namespace detail { +namespace rbts { + +template <typename T, typename MemoryPolicy, bits_t B, bits_t BL> +struct rrbtree_iterator; + +template <typename T, typename MemoryPolicy, bits_t B, bits_t BL> +struct rrbtree +{ + using node_t = node<T, MemoryPolicy, B, BL>; + using edit_t = typename node_t::edit_t; + using owner_t = typename MemoryPolicy::transience_t::owner; + + size_t size; + shift_t shift; + node_t* root; + node_t* tail; + + static const rrbtree& empty() + { + static const rrbtree empty_{ + 0, BL, node_t::make_inner_n(0u), node_t::make_leaf_n(0u)}; + return empty_; + } + + template <typename U> + static auto from_initializer_list(std::initializer_list<U> values) + { + auto e = owner_t{}; + auto result = rrbtree{empty()}; + for (auto&& v : values) + result.push_back_mut(e, v); + return result; + } + + template <typename Iter, + typename Sent, + std::enable_if_t<compatible_sentinel_v<Iter, Sent>, bool> = true> + static auto from_range(Iter first, Sent last) + { + auto e = owner_t{}; + auto result = rrbtree{empty()}; + for (; first != last; ++first) + result.push_back_mut(e, *first); + return result; + } + + static auto from_fill(size_t n, T v) + { + auto e = owner_t{}; + auto result = rrbtree{empty()}; + while (n-- > 0) + result.push_back_mut(e, v); + return result; + } + + rrbtree(size_t sz, shift_t sh, node_t* r, node_t* t) + : size{sz} + , shift{sh} + , root{r} + , tail{t} + { + assert(check_tree()); + } + + rrbtree(const rrbtree& other) + : rrbtree{other.size, other.shift, other.root, other.tail} + { + inc(); + } + + rrbtree(rrbtree&& other) + : rrbtree{empty()} + { + swap(*this, other); + } + + rrbtree& operator=(const rrbtree& other) + { + auto next{other}; + swap(*this, next); + return *this; + } + + rrbtree& operator=(rrbtree&& other) + { + swap(*this, other); + return *this; + } + + friend void swap(rrbtree& x, rrbtree& y) + { + using std::swap; + swap(x.size, y.size); + swap(x.shift, y.shift); + swap(x.root, y.root); + swap(x.tail, y.tail); + } + + ~rrbtree() { dec(); } + + void inc() const + { + root->inc(); + tail->inc(); + } + + void dec() const { traverse(dec_visitor()); } + + auto tail_size() const { return size - tail_offset(); } + + auto tail_offset() const + { + auto r = root->relaxed(); + assert(r == nullptr || r->d.count); + return r ? r->d.sizes[r->d.count - 1] + : size ? (size - 1) & ~mask<BL> + /* otherwise */ : 0; + } + + template <typename Visitor, typename... Args> + void traverse(Visitor v, Args&&... args) const + { + auto tail_off = tail_offset(); + auto tail_size = size - tail_off; + + if (tail_off) + visit_maybe_relaxed_sub(root, shift, tail_off, v, args...); + else + make_empty_regular_pos(root).visit(v, args...); + + if (tail_size) + make_leaf_sub_pos(tail, tail_size).visit(v, args...); + else + make_empty_leaf_pos(tail).visit(v, args...); + } + + template <typename Visitor, typename... Args> + void traverse(Visitor v, size_t first, size_t last, Args&&... args) const + { + auto tail_off = tail_offset(); + auto tail_size = size - tail_off; + + if (first < tail_off) + visit_maybe_relaxed_sub(root, + shift, + tail_off, + v, + first, + last < tail_off ? last : tail_off, + args...); + if (last > tail_off) + make_leaf_sub_pos(tail, tail_size) + .visit(v, + first > tail_off ? first - tail_off : 0, + last - tail_off, + args...); + } + + template <typename Visitor, typename... Args> + bool traverse_p(Visitor v, Args&&... args) const + { + auto tail_off = tail_offset(); + auto tail_size = size - tail_off; + return (tail_off + ? visit_maybe_relaxed_sub(root, shift, tail_off, v, args...) + : make_empty_regular_pos(root).visit(v, args...)) && + (tail_size ? make_leaf_sub_pos(tail, tail_size).visit(v, args...) + : make_empty_leaf_pos(tail).visit(v, args...)); + } + + template <typename Visitor, typename... Args> + bool traverse_p(Visitor v, size_t first, size_t last, Args&&... args) const + { + auto tail_off = tail_offset(); + auto tail_size = size - tail_off; + return (first < tail_off + ? visit_maybe_relaxed_sub(root, + shift, + tail_off, + v, + first, + last < tail_off ? last : tail_off, + args...) + : true) && + (last > tail_off + ? make_leaf_sub_pos(tail, tail_size) + .visit(v, + first > tail_off ? first - tail_off : 0, + last - tail_off, + args...) + : true); + } + + template <typename Visitor> + decltype(auto) descend(Visitor v, size_t idx) const + { + auto tail_off = tail_offset(); + return idx >= tail_off + ? make_leaf_descent_pos(tail).visit(v, idx - tail_off) + : visit_maybe_relaxed_descent(root, shift, v, idx); + } + + template <typename Fn> + void for_each_chunk(Fn&& fn) const + { + traverse(for_each_chunk_visitor{}, std::forward<Fn>(fn)); + } + + template <typename Fn> + void for_each_chunk(size_t first, size_t last, Fn&& fn) const + { + traverse(for_each_chunk_i_visitor{}, first, last, std::forward<Fn>(fn)); + } + + template <typename Fn> + bool for_each_chunk_p(Fn&& fn) const + { + return traverse_p(for_each_chunk_p_visitor{}, std::forward<Fn>(fn)); + } + + template <typename Fn> + bool for_each_chunk_p(size_t first, size_t last, Fn&& fn) const + { + return traverse_p( + for_each_chunk_p_i_visitor{}, first, last, std::forward<Fn>(fn)); + } + + bool equals(const rrbtree& other) const + { + using iter_t = rrbtree_iterator<T, MemoryPolicy, B, BL>; + if (size != other.size) + return false; + if (size == 0) + return true; + auto tail_off = tail_offset(); + auto tail_off_other = other.tail_offset(); + // compare trees + if (tail_off > 0 && tail_off_other > 0) { + // other.shift != shift is a theoretical possibility for + // relaxed trees that sadly we haven't managed to exercise + // in tests yet... + if (other.shift >= shift) { + if (!visit_maybe_relaxed_sub(other.root, + other.shift, + tail_off_other, + equals_visitor::rrb{}, + iter_t{other}, + root, + shift, + tail_off)) + return false; + } else { + if (!visit_maybe_relaxed_sub(root, + shift, + tail_off, + equals_visitor::rrb{}, + iter_t{*this}, + other.root, + other.shift, + tail_off_other)) + return false; + } + } + return tail_off == tail_off_other + ? make_leaf_sub_pos(tail, tail_size()) + .visit(equals_visitor{}, other.tail) + : tail_off > tail_off_other + ? std::equal(tail->leaf(), + tail->leaf() + (size - tail_off), + other.tail->leaf() + + (tail_off - tail_off_other)) + /* otherwise */ + : std::equal(tail->leaf(), + tail->leaf() + (size - tail_off), + iter_t{other} + tail_off); + } + + std::tuple<shift_t, node_t*> push_tail(node_t* root, + shift_t shift, + size_t size, + node_t* tail, + count_t tail_size) const + { + if (auto r = root->relaxed()) { + auto new_root = + make_relaxed_pos(root, shift, r) + .visit(push_tail_visitor<node_t>{}, tail, tail_size); + if (new_root) + return std::make_tuple(shift, new_root); + else { + auto new_root = node_t::make_inner_r_n(2); + try { + auto new_path = node_t::make_path(shift, tail); + new_root->inner()[0] = root->inc(); + new_root->inner()[1] = new_path; + new_root->relaxed()->d.sizes[0] = size; + new_root->relaxed()->d.sizes[1] = size + tail_size; + new_root->relaxed()->d.count = 2u; + } catch (...) { + node_t::delete_inner_r(new_root, 2); + throw; + } + return std::make_tuple(shift + B, new_root); + } + } else if (size == size_t{branches<B>} << shift) { + auto new_root = node_t::make_inner_n(2); + try { + auto new_path = node_t::make_path(shift, tail); + new_root->inner()[0] = root->inc(); + new_root->inner()[1] = new_path; + } catch (...) { + node_t::delete_inner(new_root, 2); + throw; + } + return std::make_tuple(shift + B, new_root); + } else if (size) { + auto new_root = make_regular_sub_pos(root, shift, size) + .visit(push_tail_visitor<node_t>{}, tail); + return std::make_tuple(shift, new_root); + } else { + return std::make_tuple(shift, node_t::make_path(shift, tail)); + } + } + + void + push_tail_mut(edit_t e, size_t tail_off, node_t* tail, count_t tail_size) + { + if (auto r = root->relaxed()) { + auto new_root = + make_relaxed_pos(root, shift, r) + .visit(push_tail_mut_visitor<node_t>{}, e, tail, tail_size); + if (new_root) { + root = new_root; + } else { + auto new_root = node_t::make_inner_r_e(e); + try { + auto new_path = node_t::make_path_e(e, shift, tail); + new_root->inner()[0] = root; + new_root->inner()[1] = new_path; + new_root->relaxed()->d.sizes[0] = tail_off; + new_root->relaxed()->d.sizes[1] = tail_off + tail_size; + new_root->relaxed()->d.count = 2u; + root = new_root; + shift += B; + } catch (...) { + node_t::delete_inner_r_e(new_root); + throw; + } + } + } else if (tail_off == size_t{branches<B>} << shift) { + auto new_root = node_t::make_inner_e(e); + try { + auto new_path = node_t::make_path_e(e, shift, tail); + new_root->inner()[0] = root; + new_root->inner()[1] = new_path; + root = new_root; + shift += B; + } catch (...) { + node_t::delete_inner_e(new_root); + throw; + } + } else if (tail_off) { + auto new_root = + make_regular_sub_pos(root, shift, tail_off) + .visit(push_tail_mut_visitor<node_t>{}, e, tail); + root = new_root; + } else { + auto new_root = node_t::make_path_e(e, shift, tail); + dec_empty_regular(root); + root = new_root; + } + } + + void ensure_mutable_tail(edit_t e, count_t n) + { + if (!tail->can_mutate(e)) { + auto new_tail = node_t::copy_leaf_e(e, tail, n); + dec_leaf(tail, n); + tail = new_tail; + } + } + + void push_back_mut(edit_t e, T value) + { + auto ts = tail_size(); + if (ts < branches<BL>) { + ensure_mutable_tail(e, ts); + new (&tail->leaf()[ts]) T{std::move(value)}; + } else { + using std::get; + auto new_tail = node_t::make_leaf_e(e, std::move(value)); + auto tail_off = tail_offset(); + try { + push_tail_mut(e, tail_off, tail, ts); + tail = new_tail; + } catch (...) { + node_t::delete_leaf(new_tail, 1u); + throw; + } + } + ++size; + } + + rrbtree push_back(T value) const + { + auto ts = tail_size(); + if (ts < branches<BL>) { + auto new_tail = + node_t::copy_leaf_emplace(tail, ts, std::move(value)); + return {size + 1, shift, root->inc(), new_tail}; + } else { + using std::get; + auto new_tail = node_t::make_leaf_n(1u, std::move(value)); + auto tail_off = tail_offset(); + try { + auto new_root = + push_tail(root, shift, tail_off, tail, size - tail_off); + tail->inc(); + return {size + 1, get<0>(new_root), get<1>(new_root), new_tail}; + } catch (...) { + node_t::delete_leaf(new_tail, 1u); + throw; + } + } + } + + std::tuple<const T*, size_t, size_t> region_for(size_t idx) const + { + using std::get; + auto tail_off = tail_offset(); + if (idx >= tail_off) { + return std::make_tuple(tail->leaf(), tail_off, size); + } else { + auto subs = visit_maybe_relaxed_sub( + root, shift, tail_off, region_for_visitor<T>(), idx); + auto first = idx - get<1>(subs); + auto end = first + get<2>(subs); + return std::make_tuple(get<0>(subs), first, end); + } + } + + T& get_mut(edit_t e, size_t idx) + { + auto tail_off = tail_offset(); + if (idx >= tail_off) { + ensure_mutable_tail(e, size - tail_off); + return tail->leaf()[(idx - tail_off) & mask<BL>]; + } else { + return visit_maybe_relaxed_sub(root, + shift, + tail_off, + get_mut_visitor<node_t>{}, + idx, + e, + &root); + } + } + + const T& get(size_t index) const + { + return descend(get_visitor<T>(), index); + } + + const T& get_check(size_t index) const + { + if (index >= size) + throw std::out_of_range{"out of range"}; + return descend(get_visitor<T>(), index); + } + + const T& front() const { return get(0); } + + const T& back() const { return get(size - 1); } + + template <typename FnT> + void update_mut(edit_t e, size_t idx, FnT&& fn) + { + auto& elem = get_mut(e, idx); + elem = std::forward<FnT>(fn)(std::move(elem)); + } + + template <typename FnT> + rrbtree update(size_t idx, FnT&& fn) const + { + auto tail_off = tail_offset(); + if (idx >= tail_off) { + auto tail_size = size - tail_off; + auto new_tail = + make_leaf_sub_pos(tail, tail_size) + .visit(update_visitor<node_t>{}, idx - tail_off, fn); + return {size, shift, root->inc(), new_tail}; + } else { + auto new_root = visit_maybe_relaxed_sub( + root, shift, tail_off, update_visitor<node_t>{}, idx, fn); + return {size, shift, new_root, tail->inc()}; + } + } + + void assoc_mut(edit_t e, size_t idx, T value) + { + update_mut(e, idx, [&](auto&&) { return std::move(value); }); + } + + rrbtree assoc(size_t idx, T value) const + { + return update(idx, [&](auto&&) { return std::move(value); }); + } + + void take_mut(edit_t e, size_t new_size) + { + auto tail_off = tail_offset(); + if (new_size == 0) { + *this = empty(); + } else if (new_size >= size) { + return; + } else if (new_size > tail_off) { + auto ts = size - tail_off; + auto newts = new_size - tail_off; + if (tail->can_mutate(e)) { + destroy_n(tail->leaf() + newts, ts - newts); + } else { + auto new_tail = node_t::copy_leaf_e(e, tail, newts); + dec_leaf(tail, ts); + tail = new_tail; + } + size = new_size; + return; + } else { + using std::get; + auto l = new_size - 1; + auto v = slice_right_mut_visitor<node_t>(); + auto r = visit_maybe_relaxed_sub(root, shift, tail_off, v, l, e); + auto new_shift = get<0>(r); + auto new_root = get<1>(r); + auto new_tail = get<3>(r); + if (new_root) { + root = new_root; + shift = new_shift; + } else { + root = empty().root->inc(); + shift = BL; + } + dec_leaf(tail, size - tail_off); + size = new_size; + tail = new_tail; + return; + } + } + + rrbtree take(size_t new_size) const + { + auto tail_off = tail_offset(); + if (new_size == 0) { + return empty(); + } else if (new_size >= size) { + return *this; + } else if (new_size > tail_off) { + auto new_tail = node_t::copy_leaf(tail, new_size - tail_off); + return {new_size, shift, root->inc(), new_tail}; + } else { + using std::get; + auto l = new_size - 1; + auto v = slice_right_visitor<node_t>(); + auto r = visit_maybe_relaxed_sub(root, shift, tail_off, v, l); + auto new_shift = get<0>(r); + auto new_root = get<1>(r); + auto new_tail = get<3>(r); + if (new_root) { + IMMER_ASSERT_TAGGED(new_root->compute_shift() == get<0>(r)); + assert(new_root->check(new_shift, new_size - get<2>(r))); + return {new_size, new_shift, new_root, new_tail}; + } else { + return {new_size, BL, empty().root->inc(), new_tail}; + } + } + } + + void drop_mut(edit_t e, size_t elems) + { + using std::get; + auto tail_off = tail_offset(); + if (elems == 0) { + return; + } else if (elems >= size) { + *this = empty(); + } else if (elems == tail_off) { + dec_inner(root, shift, tail_off); + shift = BL; + root = empty().root->inc(); + size -= elems; + return; + } else if (elems > tail_off) { + auto v = slice_left_mut_visitor<node_t>(); + tail = get<1>(make_leaf_sub_pos(tail, size - tail_off) + .visit(v, elems - tail_off, e)); + if (root != empty().root) { + dec_inner(root, shift, tail_off); + shift = BL; + root = empty().root->inc(); + } + size -= elems; + return; + } else { + auto v = slice_left_mut_visitor<node_t>(); + auto r = + visit_maybe_relaxed_sub(root, shift, tail_off, v, elems, e); + shift = get<0>(r); + root = get<1>(r); + size -= elems; + return; + } + } + + rrbtree drop(size_t elems) const + { + if (elems == 0) { + return *this; + } else if (elems >= size) { + return empty(); + } else if (elems == tail_offset()) { + return {size - elems, BL, empty().root->inc(), tail->inc()}; + } else if (elems > tail_offset()) { + auto tail_off = tail_offset(); + auto new_tail = + node_t::copy_leaf(tail, elems - tail_off, size - tail_off); + return {size - elems, BL, empty().root->inc(), new_tail}; + } else { + using std::get; + auto v = slice_left_visitor<node_t>(); + auto r = + visit_maybe_relaxed_sub(root, shift, tail_offset(), v, elems); + auto new_root = get<1>(r); + auto new_shift = get<0>(r); + return {size - elems, new_shift, new_root, tail->inc()}; + } + return *this; + } + + rrbtree concat(const rrbtree& r) const + { + assert(r.size < (std::numeric_limits<size_t>::max() - size)); + using std::get; + if (size == 0) + return r; + else if (r.size == 0) + return *this; + else if (r.tail_offset() == 0) { + // just concat the tail, similar to push_back + auto tail_offst = tail_offset(); + auto tail_size = size - tail_offst; + if (tail_size == branches<BL>) { + auto new_root = + push_tail(root, shift, tail_offst, tail, tail_size); + tail->inc(); + return {size + r.size, + get<0>(new_root), + get<1>(new_root), + r.tail->inc()}; + } else if (tail_size + r.size <= branches<BL>) { + auto new_tail = + node_t::copy_leaf(tail, tail_size, r.tail, r.size); + return {size + r.size, shift, root->inc(), new_tail}; + } else { + auto remaining = branches<BL> - tail_size; + auto add_tail = + node_t::copy_leaf(tail, tail_size, r.tail, remaining); + try { + auto new_tail = + node_t::copy_leaf(r.tail, remaining, r.size); + try { + auto new_root = push_tail( + root, shift, tail_offst, add_tail, branches<BL>); + return {size + r.size, + get<0>(new_root), + get<1>(new_root), + new_tail}; + } catch (...) { + node_t::delete_leaf(new_tail, r.size - remaining); + throw; + } + } catch (...) { + node_t::delete_leaf(add_tail, branches<BL>); + throw; + } + } + } else if (tail_offset() == 0) { + auto tail_offst = tail_offset(); + auto tail_size = size - tail_offst; + auto concated = + concat_trees(tail, tail_size, r.root, r.shift, r.tail_offset()); + auto new_shift = concated.shift(); + auto new_root = concated.node(); + IMMER_ASSERT_TAGGED(new_shift == new_root->compute_shift()); + assert(new_root->check(new_shift, size + r.tail_offset())); + return {size + r.size, new_shift, new_root, r.tail->inc()}; + } else { + auto tail_offst = tail_offset(); + auto tail_size = size - tail_offst; + auto concated = concat_trees(root, + shift, + tail_offst, + tail, + tail_size, + r.root, + r.shift, + r.tail_offset()); + auto new_shift = concated.shift(); + auto new_root = concated.node(); + IMMER_ASSERT_TAGGED(new_shift == new_root->compute_shift()); + assert(new_root->check(new_shift, size + r.tail_offset())); + return {size + r.size, new_shift, new_root, r.tail->inc()}; + } + } + + constexpr static bool supports_transient_concat = + !std::is_empty<edit_t>::value; + + friend void concat_mut_l(rrbtree& l, edit_t el, const rrbtree& r) + { + assert(&l != &r); + assert(r.size < (std::numeric_limits<size_t>::max() - l.size)); + using std::get; + if (l.size == 0) + l = r; + else if (r.size == 0) + return; + else if (r.tail_offset() == 0) { + // just concat the tail, similar to push_back + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + if (tail_size == branches<BL>) { + l.push_tail_mut(el, tail_offst, l.tail, tail_size); + l.tail = r.tail->inc(); + l.size += r.size; + return; + } else if (tail_size + r.size <= branches<BL>) { + l.ensure_mutable_tail(el, tail_size); + std::uninitialized_copy(r.tail->leaf(), + r.tail->leaf() + r.size, + l.tail->leaf() + tail_size); + l.size += r.size; + return; + } else { + auto remaining = branches<BL> - tail_size; + l.ensure_mutable_tail(el, tail_size); + std::uninitialized_copy(r.tail->leaf(), + r.tail->leaf() + remaining, + l.tail->leaf() + tail_size); + try { + auto new_tail = + node_t::copy_leaf_e(el, r.tail, remaining, r.size); + try { + l.push_tail_mut(el, tail_offst, l.tail, branches<BL>); + l.tail = new_tail; + l.size += r.size; + return; + } catch (...) { + node_t::delete_leaf(new_tail, r.size - remaining); + throw; + } + } catch (...) { + destroy_n(r.tail->leaf() + tail_size, remaining); + throw; + } + } + } else if (l.tail_offset() == 0) { + if (supports_transient_concat) { + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + auto concated = + concat_trees_mut(el, + el, + l.tail, + tail_size, + MemoryPolicy::transience_t::noone, + r.root, + r.shift, + r.tail_offset()); + IMMER_ASSERT_TAGGED(concated.shift() == + concated.node()->compute_shift()); + assert(concated.node()->check(concated.shift(), + l.size + r.tail_offset())); + l.size += r.size; + l.shift = concated.shift(); + l.root = concated.node(); + l.tail = r.tail; + } else { + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + auto concated = concat_trees( + l.tail, tail_size, r.root, r.shift, r.tail_offset()); + l = {l.size + r.size, + concated.shift(), + concated.node(), + r.tail->inc()}; + return; + } + } else { + if (supports_transient_concat) { + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + auto concated = + concat_trees_mut(el, + el, + l.root, + l.shift, + tail_offst, + l.tail, + tail_size, + MemoryPolicy::transience_t::noone, + r.root, + r.shift, + r.tail_offset()); + IMMER_ASSERT_TAGGED(concated.shift() == + concated.node()->compute_shift()); + assert(concated.node()->check(concated.shift(), + l.size + r.tail_offset())); + l.size += r.size; + l.shift = concated.shift(); + l.root = concated.node(); + l.tail = r.tail; + } else { + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + auto concated = concat_trees(l.root, + l.shift, + tail_offst, + l.tail, + tail_size, + r.root, + r.shift, + r.tail_offset()); + l = {l.size + r.size, + concated.shift(), + concated.node(), + r.tail->inc()}; + } + } + } + + friend void concat_mut_r(const rrbtree& l, rrbtree& r, edit_t er) + { + assert(&l != &r); + assert(r.size < (std::numeric_limits<size_t>::max() - l.size)); + using std::get; + if (r.size == 0) + r = std::move(l); + else if (l.size == 0) + return; + else if (r.tail_offset() == 0) { + // just concat the tail, similar to push_back + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + if (tail_size == branches<BL>) { + // this could be improved by making sure that the + // newly created nodes as part of the `push_tail()` + // are tagged with `er` + auto res = + l.push_tail(l.root, l.shift, tail_offst, l.tail, tail_size); + l.tail->inc(); // note: leak if mutably concatenated + // with itself, but this is forbidden + // by the interface + r = {l.size + r.size, get<0>(res), get<1>(res), r.tail->inc()}; + return; + } else if (tail_size + r.size <= branches<BL>) { + // doing this in a exception way mutating way is very + // tricky while potential performance gains are + // minimal (we need to move every element of the right + // tail anyways to make space for the left tail) + // + // we could however improve this by at least moving the + // elements of the right tail... + auto new_tail = + node_t::copy_leaf(l.tail, tail_size, r.tail, r.size); + r = {l.size + r.size, l.shift, l.root->inc(), new_tail}; + return; + } else { + // like the immutable version + auto remaining = branches<BL> - tail_size; + auto add_tail = node_t::copy_leaf_e( + er, l.tail, tail_size, r.tail, remaining); + try { + auto new_tail = + node_t::copy_leaf_e(er, r.tail, remaining, r.size); + try { + // this could be improved by making sure that the + // newly created nodes as part of the `push_tail()` + // are tagged with `er` + auto new_root = l.push_tail(l.root, + l.shift, + tail_offst, + add_tail, + branches<BL>); + r = {l.size + r.size, + get<0>(new_root), + get<1>(new_root), + new_tail}; + return; + } catch (...) { + node_t::delete_leaf(new_tail, r.size - remaining); + throw; + } + } catch (...) { + node_t::delete_leaf(add_tail, branches<BL>); + throw; + } + return; + } + } else if (l.tail_offset() == 0) { + if (supports_transient_concat) { + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + auto concated = + concat_trees_mut(er, + MemoryPolicy::transience_t::noone, + l.tail, + tail_size, + er, + r.root, + r.shift, + r.tail_offset()); + IMMER_ASSERT_TAGGED(concated.shift() == + concated.node()->compute_shift()); + assert(concated.node()->check(concated.shift(), + l.size + r.tail_offset())); + r.size += l.size; + r.shift = concated.shift(); + r.root = concated.node(); + } else { + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + auto concated = concat_trees( + l.tail, tail_size, r.root, r.shift, r.tail_offset()); + r = {l.size + r.size, + concated.shift(), + concated.node(), + r.tail->inc()}; + return; + } + } else { + if (supports_transient_concat) { + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + auto concated = + concat_trees_mut(er, + MemoryPolicy::transience_t::noone, + l.root, + l.shift, + tail_offst, + l.tail, + tail_size, + er, + r.root, + r.shift, + r.tail_offset()); + IMMER_ASSERT_TAGGED(concated.shift() == + concated.node()->compute_shift()); + assert(concated.node()->check(concated.shift(), + l.size + r.tail_offset())); + r.size += l.size; + r.shift = concated.shift(); + r.root = concated.node(); + return; + } else { + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + auto concated = concat_trees(l.root, + l.shift, + tail_offst, + l.tail, + tail_size, + r.root, + r.shift, + r.tail_offset()); + r = {l.size + r.size, + concated.shift(), + concated.node(), + r.tail->inc()}; + return; + } + } + } + + friend void concat_mut_lr_l(rrbtree& l, edit_t el, rrbtree& r, edit_t er) + { + assert(&l != &r); + assert(r.size < (std::numeric_limits<size_t>::max() - l.size)); + using std::get; + if (l.size == 0) + l = r; + else if (r.size == 0) + return; + else if (r.tail_offset() == 0) { + // just concat the tail, similar to push_back + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + if (tail_size == branches<BL>) { + l.push_tail_mut(el, tail_offst, l.tail, tail_size); + l.tail = r.tail->inc(); + l.size += r.size; + return; + } else if (tail_size + r.size <= branches<BL>) { + l.ensure_mutable_tail(el, tail_size); + if (r.tail->can_mutate(er)) + detail::uninitialized_move(r.tail->leaf(), + r.tail->leaf() + r.size, + l.tail->leaf() + tail_size); + else + std::uninitialized_copy(r.tail->leaf(), + r.tail->leaf() + r.size, + l.tail->leaf() + tail_size); + l.size += r.size; + return; + } else { + auto remaining = branches<BL> - tail_size; + l.ensure_mutable_tail(el, tail_size); + if (r.tail->can_mutate(er)) + detail::uninitialized_move(r.tail->leaf(), + r.tail->leaf() + remaining, + l.tail->leaf() + tail_size); + else + std::uninitialized_copy(r.tail->leaf(), + r.tail->leaf() + remaining, + l.tail->leaf() + tail_size); + try { + auto new_tail = + node_t::copy_leaf_e(el, r.tail, remaining, r.size); + try { + l.push_tail_mut(el, tail_offst, l.tail, branches<BL>); + l.tail = new_tail; + l.size += r.size; + return; + } catch (...) { + node_t::delete_leaf(new_tail, r.size - remaining); + throw; + } + } catch (...) { + destroy_n(r.tail->leaf() + tail_size, remaining); + throw; + } + } + } else if (l.tail_offset() == 0) { + if (supports_transient_concat) { + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + auto concated = concat_trees_mut(el, + el, + l.tail, + tail_size, + er, + r.root, + r.shift, + r.tail_offset()); + IMMER_ASSERT_TAGGED(concated.shift() == + concated.node()->compute_shift()); + assert(concated.node()->check(concated.shift(), + l.size + r.tail_offset())); + l.size += r.size; + l.shift = concated.shift(); + l.root = concated.node(); + l.tail = r.tail; + r.hard_reset(); + } else { + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + auto concated = concat_trees( + l.tail, tail_size, r.root, r.shift, r.tail_offset()); + l = {l.size + r.size, + concated.shift(), + concated.node(), + r.tail->inc()}; + return; + } + } else { + if (supports_transient_concat) { + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + auto concated = concat_trees_mut(el, + el, + l.root, + l.shift, + tail_offst, + l.tail, + tail_size, + er, + r.root, + r.shift, + r.tail_offset()); + IMMER_ASSERT_TAGGED(concated.shift() == + concated.node()->compute_shift()); + assert(concated.node()->check(concated.shift(), + l.size + r.tail_offset())); + l.size += r.size; + l.shift = concated.shift(); + l.root = concated.node(); + l.tail = r.tail; + r.hard_reset(); + } else { + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + auto concated = concat_trees(l.root, + l.shift, + tail_offst, + l.tail, + tail_size, + r.root, + r.shift, + r.tail_offset()); + l = {l.size + r.size, + concated.shift(), + concated.node(), + r.tail->inc()}; + } + } + } + + friend void concat_mut_lr_r(rrbtree& l, edit_t el, rrbtree& r, edit_t er) + { + assert(&l != &r); + assert(r.size < (std::numeric_limits<size_t>::max() - l.size)); + using std::get; + if (r.size == 0) + r = l; + else if (l.size == 0) + return; + else if (r.tail_offset() == 0) { + // just concat the tail, similar to push_back + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + if (tail_size == branches<BL>) { + // this could be improved by making sure that the + // newly created nodes as part of the `push_tail()` + // are tagged with `er` + auto res = + l.push_tail(l.root, l.shift, tail_offst, l.tail, tail_size); + r = {l.size + r.size, get<0>(res), get<1>(res), r.tail->inc()}; + return; + } else if (tail_size + r.size <= branches<BL>) { + // doing this in a exception way mutating way is very + // tricky while potential performance gains are + // minimal (we need to move every element of the right + // tail anyways to make space for the left tail) + // + // we could however improve this by at least moving the + // elements of the mutable tails... + auto new_tail = + node_t::copy_leaf(l.tail, tail_size, r.tail, r.size); + r = {l.size + r.size, l.shift, l.root->inc(), new_tail}; + return; + } else { + // like the immutable version. + // we could improve this also by moving elements + // instead of just copying them + auto remaining = branches<BL> - tail_size; + auto add_tail = node_t::copy_leaf_e( + er, l.tail, tail_size, r.tail, remaining); + try { + auto new_tail = + node_t::copy_leaf_e(er, r.tail, remaining, r.size); + try { + // this could be improved by making sure that the + // newly created nodes as part of the `push_tail()` + // are tagged with `er` + auto new_root = l.push_tail(l.root, + l.shift, + tail_offst, + add_tail, + branches<BL>); + r = {l.size + r.size, + get<0>(new_root), + get<1>(new_root), + new_tail}; + return; + } catch (...) { + node_t::delete_leaf(new_tail, r.size - remaining); + throw; + } + } catch (...) { + node_t::delete_leaf(add_tail, branches<BL>); + throw; + } + return; + } + } else if (l.tail_offset() == 0) { + if (supports_transient_concat) { + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + auto concated = concat_trees_mut(er, + el, + l.tail, + tail_size, + er, + r.root, + r.shift, + r.tail_offset()); + IMMER_ASSERT_TAGGED(concated.shift() == + concated.node()->compute_shift()); + assert(concated.node()->check(concated.shift(), + l.size + r.tail_offset())); + r.size += l.size; + r.shift = concated.shift(); + r.root = concated.node(); + l.hard_reset(); + } else { + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + auto concated = concat_trees( + l.tail, tail_size, r.root, r.shift, r.tail_offset()); + r = {l.size + r.size, + concated.shift(), + concated.node(), + r.tail->inc()}; + return; + } + } else { + if (supports_transient_concat) { + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + auto concated = concat_trees_mut(er, + el, + l.root, + l.shift, + tail_offst, + l.tail, + tail_size, + er, + r.root, + r.shift, + r.tail_offset()); + IMMER_ASSERT_TAGGED(concated.shift() == + concated.node()->compute_shift()); + assert(concated.node()->check(concated.shift(), + l.size + r.tail_offset())); + r.size += l.size; + r.shift = concated.shift(); + r.root = concated.node(); + l.hard_reset(); + } else { + auto tail_offst = l.tail_offset(); + auto tail_size = l.size - tail_offst; + auto concated = concat_trees(l.root, + l.shift, + tail_offst, + l.tail, + tail_size, + r.root, + r.shift, + r.tail_offset()); + r = {l.size + r.size, + concated.shift(), + concated.node(), + r.tail->inc()}; + } + } + } + + void hard_reset() + { + assert(supports_transient_concat); + auto&& empty_ = empty(); + size = empty_.size; + shift = empty_.shift; + root = empty_.root; + tail = empty_.tail; + } + + bool check_tree() const + { + assert(shift <= sizeof(size_t) * 8 - BL); + assert(shift >= BL); + assert(tail_offset() <= size); + assert(tail_size() <= branches<BL>); +#if IMMER_DEBUG_DEEP_CHECK + assert(check_root()); + assert(check_tail()); +#endif + return true; + } + + bool check_tail() const + { +#if IMMER_DEBUG_DEEP_CHECK + if (tail_size() > 0) + assert(tail->check(endshift<B, BL>, tail_size())); +#endif + return true; + } + + bool check_root() const + { +#if IMMER_DEBUG_DEEP_CHECK + if (tail_offset() > 0) + assert(root->check(shift, tail_offset())); + else { + IMMER_ASSERT_TAGGED(root->kind() == node_t::kind_t::inner); + assert(shift == BL); + } +#endif + return true; + } + +#if IMMER_DEBUG_PRINT + void debug_print(std::ostream& out) const + { + out << "--" << std::endl + << "{" << std::endl + << " size = " << size << std::endl + << " shift = " << shift << std::endl + << " root = " << std::endl; + debug_print_node(out, root, shift, tail_offset()); + out << " tail = " << std::endl; + debug_print_node(out, tail, endshift<B, BL>, tail_size()); + out << "}" << std::endl; + } + + void debug_print_indent(std::ostream& out, unsigned indent) const + { + while (indent-- > 0) + out << ' '; + } + + void debug_print_node(std::ostream& out, + node_t* node, + shift_t shift, + size_t size, + unsigned indent = 8) const + { + const auto indent_step = 4; + + if (shift == endshift<B, BL>) { + debug_print_indent(out, indent); + out << "- {" << size << "} " + << pretty_print_array(node->leaf(), size) << std::endl; + } else if (auto r = node->relaxed()) { + auto count = r->d.count; + debug_print_indent(out, indent); + out << "# {" << size << "} " + << pretty_print_array(r->d.sizes, r->d.count) << std::endl; + auto last_size = size_t{}; + for (auto i = count_t{}; i < count; ++i) { + debug_print_node(out, + node->inner()[i], + shift - B, + r->d.sizes[i] - last_size, + indent + indent_step); + last_size = r->d.sizes[i]; + } + } else { + debug_print_indent(out, indent); + out << "+ {" << size << "}" << std::endl; + auto count = + (size >> shift) + (size - ((size >> shift) << shift) > 0); + if (count) { + for (auto i = count_t{}; i < count - 1; ++i) + debug_print_node(out, + node->inner()[i], + shift - B, + 1 << shift, + indent + indent_step); + debug_print_node(out, + node->inner()[count - 1], + shift - B, + size - ((count - 1) << shift), + indent + indent_step); + } + } + } +#endif // IMMER_DEBUG_PRINT +}; + +} // namespace rbts +} // namespace detail +} // namespace immer diff --git a/immer/detail/rbts/rrbtree_iterator.hpp b/immer/detail/rbts/rrbtree_iterator.hpp new file mode 100644 index 000000000000..af967774e7ef --- /dev/null +++ b/immer/detail/rbts/rrbtree_iterator.hpp @@ -0,0 +1,98 @@ +// +// immer: immutable data structures for C++ +// Copyright (C) 2016, 2017, 2018 Juan Pedro Bolivar Puente +// +// This software is distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE or copy at http://boost.org/LICENSE_1_0.txt +// + +#pragma once + +#include <immer/detail/iterator_facade.hpp> +#include <immer/detail/rbts/rrbtree.hpp> + +namespace immer { +namespace detail { +namespace rbts { + +template <typename T, typename MP, bits_t B, bits_t BL> +struct rrbtree_iterator + : iterator_facade<rrbtree_iterator<T, MP, B, BL>, + std::random_access_iterator_tag, + T, + const T&, + std::ptrdiff_t, + const T*> +{ + using tree_t = rrbtree<T, MP, B, BL>; + using region_t = std::tuple<const T*, size_t, size_t>; + + struct end_t + {}; + + const tree_t& impl() const { return *v_; } + size_t index() const { return i_; } + + rrbtree_iterator() = default; + + rrbtree_iterator(const tree_t& v) + : v_{&v} + , i_{0} + , curr_{nullptr, ~size_t{}, ~size_t{}} + {} + + rrbtree_iterator(const tree_t& v, end_t) + : v_{&v} + , i_{v.size} + , curr_{nullptr, ~size_t{}, ~size_t{}} + {} + +private: + friend iterator_core_access; + + const tree_t* v_; + size_t i_; + mutable region_t curr_; + + void increment() + { + using std::get; + assert(i_ < v_->size); + ++i_; + } + + void decrement() + { + using std::get; + assert(i_ > 0); + --i_; + } + + void advance(std::ptrdiff_t n) + { + using std::get; + assert(n <= 0 || i_ + static_cast<size_t>(n) <= v_->size); + assert(n >= 0 || static_cast<size_t>(-n) <= i_); + i_ += n; + } + + bool equal(const rrbtree_iterator& other) const { return i_ == other.i_; } + + std::ptrdiff_t distance_to(const rrbtree_iterator& other) const + { + return other.i_ > i_ ? static_cast<std::ptrdiff_t>(other.i_ - i_) + : -static_cast<std::ptrdiff_t>(i_ - other.i_); + } + + const T& dereference() const + { + using std::get; + if (i_ < get<1>(curr_) || i_ >= get<2>(curr_)) + curr_ = v_->region_for(i_); + return get<0>(curr_)[i_ - get<1>(curr_)]; + } +}; + +} // namespace rbts +} // namespace detail +} // namespace immer diff --git a/immer/detail/rbts/visitor.hpp b/immer/detail/rbts/visitor.hpp new file mode 100644 index 000000000000..38cd030c4a9f --- /dev/null +++ b/immer/detail/rbts/visitor.hpp @@ -0,0 +1,56 @@ +// +// immer: immutable data structures for C++ +// Copyright (C) 2016, 2017, 2018 Juan Pedro Bolivar Puente +// +// This software is distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE or copy at http://boost.org/LICENSE_1_0.txt +// + +#pragma once + +#include <immer/config.hpp> + +#include <tuple> +#include <utility> + +namespace immer { +namespace detail { +namespace rbts { + +template <typename Deriv> +struct visitor_base +{ + template <typename... Args> + static decltype(auto) visit_node(Args&&... args) + { + IMMER_UNREACHABLE; + } + + template <typename... Args> + static decltype(auto) visit_relaxed(Args&&... args) + { + return Deriv::visit_inner(std::forward<Args>(args)...); + } + + template <typename... Args> + static decltype(auto) visit_regular(Args&&... args) + { + return Deriv::visit_inner(std::forward<Args>(args)...); + } + + template <typename... Args> + static decltype(auto) visit_inner(Args&&... args) + { + return Deriv::visit_node(std::forward<Args>(args)...); + } + + template <typename... Args> + static decltype(auto) visit_leaf(Args&&... args) + { + return Deriv::visit_node(std::forward<Args>(args)...); + } +}; + +} // namespace rbts +} // namespace detail +} // namespace immer |