uvgVPCCenc 1.0.0
uvgVPCCenc is an open-source real-time V-PCC encoder library written in C++ from scratch.
Loading...
Searching...
No Matches
robin_hood.h
Go to the documentation of this file.
1// ______ _____ ______ _________
2// ______________ ___ /_ ___(_)_______ ___ /_ ______ ______ ______ /
3// __ ___/_ __ \__ __ \__ / __ __ \ __ __ \_ __ \_ __ \_ __ /
4// _ / / /_/ /_ /_/ /_ / _ / / / _ / / // /_/ // /_/ // /_/ /
5// /_/ \____/ /_.___/ /_/ /_/ /_/ ________/_/ /_/ \____/ \____/ \__,_/
6// _/_____/
7//
8// Fast & memory efficient hashtable based on robin hood hashing for C++11/14/17/20
9// https://github.com/martinus/robin-hood-hashing
10//
11// Licensed under the MIT License <http://opensource.org/licenses/MIT>.
12// SPDX-License-Identifier: MIT
13// Copyright (c) 2018-2021 Martin Ankerl <http://martin.ankerl.com>
14//
15// Permission is hereby granted, free of charge, to any person obtaining a copy
16// of this software and associated documentation files (the "Software"), to deal
17// in the Software without restriction, including without limitation the rights
18// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
19// copies of the Software, and to permit persons to whom the Software is
20// furnished to do so, subject to the following conditions:
21//
22// The above copyright notice and this permission notice shall be included in all
23// copies or substantial portions of the Software.
24//
25// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
26// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
28// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
30// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31// SOFTWARE.
32
33#ifndef ROBIN_HOOD_H_INCLUDED
34#define ROBIN_HOOD_H_INCLUDED
35
36// see https://semver.org/
37#define ROBIN_HOOD_VERSION_MAJOR 3 // for incompatible API changes
38#define ROBIN_HOOD_VERSION_MINOR 11 // for adding functionality in a backwards-compatible manner
39#define ROBIN_HOOD_VERSION_PATCH 5 // for backwards-compatible bug fixes
40
41#include <algorithm>
42#include <cstdlib>
43#include <cstring>
44#include <functional>
45#include <limits>
46#include <memory> // only to support hash of smart pointers
47#include <stdexcept>
48#include <string>
49#include <type_traits>
50#include <utility>
51#if __cplusplus >= 201703L
52# include <string_view>
53#endif
54
55// #define ROBIN_HOOD_LOG_ENABLED
56#ifdef ROBIN_HOOD_LOG_ENABLED
57# include <iostream>
58# define ROBIN_HOOD_LOG(...) \
59 std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl;
60#else
61# define ROBIN_HOOD_LOG(x)
62#endif
63
64// #define ROBIN_HOOD_TRACE_ENABLED
65#ifdef ROBIN_HOOD_TRACE_ENABLED
66# include <iostream>
67# define ROBIN_HOOD_TRACE(...) \
68 std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl;
69#else
70# define ROBIN_HOOD_TRACE(x)
71#endif
72
73// #define ROBIN_HOOD_COUNT_ENABLED
74#ifdef ROBIN_HOOD_COUNT_ENABLED
75# include <iostream>
76# define ROBIN_HOOD_COUNT(x) ++counts().x;
77namespace robin_hood {
78struct Counts {
79 uint64_t shiftUp{};
80 uint64_t shiftDown{};
81};
82inline std::ostream& operator<<(std::ostream& os, Counts const& c) {
83 return os << c.shiftUp << " shiftUp" << std::endl << c.shiftDown << " shiftDown" << std::endl;
84}
85
86static Counts& counts() {
87 static Counts counts{};
88 return counts;
89}
90} // namespace robin_hood
91#else
92# define ROBIN_HOOD_COUNT(x)
93#endif
94
95// all non-argument macros should use this facility. See
96// https://www.fluentcpp.com/2019/05/28/better-macros-better-flags/
97#define ROBIN_HOOD(x) ROBIN_HOOD_PRIVATE_DEFINITION_##x()
98
99// mark unused members with this macro
100#define ROBIN_HOOD_UNUSED(identifier)
101
102// bitness
103#if SIZE_MAX == UINT32_MAX
104# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 32
105#elif SIZE_MAX == UINT64_MAX
106# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 64
107#else
108# error Unsupported bitness
109#endif
110
111// endianess
112#ifdef _MSC_VER
113# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() 1
114# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() 0
115#else
116# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() \
117 (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
118# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
119#endif
120
121// inline
122#ifdef _MSC_VER
123# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __declspec(noinline)
124#else
125# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __attribute__((noinline))
126#endif
127
128// exceptions
129#if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND)
130# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 0
131#else
132# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 1
133#endif
134
135// count leading/trailing bits
136#if !defined(ROBIN_HOOD_DISABLE_INTRINSICS)
137# ifdef _MSC_VER
138# if ROBIN_HOOD(BITNESS) == 32
139# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward
140# else
141# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward64
142# endif
143# include <intrin.h>
144# pragma intrinsic(ROBIN_HOOD(BITSCANFORWARD))
145# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) \
146 [](size_t mask) noexcept -> int { \
147 unsigned long index; \
148 return ROBIN_HOOD(BITSCANFORWARD)(&index, mask) ? static_cast<int>(index) \
149 : ROBIN_HOOD(BITNESS); \
150 }(x)
151# else
152# if ROBIN_HOOD(BITNESS) == 32
153# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzl
154# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzl
155# else
156# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzll
157# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzll
158# endif
159# define ROBIN_HOOD_COUNT_LEADING_ZEROES(x) ((x) ? ROBIN_HOOD(CLZ)(x) : ROBIN_HOOD(BITNESS))
160# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) ((x) ? ROBIN_HOOD(CTZ)(x) : ROBIN_HOOD(BITNESS))
161# endif
162#endif
163
164// fallthrough
165#ifndef __has_cpp_attribute // For backwards compatibility
166# define __has_cpp_attribute(x) 0
167#endif
168#if __has_cpp_attribute(clang::fallthrough)
169# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[clang::fallthrough]]
170#elif __has_cpp_attribute(gnu::fallthrough)
171# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[gnu::fallthrough]]
172#else
173# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH()
174#endif
175
176// likely/unlikely
177#ifdef _MSC_VER
178# define ROBIN_HOOD_LIKELY(condition) condition
179# define ROBIN_HOOD_UNLIKELY(condition) condition
180#else
181# define ROBIN_HOOD_LIKELY(condition) __builtin_expect(condition, 1)
182# define ROBIN_HOOD_UNLIKELY(condition) __builtin_expect(condition, 0)
183#endif
184
185// detect if native wchar_t type is availiable in MSVC
186#ifdef _MSC_VER
187# ifdef _NATIVE_WCHAR_T_DEFINED
188# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1
189# else
190# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 0
191# endif
192#else
193# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1
194#endif
195
196// detect if MSVC supports the pair(std::piecewise_construct_t,...) consructor being constexpr
197#ifdef _MSC_VER
198# if _MSC_VER <= 1900
199# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 1
200# else
201# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0
202# endif
203#else
204# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0
205#endif
206
207// workaround missing "is_trivially_copyable" in g++ < 5.0
208// See https://stackoverflow.com/a/31798726/48181
209#if defined(__GNUC__) && __GNUC__ < 5
210# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) __has_trivial_copy(__VA_ARGS__)
211#else
212# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value
213#endif
214
215// helpers for C++ versions, see https://gcc.gnu.org/onlinedocs/cpp/Standard-Predefined-Macros.html
216#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX() __cplusplus
217#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX98() 199711L
218#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX11() 201103L
219#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX14() 201402L
220#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX17() 201703L
221
222#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17)
223# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() [[nodiscard]]
224#else
225# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD()
226#endif
227
228namespace robin_hood {
229
230#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14)
231# define ROBIN_HOOD_STD std
232#else
233
234// c++11 compatibility layer
235namespace ROBIN_HOOD_STD {
236template <class T>
238 : std::integral_constant<std::size_t, alignof(typename std::remove_all_extents<T>::type)> {};
239
240template <class T, T... Ints>
242public:
243 using value_type = T;
244 static_assert(std::is_integral<value_type>::value, "not integral type");
245 static constexpr std::size_t size() noexcept {
246 return sizeof...(Ints);
247 }
248};
249template <std::size_t... Inds>
250using index_sequence = integer_sequence<std::size_t, Inds...>;
251
252namespace detail_ {
253template <class T, T Begin, T End, bool>
255 using TValue = T;
256 static_assert(std::is_integral<TValue>::value, "not integral type");
257 static_assert(Begin >= 0 && Begin < End, "unexpected argument (Begin<0 || Begin<=End)");
258
259 template <class, class>
261
262 template <TValue... Inds0, TValue... Inds1>
264 using TResult = integer_sequence<TValue, Inds0..., Inds1...>;
265 };
266
267 using TResult =
268 typename IntSeqCombiner<typename IntSeqImpl<TValue, Begin, Begin + (End - Begin) / 2,
269 (End - Begin) / 2 == 1>::TResult,
270 typename IntSeqImpl<TValue, Begin + (End - Begin) / 2, End,
271 (End - Begin + 1) / 2 == 1>::TResult>::TResult;
272};
273
274template <class T, T Begin>
275struct IntSeqImpl<T, Begin, Begin, false> {
276 using TValue = T;
277 static_assert(std::is_integral<TValue>::value, "not integral type");
278 static_assert(Begin >= 0, "unexpected argument (Begin<0)");
280};
281
282template <class T, T Begin, T End>
283struct IntSeqImpl<T, Begin, End, true> {
284 using TValue = T;
285 static_assert(std::is_integral<TValue>::value, "not integral type");
286 static_assert(Begin >= 0, "unexpected argument (Begin<0)");
288};
289} // namespace detail_
290
291template <class T, T N>
292using make_integer_sequence = typename detail_::IntSeqImpl<T, 0, N, (N - 0) == 1>::TResult;
293
294template <std::size_t N>
296
297template <class... T>
299
300} // namespace ROBIN_HOOD_STD
301
302#endif
303
304namespace detail {
305
306// make sure we static_cast to the correct type for hash_int
307#if ROBIN_HOOD(BITNESS) == 64
308using SizeT = uint64_t;
309#else
310using SizeT = uint32_t;
311#endif
312
313template <typename T>
314T rotr(T x, unsigned k) {
315 return (x >> k) | (x << (8U * sizeof(T) - k));
316}
317
318// This cast gets rid of warnings like "cast from 'uint8_t*' {aka 'unsigned char*'} to
319// 'uint64_t*' {aka 'long unsigned int*'} increases required alignment of target type". Use with
320// care!
321template <typename T>
322inline T reinterpret_cast_no_cast_align_warning(void* ptr) noexcept {
323 return reinterpret_cast<T>(ptr);
324}
325
326template <typename T>
327inline T reinterpret_cast_no_cast_align_warning(void const* ptr) noexcept {
328 return reinterpret_cast<T>(ptr);
329}
330
331// make sure this is not inlined as it is slow and dramatically enlarges code, thus making other
332// inlinings more difficult. Throws are also generally the slow path.
333template <typename E, typename... Args>
334[[noreturn]] ROBIN_HOOD(NOINLINE)
335#if ROBIN_HOOD(HAS_EXCEPTIONS)
336 void doThrow(Args&&... args) {
337 // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay)
338 throw E(std::forward<Args>(args)...);
339}
340#else
341 void doThrow(Args&&... ROBIN_HOOD_UNUSED(args) /*unused*/) {
342 abort();
343}
344#endif
345
346template <typename E, typename T, typename... Args>
347T* assertNotNull(T* t, Args&&... args) {
348 if (ROBIN_HOOD_UNLIKELY(nullptr == t)) {
349 doThrow<E>(std::forward<Args>(args)...);
350 }
351 return t;
352}
353
354template <typename T>
355inline T unaligned_load(void const* ptr) noexcept {
356 // using memcpy so we don't get into unaligned load problems.
357 // compiler should optimize this very well anyways.
358 T t;
359 std::memcpy(&t, ptr, sizeof(T));
360 return t;
361}
362
363// Allocates bulks of memory for objects of type T. This deallocates the memory in the destructor,
364// and keeps a linked list of the allocated memory around. Overhead per allocation is the size of a
365// pointer.
366template <typename T, size_t MinNumAllocs = 4, size_t MaxNumAllocs = 256>
368public:
369 BulkPoolAllocator() noexcept = default;
370
371 // does not copy anything, just creates a new allocator.
373 : mHead(nullptr)
374 , mListForFree(nullptr) {}
375
377 : mHead(o.mHead)
378 , mListForFree(o.mListForFree) {
379 o.mListForFree = nullptr;
380 o.mHead = nullptr;
381 }
382
384 reset();
385 mHead = o.mHead;
386 mListForFree = o.mListForFree;
387 o.mListForFree = nullptr;
388 o.mHead = nullptr;
389 return *this;
390 }
391
393 // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp)
394 operator=(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept {
395 // does not do anything
396 return *this;
397 }
398
400 reset();
401 }
402
403 // Deallocates all allocated memory.
404 void reset() noexcept {
405 while (mListForFree) {
406 T* tmp = *mListForFree;
407 ROBIN_HOOD_LOG("std::free")
408 std::free(mListForFree);
409 mListForFree = reinterpret_cast_no_cast_align_warning<T**>(tmp);
410 }
411 mHead = nullptr;
412 }
413
414 // allocates, but does NOT initialize. Use in-place new constructor, e.g.
415 // T* obj = pool.allocate();
416 // ::new (static_cast<void*>(obj)) T();
417 T* allocate() {
418 T* tmp = mHead;
419 if (!tmp) {
420 tmp = performAllocation();
421 }
422
423 mHead = *reinterpret_cast_no_cast_align_warning<T**>(tmp);
424 return tmp;
425 }
426
427 // does not actually deallocate but puts it in store.
428 // make sure you have already called the destructor! e.g. with
429 // obj->~T();
430 // pool.deallocate(obj);
431 void deallocate(T* obj) noexcept {
432 *reinterpret_cast_no_cast_align_warning<T**>(obj) = mHead;
433 mHead = obj;
434 }
435
436 // Adds an already allocated block of memory to the allocator. This allocator is from now on
437 // responsible for freeing the data (with free()). If the provided data is not large enough to
438 // make use of, it is immediately freed. Otherwise it is reused and freed in the destructor.
439 void addOrFree(void* ptr, const size_t numBytes) noexcept {
440 // calculate number of available elements in ptr
441 if (numBytes < ALIGNMENT + ALIGNED_SIZE) {
442 // not enough data for at least one element. Free and return.
443 ROBIN_HOOD_LOG("std::free")
444 std::free(ptr);
445 } else {
446 ROBIN_HOOD_LOG("add to buffer")
447 add(ptr, numBytes);
448 }
449 }
450
452 using std::swap;
453 swap(mHead, other.mHead);
454 swap(mListForFree, other.mListForFree);
455 }
456
457private:
458 // iterates the list of allocated memory to calculate how many to alloc next.
459 // Recalculating this each time saves us a size_t member.
460 // This ignores the fact that memory blocks might have been added manually with addOrFree. In
461 // practice, this should not matter much.
462 ROBIN_HOOD(NODISCARD) size_t calcNumElementsToAlloc() const noexcept {
463 auto tmp = mListForFree;
464 size_t numAllocs = MinNumAllocs;
465
466 while (numAllocs * 2 <= MaxNumAllocs && tmp) {
467 auto x = reinterpret_cast<T***>(tmp);
468 tmp = *x;
469 numAllocs *= 2;
470 }
471
472 return numAllocs;
473 }
474
475 // WARNING: Underflow if numBytes < ALIGNMENT! This is guarded in addOrFree().
476 void add(void* ptr, const size_t numBytes) noexcept {
477 const size_t numElements = (numBytes - ALIGNMENT) / ALIGNED_SIZE;
478
479 auto data = reinterpret_cast<T**>(ptr);
480
481 // link free list
482 auto x = reinterpret_cast<T***>(data);
483 *x = mListForFree;
484 mListForFree = data;
485
486 // create linked list for newly allocated data
487 auto* const headT =
488 reinterpret_cast_no_cast_align_warning<T*>(reinterpret_cast<char*>(ptr) + ALIGNMENT);
489
490 auto* const head = reinterpret_cast<char*>(headT);
491
492 // Visual Studio compiler automatically unrolls this loop, which is pretty cool
493 for (size_t i = 0; i < numElements; ++i) {
494 *reinterpret_cast_no_cast_align_warning<char**>(head + i * ALIGNED_SIZE) =
495 head + (i + 1) * ALIGNED_SIZE;
496 }
497
498 // last one points to 0
499 *reinterpret_cast_no_cast_align_warning<T**>(head + (numElements - 1) * ALIGNED_SIZE) =
500 mHead;
501 mHead = headT;
502 }
503
504 // Called when no memory is available (mHead == 0).
505 // Don't inline this slow path.
506 ROBIN_HOOD(NOINLINE) T* performAllocation() {
507 size_t const numElementsToAlloc = calcNumElementsToAlloc();
508
509 // alloc new memory: [prev |T, T, ... T]
510 size_t const bytes = ALIGNMENT + ALIGNED_SIZE * numElementsToAlloc;
511 ROBIN_HOOD_LOG("std::malloc " << bytes << " = " << ALIGNMENT << " + " << ALIGNED_SIZE
512 << " * " << numElementsToAlloc)
513 add(assertNotNull<std::bad_alloc>(std::malloc(bytes)), bytes);
514 return mHead;
515 }
516
517 // enforce byte alignment of the T's
518#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14)
519 static constexpr size_t ALIGNMENT =
520 (std::max)(std::alignment_of<T>::value, std::alignment_of<T*>::value);
521#else
522 static const size_t ALIGNMENT =
523 (ROBIN_HOOD_STD::alignment_of<T>::value > ROBIN_HOOD_STD::alignment_of<T*>::value)
524 ? ROBIN_HOOD_STD::alignment_of<T>::value
525 : +ROBIN_HOOD_STD::alignment_of<T*>::value; // the + is for walkarround
526#endif
527
528 static constexpr size_t ALIGNED_SIZE = ((sizeof(T) - 1) / ALIGNMENT + 1) * ALIGNMENT;
529
530 static_assert(MinNumAllocs >= 1, "MinNumAllocs");
531 static_assert(MaxNumAllocs >= MinNumAllocs, "MaxNumAllocs");
532 static_assert(ALIGNED_SIZE >= sizeof(T*), "ALIGNED_SIZE");
533 static_assert(0 == (ALIGNED_SIZE % sizeof(T*)), "ALIGNED_SIZE mod");
534 static_assert(ALIGNMENT >= sizeof(T*), "ALIGNMENT");
535
536 T* mHead{nullptr};
537 T** mListForFree{nullptr};
538};
539
540template <typename T, size_t MinSize, size_t MaxSize, bool IsFlat>
542
543// dummy allocator that does nothing
544template <typename T, size_t MinSize, size_t MaxSize>
545struct NodeAllocator<T, MinSize, MaxSize, true> {
546
547 // we are not using the data, so just free it.
548 void addOrFree(void* ptr, size_t ROBIN_HOOD_UNUSED(numBytes) /*unused*/) noexcept {
549 ROBIN_HOOD_LOG("std::free")
550 std::free(ptr);
551 }
552};
553
554template <typename T, size_t MinSize, size_t MaxSize>
555struct NodeAllocator<T, MinSize, MaxSize, false> : public BulkPoolAllocator<T, MinSize, MaxSize> {};
556
557// c++14 doesn't have is_nothrow_swappable, and clang++ 6.0.1 doesn't like it either, so I'm making
558// my own here.
559namespace swappable {
560#if ROBIN_HOOD(CXX) < ROBIN_HOOD(CXX17)
561using std::swap;
562template <typename T>
563struct nothrow {
564 static const bool value = noexcept(swap(std::declval<T&>(), std::declval<T&>()));
565};
566#else
567template <typename T>
568struct nothrow {
569 static const bool value = std::is_nothrow_swappable<T>::value;
570};
571#endif
572} // namespace swappable
573
574} // namespace detail
575
577
578// A custom pair implementation is used in the map because std::pair is not is_trivially_copyable,
579// which means it would not be allowed to be used in std::memcpy. This struct is copyable, which is
580// also tested.
581template <typename T1, typename T2>
582struct pair {
583 using first_type = T1;
584 using second_type = T2;
585
586 template <typename U1 = T1, typename U2 = T2,
587 typename = typename std::enable_if<std::is_default_constructible<U1>::value &&
588 std::is_default_constructible<U2>::value>::type>
589 constexpr pair() noexcept(noexcept(U1()) && noexcept(U2()))
590 : first()
591 , second() {}
592
593 // pair constructors are explicit so we don't accidentally call this ctor when we don't have to.
594 explicit constexpr pair(std::pair<T1, T2> const& o) noexcept(
595 noexcept(T1(std::declval<T1 const&>())) && noexcept(T2(std::declval<T2 const&>())))
596 : first(o.first)
597 , second(o.second) {}
598
599 // pair constructors are explicit so we don't accidentally call this ctor when we don't have to.
600 explicit constexpr pair(std::pair<T1, T2>&& o) noexcept(noexcept(
601 T1(std::move(std::declval<T1&&>()))) && noexcept(T2(std::move(std::declval<T2&&>()))))
602 : first(std::move(o.first))
603 , second(std::move(o.second)) {}
604
605 constexpr pair(T1&& a, T2&& b) noexcept(noexcept(
606 T1(std::move(std::declval<T1&&>()))) && noexcept(T2(std::move(std::declval<T2&&>()))))
607 : first(std::move(a))
608 , second(std::move(b)) {}
609
610 template <typename U1, typename U2>
611 constexpr pair(U1&& a, U2&& b) noexcept(noexcept(T1(std::forward<U1>(
612 std::declval<U1&&>()))) && noexcept(T2(std::forward<U2>(std::declval<U2&&>()))))
613 : first(std::forward<U1>(a))
614 , second(std::forward<U2>(b)) {}
615
616 template <typename... U1, typename... U2>
617 // MSVC 2015 produces error "C2476: ‘constexpr’ constructor does not initialize all members"
618 // if this constructor is constexpr
619#if !ROBIN_HOOD(BROKEN_CONSTEXPR)
620 constexpr
621#endif
622 pair(std::piecewise_construct_t /*unused*/, std::tuple<U1...> a,
623 std::tuple<U2...>
624 b) noexcept(noexcept(pair(std::declval<std::tuple<U1...>&>(),
625 std::declval<std::tuple<U2...>&>(),
630 }
631
632 // constructor called from the std::piecewise_construct_t ctor
633 template <typename... U1, size_t... I1, typename... U2, size_t... I2>
634 pair(std::tuple<U1...>& a, std::tuple<U2...>& b, ROBIN_HOOD_STD::index_sequence<I1...> /*unused*/, ROBIN_HOOD_STD::index_sequence<I2...> /*unused*/) noexcept(
635 noexcept(T1(std::forward<U1>(std::get<I1>(
636 std::declval<std::tuple<
637 U1...>&>()))...)) && noexcept(T2(std::
638 forward<U2>(std::get<I2>(
639 std::declval<std::tuple<U2...>&>()))...)))
640 : first(std::forward<U1>(std::get<I1>(a))...)
641 , second(std::forward<U2>(std::get<I2>(b))...) {
642 // make visual studio compiler happy about warning about unused a & b.
643 // Visual studio's pair implementation disables warning 4100.
644 (void)a;
645 (void)b;
646 }
647
650 using std::swap;
651 swap(first, o.first);
652 swap(second, o.second);
653 }
654
655 T1 first; // NOLINT(misc-non-private-member-variables-in-classes)
656 T2 second; // NOLINT(misc-non-private-member-variables-in-classes)
657};
658
659template <typename A, typename B>
660inline void swap(pair<A, B>& a, pair<A, B>& b) noexcept(
661 noexcept(std::declval<pair<A, B>&>().swap(std::declval<pair<A, B>&>()))) {
662 a.swap(b);
663}
664
665template <typename A, typename B>
666inline constexpr bool operator==(pair<A, B> const& x, pair<A, B> const& y) {
667 return (x.first == y.first) && (x.second == y.second);
668}
669template <typename A, typename B>
670inline constexpr bool operator!=(pair<A, B> const& x, pair<A, B> const& y) {
671 return !(x == y);
672}
673template <typename A, typename B>
674inline constexpr bool operator<(pair<A, B> const& x, pair<A, B> const& y) noexcept(noexcept(
675 std::declval<A const&>() < std::declval<A const&>()) && noexcept(std::declval<B const&>() <
676 std::declval<B const&>())) {
677 return x.first < y.first || (!(y.first < x.first) && x.second < y.second);
678}
679template <typename A, typename B>
680inline constexpr bool operator>(pair<A, B> const& x, pair<A, B> const& y) {
681 return y < x;
682}
683template <typename A, typename B>
684inline constexpr bool operator<=(pair<A, B> const& x, pair<A, B> const& y) {
685 return !(x > y);
686}
687template <typename A, typename B>
688inline constexpr bool operator>=(pair<A, B> const& x, pair<A, B> const& y) {
689 return !(x < y);
690}
691
692inline size_t hash_bytes(void const* ptr, size_t len) noexcept {
693 static constexpr uint64_t m = UINT64_C(0xc6a4a7935bd1e995);
694 static constexpr uint64_t seed = UINT64_C(0xe17a1465);
695 static constexpr unsigned int r = 47;
696
697 auto const* const data64 = static_cast<uint64_t const*>(ptr);
698 uint64_t h = seed ^ (len * m);
699
700 size_t const n_blocks = len / 8;
701 for (size_t i = 0; i < n_blocks; ++i) {
702 auto k = detail::unaligned_load<uint64_t>(data64 + i);
703
704 k *= m;
705 k ^= k >> r;
706 k *= m;
707
708 h ^= k;
709 h *= m;
710 }
711
712 auto const* const data8 = reinterpret_cast<uint8_t const*>(data64 + n_blocks);
713 switch (len & 7U) {
714 case 7:
715 h ^= static_cast<uint64_t>(data8[6]) << 48U;
716 ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
717 case 6:
718 h ^= static_cast<uint64_t>(data8[5]) << 40U;
719 ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
720 case 5:
721 h ^= static_cast<uint64_t>(data8[4]) << 32U;
722 ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
723 case 4:
724 h ^= static_cast<uint64_t>(data8[3]) << 24U;
725 ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
726 case 3:
727 h ^= static_cast<uint64_t>(data8[2]) << 16U;
728 ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
729 case 2:
730 h ^= static_cast<uint64_t>(data8[1]) << 8U;
731 ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
732 case 1:
733 h ^= static_cast<uint64_t>(data8[0]);
734 h *= m;
735 ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
736 default:
737 break;
738 }
739
740 h ^= h >> r;
741
742 // not doing the final step here, because this will be done by keyToIdx anyways
743 // h *= m;
744 // h ^= h >> r;
745 return static_cast<size_t>(h);
746}
747
748inline size_t hash_int(uint64_t x) noexcept {
749 // tried lots of different hashes, let's stick with murmurhash3. It's simple, fast, well tested,
750 // and doesn't need any special 128bit operations.
751 x ^= x >> 33U;
752 x *= UINT64_C(0xff51afd7ed558ccd);
753 x ^= x >> 33U;
754
755 // not doing the final step here, because this will be done by keyToIdx anyways
756 // x *= UINT64_C(0xc4ceb9fe1a85ec53);
757 // x ^= x >> 33U;
758 return static_cast<size_t>(x);
759}
760
761// A thin wrapper around std::hash, performing an additional simple mixing step of the result.
762template <typename T, typename Enable = void>
763struct hash : public std::hash<T> {
764 size_t operator()(T const& obj) const
765 noexcept(noexcept(std::declval<std::hash<T>>().operator()(std::declval<T const&>()))) {
766 // call base hash
767 auto result = std::hash<T>::operator()(obj);
768 // return mixed of that, to be save against identity has
769 return hash_int(static_cast<detail::SizeT>(result));
770 }
771};
772
773template <typename CharT>
774struct hash<std::basic_string<CharT>> {
775 size_t operator()(std::basic_string<CharT> const& str) const noexcept {
776 return hash_bytes(str.data(), sizeof(CharT) * str.size());
777 }
778};
779
780#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17)
781template <typename CharT>
782struct hash<std::basic_string_view<CharT>> {
783 size_t operator()(std::basic_string_view<CharT> const& sv) const noexcept {
784 return hash_bytes(sv.data(), sizeof(CharT) * sv.size());
785 }
786};
787#endif
788
789template <class T>
790struct hash<T*> {
791 size_t operator()(T* ptr) const noexcept {
792 return hash_int(reinterpret_cast<detail::SizeT>(ptr));
793 }
794};
795
796template <class T>
797struct hash<std::unique_ptr<T>> {
798 size_t operator()(std::unique_ptr<T> const& ptr) const noexcept {
799 return hash_int(reinterpret_cast<detail::SizeT>(ptr.get()));
800 }
801};
802
803template <class T>
804struct hash<std::shared_ptr<T>> {
805 size_t operator()(std::shared_ptr<T> const& ptr) const noexcept {
806 return hash_int(reinterpret_cast<detail::SizeT>(ptr.get()));
807 }
808};
809
810template <typename Enum>
811struct hash<Enum, typename std::enable_if<std::is_enum<Enum>::value>::type> {
812 size_t operator()(Enum e) const noexcept {
813 using Underlying = typename std::underlying_type<Enum>::type;
814 return hash<Underlying>{}(static_cast<Underlying>(e));
815 }
816};
817
818#define ROBIN_HOOD_HASH_INT(T) \
819 template <> \
820 struct hash<T> { \
821 size_t operator()(T const& obj) const noexcept { \
822 return hash_int(static_cast<uint64_t>(obj)); \
823 } \
824 }
825
826#if defined(__GNUC__) && !defined(__clang__)
827# pragma GCC diagnostic push
828# pragma GCC diagnostic ignored "-Wuseless-cast"
829#endif
830// see https://en.cppreference.com/w/cpp/utility/hash
834ROBIN_HOOD_HASH_INT(unsigned char);
837#if ROBIN_HOOD(HAS_NATIVE_WCHART)
839#endif
841ROBIN_HOOD_HASH_INT(unsigned short);
846ROBIN_HOOD_HASH_INT(unsigned long);
847ROBIN_HOOD_HASH_INT(unsigned long long);
848#if defined(__GNUC__) && !defined(__clang__)
849# pragma GCC diagnostic pop
850#endif
851namespace detail {
852
853template <typename T>
854struct void_type {
855 using type = void;
856};
857
858template <typename T, typename = void>
859struct has_is_transparent : public std::false_type {};
860
861template <typename T>
862struct has_is_transparent<T, typename void_type<typename T::is_transparent>::type>
863 : public std::true_type {};
864
865// using wrapper classes for hash and key_equal prevents the diamond problem when the same type
866// is used. see https://stackoverflow.com/a/28771920/48181
867template <typename T>
868struct WrapHash : public T {
869 WrapHash() = default;
870 explicit WrapHash(T const& o) noexcept(noexcept(T(std::declval<T const&>())))
871 : T(o) {}
872};
873
874template <typename T>
875struct WrapKeyEqual : public T {
876 WrapKeyEqual() = default;
877 explicit WrapKeyEqual(T const& o) noexcept(noexcept(T(std::declval<T const&>())))
878 : T(o) {}
879};
880
881// A highly optimized hashmap implementation, using the Robin Hood algorithm.
882//
883// In most cases, this map should be usable as a drop-in replacement for std::unordered_map, but
884// be about 2x faster in most cases and require much less allocations.
885//
886// This implementation uses the following memory layout:
887//
888// [Node, Node, ... Node | info, info, ... infoSentinel ]
889//
890// * Node: either a DataNode that directly has the std::pair<key, val> as member,
891// or a DataNode with a pointer to std::pair<key,val>. Which DataNode representation to use
892// depends on how fast the swap() operation is. Heuristically, this is automatically choosen
893// based on sizeof(). there are always 2^n Nodes.
894//
895// * info: Each Node in the map has a corresponding info byte, so there are 2^n info bytes.
896// Each byte is initialized to 0, meaning the corresponding Node is empty. Set to 1 means the
897// corresponding node contains data. Set to 2 means the corresponding Node is filled, but it
898// actually belongs to the previous position and was pushed out because that place is already
899// taken.
900//
901// * infoSentinel: Sentinel byte set to 1, so that iterator's ++ can stop at end() without the
902// need for a idx variable.
903//
904// According to STL, order of templates has effect on throughput. That's why I've moved the
905// boolean to the front.
906// https://www.reddit.com/r/cpp/comments/ahp6iu/compile_time_binary_size_reductions_and_cs_future/eeguck4/
907template <bool IsFlat, size_t MaxLoadFactor100, typename Key, typename T, typename Hash,
908 typename KeyEqual>
909class Table
910 : public WrapHash<Hash>,
911 public WrapKeyEqual<KeyEqual>,
913 typename std::conditional<
914 std::is_void<T>::value, Key,
915 robin_hood::pair<typename std::conditional<IsFlat, Key, Key const>::type, T>>::type,
916 4, 16384, IsFlat> {
917public:
918 static constexpr bool is_flat = IsFlat;
919 static constexpr bool is_map = !std::is_void<T>::value;
920 static constexpr bool is_set = !is_map;
921 static constexpr bool is_transparent =
923
924 using key_type = Key;
925 using mapped_type = T;
926 using value_type = typename std::conditional<
927 is_set, Key,
929 using size_type = size_t;
930 using hasher = Hash;
931 using key_equal = KeyEqual;
933
934private:
935 static_assert(MaxLoadFactor100 > 10 && MaxLoadFactor100 < 100,
936 "MaxLoadFactor100 needs to be >10 && < 100");
937
938 using WHash = WrapHash<Hash>;
940
941 // configuration defaults
942
943 // make sure we have 8 elements, needed to quickly rehash mInfo
944 static constexpr size_t InitialNumElements = sizeof(uint64_t);
945 static constexpr uint32_t InitialInfoNumBits = 5;
946 static constexpr uint8_t InitialInfoInc = 1U << InitialInfoNumBits;
947 static constexpr size_t InfoMask = InitialInfoInc - 1U;
948 static constexpr uint8_t InitialInfoHashShift = 0;
950
951 // type needs to be wider than uint8_t.
952 using InfoType = uint32_t;
953
954 // DataNode ////////////////////////////////////////////////////////
955
956 // Primary template for the data node. We have special implementations for small and big
957 // objects. For large objects it is assumed that swap() is fairly slow, so we allocate these
958 // on the heap so swap merely swaps a pointer.
959 template <typename M, bool>
960 class DataNode {};
961
962 // Small: just allocate on the stack.
963 template <typename M>
964 class DataNode<M, true> final {
965 public:
966 template <typename... Args>
967 explicit DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, Args&&... args) noexcept(
968 noexcept(value_type(std::forward<Args>(args)...)))
969 : mData(std::forward<Args>(args)...) {}
970
971 DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode<M, true>&& n) noexcept(
972 std::is_nothrow_move_constructible<value_type>::value)
973 : mData(std::move(n.mData)) {}
974
975 // doesn't do anything
976 void destroy(M& ROBIN_HOOD_UNUSED(map) /*unused*/) noexcept {}
977 void destroyDoNotDeallocate() noexcept {}
978
979 value_type const* operator->() const noexcept {
980 return &mData;
981 }
982 value_type* operator->() noexcept {
983 return &mData;
984 }
985
986 const value_type& operator*() const noexcept {
987 return mData;
988 }
989
990 value_type& operator*() noexcept {
991 return mData;
992 }
993
994 template <typename VT = value_type>
995 ROBIN_HOOD(NODISCARD)
996 typename std::enable_if<is_map, typename VT::first_type&>::type getFirst() noexcept {
997 return mData.first;
998 }
999 template <typename VT = value_type>
1000 ROBIN_HOOD(NODISCARD)
1001 typename std::enable_if<is_set, VT&>::type getFirst() noexcept {
1002 return mData;
1003 }
1004
1005 template <typename VT = value_type>
1006 ROBIN_HOOD(NODISCARD)
1007 typename std::enable_if<is_map, typename VT::first_type const&>::type
1008 getFirst() const noexcept {
1009 return mData.first;
1010 }
1011 template <typename VT = value_type>
1012 ROBIN_HOOD(NODISCARD)
1013 typename std::enable_if<is_set, VT const&>::type getFirst() const noexcept {
1014 return mData;
1015 }
1016
1017 template <typename MT = mapped_type>
1018 ROBIN_HOOD(NODISCARD)
1019 typename std::enable_if<is_map, MT&>::type getSecond() noexcept {
1020 return mData.second;
1021 }
1022
1023 template <typename MT = mapped_type>
1024 ROBIN_HOOD(NODISCARD)
1025 typename std::enable_if<is_set, MT const&>::type getSecond() const noexcept {
1026 return mData.second;
1027 }
1028
1029 void swap(DataNode<M, true>& o) noexcept(
1030 noexcept(std::declval<value_type>().swap(std::declval<value_type>()))) {
1031 mData.swap(o.mData);
1032 }
1033
1034 private:
1035 value_type mData;
1036 };
1037
1038 // big object: allocate on heap.
1039 template <typename M>
1040 class DataNode<M, false> {
1041 public:
1042 template <typename... Args>
1043 explicit DataNode(M& map, Args&&... args)
1044 : mData(map.allocate()) {
1045 ::new (static_cast<void*>(mData)) value_type(std::forward<Args>(args)...);
1046 }
1047
1048 DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode<M, false>&& n) noexcept
1049 : mData(std::move(n.mData)) {}
1050
1051 void destroy(M& map) noexcept {
1052 // don't deallocate, just put it into list of datapool.
1053 mData->~value_type();
1054 map.deallocate(mData);
1055 }
1056
1057 void destroyDoNotDeallocate() noexcept {
1058 mData->~value_type();
1059 }
1060
1061 value_type const* operator->() const noexcept {
1062 return mData;
1063 }
1064
1065 value_type* operator->() noexcept {
1066 return mData;
1067 }
1068
1069 const value_type& operator*() const {
1070 return *mData;
1071 }
1072
1073 value_type& operator*() {
1074 return *mData;
1075 }
1076
1077 template <typename VT = value_type>
1078 ROBIN_HOOD(NODISCARD)
1079 typename std::enable_if<is_map, typename VT::first_type&>::type getFirst() noexcept {
1080 return mData->first;
1081 }
1082 template <typename VT = value_type>
1083 ROBIN_HOOD(NODISCARD)
1084 typename std::enable_if<is_set, VT&>::type getFirst() noexcept {
1085 return *mData;
1086 }
1087
1088 template <typename VT = value_type>
1089 ROBIN_HOOD(NODISCARD)
1090 typename std::enable_if<is_map, typename VT::first_type const&>::type
1091 getFirst() const noexcept {
1092 return mData->first;
1093 }
1094 template <typename VT = value_type>
1095 ROBIN_HOOD(NODISCARD)
1096 typename std::enable_if<is_set, VT const&>::type getFirst() const noexcept {
1097 return *mData;
1098 }
1099
1100 template <typename MT = mapped_type>
1101 ROBIN_HOOD(NODISCARD)
1102 typename std::enable_if<is_map, MT&>::type getSecond() noexcept {
1103 return mData->second;
1104 }
1105
1106 template <typename MT = mapped_type>
1107 ROBIN_HOOD(NODISCARD)
1108 typename std::enable_if<is_map, MT const&>::type getSecond() const noexcept {
1109 return mData->second;
1110 }
1111
1112 void swap(DataNode<M, false>& o) noexcept {
1113 using std::swap;
1114 swap(mData, o.mData);
1115 }
1116
1117 private:
1118 value_type* mData;
1119 };
1120
1121 using Node = DataNode<Self, IsFlat>;
1122
1123 // helpers for insertKeyPrepareEmptySpot: extract first entry (only const required)
1124 ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(Node const& n) const noexcept {
1125 return n.getFirst();
1126 }
1127
1128 // in case we have void mapped_type, we are not using a pair, thus we just route k through.
1129 // No need to disable this because it's just not used if not applicable.
1130 ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(key_type const& k) const noexcept {
1131 return k;
1132 }
1133
1134 // in case we have non-void mapped_type, we have a standard robin_hood::pair
1135 template <typename Q = mapped_type>
1136 ROBIN_HOOD(NODISCARD)
1137 typename std::enable_if<!std::is_void<Q>::value, key_type const&>::type
1138 getFirstConst(value_type const& vt) const noexcept {
1139 return vt.first;
1140 }
1141
1142 // Cloner //////////////////////////////////////////////////////////
1143
1144 template <typename M, bool UseMemcpy>
1145 struct Cloner;
1146
1147 // fast path: Just copy data, without allocating anything.
1148 template <typename M>
1149 struct Cloner<M, true> {
1150 void operator()(M const& source, M& target) const {
1151 auto const* const src = reinterpret_cast<char const*>(source.mKeyVals);
1152 auto* tgt = reinterpret_cast<char*>(target.mKeyVals);
1153 auto const numElementsWithBuffer = target.calcNumElementsWithBuffer(target.mMask + 1);
1154 std::copy(src, src + target.calcNumBytesTotal(numElementsWithBuffer), tgt);
1155 }
1156 };
1157
1158 template <typename M>
1159 struct Cloner<M, false> {
1160 void operator()(M const& s, M& t) const {
1161 auto const numElementsWithBuffer = t.calcNumElementsWithBuffer(t.mMask + 1);
1162 std::copy(s.mInfo, s.mInfo + t.calcNumBytesInfo(numElementsWithBuffer), t.mInfo);
1163
1164 for (size_t i = 0; i < numElementsWithBuffer; ++i) {
1165 if (t.mInfo[i]) {
1166 ::new (static_cast<void*>(t.mKeyVals + i)) Node(t, *s.mKeyVals[i]);
1167 }
1168 }
1169 }
1170 };
1171
1172 // Destroyer ///////////////////////////////////////////////////////
1173
1174 template <typename M, bool IsFlatAndTrivial>
1175 struct Destroyer {};
1176
1177 template <typename M>
1178 struct Destroyer<M, true> {
1179 void nodes(M& m) const noexcept {
1180 m.mNumElements = 0;
1181 }
1182
1183 void nodesDoNotDeallocate(M& m) const noexcept {
1184 m.mNumElements = 0;
1185 }
1186 };
1187
1188 template <typename M>
1189 struct Destroyer<M, false> {
1190 void nodes(M& m) const noexcept {
1191 m.mNumElements = 0;
1192 // clear also resets mInfo to 0, that's sometimes not necessary.
1193 auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1);
1194
1195 for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) {
1196 if (0 != m.mInfo[idx]) {
1197 Node& n = m.mKeyVals[idx];
1198 n.destroy(m);
1199 n.~Node();
1200 }
1201 }
1202 }
1203
1204 void nodesDoNotDeallocate(M& m) const noexcept {
1205 m.mNumElements = 0;
1206 // clear also resets mInfo to 0, that's sometimes not necessary.
1207 auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1);
1208 for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) {
1209 if (0 != m.mInfo[idx]) {
1210 Node& n = m.mKeyVals[idx];
1211 n.destroyDoNotDeallocate();
1212 n.~Node();
1213 }
1214 }
1215 }
1216 };
1217
1218 // Iter ////////////////////////////////////////////////////////////
1219
1220 struct fast_forward_tag {};
1221
1222 // generic iterator for both const_iterator and iterator.
1223 template <bool IsConst>
1224 // NOLINTNEXTLINE(hicpp-special-member-functions,cppcoreguidelines-special-member-functions)
1225 class Iter {
1226 private:
1227 using NodePtr = typename std::conditional<IsConst, Node const*, Node*>::type;
1228
1229 public:
1230 using difference_type = std::ptrdiff_t;
1231 using value_type = typename Self::value_type;
1232 using reference = typename std::conditional<IsConst, value_type const&, value_type&>::type;
1233 using pointer = typename std::conditional<IsConst, value_type const*, value_type*>::type;
1234 using iterator_category = std::forward_iterator_tag;
1235
1236 // default constructed iterator can be compared to itself, but WON'T return true when
1237 // compared to end().
1238 Iter() = default;
1239
1240 // Rule of zero: nothing specified. The conversion constructor is only enabled for
1241 // iterator to const_iterator, so it doesn't accidentally work as a copy ctor.
1242
1243 // Conversion constructor from iterator to const_iterator.
1244 template <bool OtherIsConst,
1245 typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
1246 // NOLINTNEXTLINE(hicpp-explicit-conversions)
1247 Iter(Iter<OtherIsConst> const& other) noexcept
1248 : mKeyVals(other.mKeyVals)
1249 , mInfo(other.mInfo) {}
1250
1251 Iter(NodePtr valPtr, uint8_t const* infoPtr) noexcept
1252 : mKeyVals(valPtr)
1253 , mInfo(infoPtr) {}
1254
1255 Iter(NodePtr valPtr, uint8_t const* infoPtr,
1256 fast_forward_tag ROBIN_HOOD_UNUSED(tag) /*unused*/) noexcept
1257 : mKeyVals(valPtr)
1258 , mInfo(infoPtr) {
1259 fastForward();
1260 }
1261
1262 template <bool OtherIsConst,
1263 typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
1264 Iter& operator=(Iter<OtherIsConst> const& other) noexcept {
1265 mKeyVals = other.mKeyVals;
1266 mInfo = other.mInfo;
1267 return *this;
1268 }
1269
1270 // prefix increment. Undefined behavior if we are at end()!
1271 Iter& operator++() noexcept {
1272 mInfo++;
1273 mKeyVals++;
1274 fastForward();
1275 return *this;
1276 }
1277
1278 Iter operator++(int) noexcept {
1279 Iter tmp = *this;
1280 ++(*this);
1281 return tmp;
1282 }
1283
1284 reference operator*() const {
1285 return **mKeyVals;
1286 }
1287
1288 pointer operator->() const {
1289 return &**mKeyVals;
1290 }
1291
1292 template <bool O>
1293 bool operator==(Iter<O> const& o) const noexcept {
1294 return mKeyVals == o.mKeyVals;
1295 }
1296
1297 template <bool O>
1298 bool operator!=(Iter<O> const& o) const noexcept {
1299 return mKeyVals != o.mKeyVals;
1300 }
1301
1302 private:
1303 // fast forward to the next non-free info byte
1304 // I've tried a few variants that don't depend on intrinsics, but unfortunately they are
1305 // quite a bit slower than this one. So I've reverted that change again. See map_benchmark.
1306 void fastForward() noexcept {
1307 size_t n = 0;
1308 while (0U == (n = detail::unaligned_load<size_t>(mInfo))) {
1309 mInfo += sizeof(size_t);
1310 mKeyVals += sizeof(size_t);
1311 }
1312#if defined(ROBIN_HOOD_DISABLE_INTRINSICS)
1313 // we know for certain that within the next 8 bytes we'll find a non-zero one.
1314 if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load<uint32_t>(mInfo))) {
1315 mInfo += 4;
1316 mKeyVals += 4;
1317 }
1318 if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load<uint16_t>(mInfo))) {
1319 mInfo += 2;
1320 mKeyVals += 2;
1321 }
1322 if (ROBIN_HOOD_UNLIKELY(0U == *mInfo)) {
1323 mInfo += 1;
1324 mKeyVals += 1;
1325 }
1326#else
1327# if ROBIN_HOOD(LITTLE_ENDIAN)
1328 auto inc = ROBIN_HOOD_COUNT_TRAILING_ZEROES(n) / 8;
1329# else
1330 auto inc = ROBIN_HOOD_COUNT_LEADING_ZEROES(n) / 8;
1331# endif
1332 mInfo += inc;
1333 mKeyVals += inc;
1334#endif
1335 }
1336
1337 friend class Table<IsFlat, MaxLoadFactor100, key_type, mapped_type, hasher, key_equal>;
1338 NodePtr mKeyVals{nullptr};
1339 uint8_t const* mInfo{nullptr};
1340 };
1341
1343
1344 // highly performance relevant code.
1345 // Lower bits are used for indexing into the array (2^n size)
1346 // The upper 1-5 bits need to be a reasonable good hash, to save comparisons.
1347 template <typename HashKey>
1348 void keyToIdx(HashKey&& key, size_t* idx, InfoType* info) const {
1349 // In addition to whatever hash is used, add another mul & shift so we get better hashing.
1350 // This serves as a bad hash prevention, if the given data is
1351 // badly mixed.
1352 auto h = static_cast<uint64_t>(WHash::operator()(key));
1353
1354 h *= mHashMultiplier;
1355 h ^= h >> 33U;
1356
1357 // the lower InitialInfoNumBits are reserved for info.
1358 *info = mInfoInc + static_cast<InfoType>((h & InfoMask) >> mInfoHashShift);
1359 *idx = (static_cast<size_t>(h) >> InitialInfoNumBits) & mMask;
1360 }
1361
1362 // forwards the index by one, wrapping around at the end
1363 void next(InfoType* info, size_t* idx) const noexcept {
1364 *idx = *idx + 1;
1365 *info += mInfoInc;
1366 }
1367
1368 void nextWhileLess(InfoType* info, size_t* idx) const noexcept {
1369 // unrolling this by hand did not bring any speedups.
1370 while (*info < mInfo[*idx]) {
1371 next(info, idx);
1372 }
1373 }
1374
1375 // Shift everything up by one element. Tries to move stuff around.
1376 void
1377 shiftUp(size_t startIdx,
1378 size_t const insertion_idx) noexcept(std::is_nothrow_move_assignable<Node>::value) {
1379 auto idx = startIdx;
1380 ::new (static_cast<void*>(mKeyVals + idx)) Node(std::move(mKeyVals[idx - 1]));
1381 while (--idx != insertion_idx) {
1382 mKeyVals[idx] = std::move(mKeyVals[idx - 1]);
1383 }
1384
1385 idx = startIdx;
1386 while (idx != insertion_idx) {
1387 ROBIN_HOOD_COUNT(shiftUp)
1388 mInfo[idx] = static_cast<uint8_t>(mInfo[idx - 1] + mInfoInc);
1389 if (ROBIN_HOOD_UNLIKELY(mInfo[idx] + mInfoInc > 0xFF)) {
1390 mMaxNumElementsAllowed = 0;
1391 }
1392 --idx;
1393 }
1394 }
1395
1396 void shiftDown(size_t idx) noexcept(std::is_nothrow_move_assignable<Node>::value) {
1397 // until we find one that is either empty or has zero offset.
1398 // TODO(martinus) we don't need to move everything, just the last one for the same
1399 // bucket.
1400 mKeyVals[idx].destroy(*this);
1401
1402 // until we find one that is either empty or has zero offset.
1403 while (mInfo[idx + 1] >= 2 * mInfoInc) {
1404 ROBIN_HOOD_COUNT(shiftDown)
1405 mInfo[idx] = static_cast<uint8_t>(mInfo[idx + 1] - mInfoInc);
1406 mKeyVals[idx] = std::move(mKeyVals[idx + 1]);
1407 ++idx;
1408 }
1409
1410 mInfo[idx] = 0;
1411 // don't destroy, we've moved it
1412 // mKeyVals[idx].destroy(*this);
1413 mKeyVals[idx].~Node();
1414 }
1415
1416 // copy of find(), except that it returns iterator instead of const_iterator.
1417 template <typename Other>
1418 ROBIN_HOOD(NODISCARD)
1419 size_t findIdx(Other const& key) const {
1420 size_t idx{};
1421 InfoType info{};
1422 keyToIdx(key, &idx, &info);
1423
1424 do {
1425 // unrolling this twice gives a bit of a speedup. More unrolling did not help.
1426 if (info == mInfo[idx] &&
1427 ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) {
1428 return idx;
1429 }
1430 next(&info, &idx);
1431 if (info == mInfo[idx] &&
1432 ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) {
1433 return idx;
1434 }
1435 next(&info, &idx);
1436 } while (info <= mInfo[idx]);
1437
1438 // nothing found!
1439 return mMask == 0 ? 0
1440 : static_cast<size_t>(std::distance(
1441 mKeyVals, reinterpret_cast_no_cast_align_warning<Node*>(mInfo)));
1442 }
1443
1444 void cloneData(const Table& o) {
1445 Cloner<Table, IsFlat && ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(Node)>()(o, *this);
1446 }
1447
1448 // inserts a keyval that is guaranteed to be new, e.g. when the hashmap is resized.
1449 // @return True on success, false if something went wrong
1450 void insert_move(Node&& keyval) {
1451 // we don't retry, fail if overflowing
1452 // don't need to check max num elements
1453 if (0 == mMaxNumElementsAllowed && !try_increase_info()) {
1454 throwOverflowError();
1455 }
1456
1457 size_t idx{};
1458 InfoType info{};
1459 keyToIdx(keyval.getFirst(), &idx, &info);
1460
1461 // skip forward. Use <= because we are certain that the element is not there.
1462 while (info <= mInfo[idx]) {
1463 idx = idx + 1;
1464 info += mInfoInc;
1465 }
1466
1467 // key not found, so we are now exactly where we want to insert it.
1468 auto const insertion_idx = idx;
1469 auto const insertion_info = static_cast<uint8_t>(info);
1470 if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) {
1471 mMaxNumElementsAllowed = 0;
1472 }
1473
1474 // find an empty spot
1475 while (0 != mInfo[idx]) {
1476 next(&info, &idx);
1477 }
1478
1479 auto& l = mKeyVals[insertion_idx];
1480 if (idx == insertion_idx) {
1481 ::new (static_cast<void*>(&l)) Node(std::move(keyval));
1482 } else {
1483 shiftUp(idx, insertion_idx);
1484 l = std::move(keyval);
1485 }
1486
1487 // put at empty spot
1488 mInfo[insertion_idx] = insertion_info;
1489
1490 ++mNumElements;
1491 }
1492
1493public:
1494 using iterator = Iter<false>;
1495 using const_iterator = Iter<true>;
1496
1497 Table() noexcept(noexcept(Hash()) && noexcept(KeyEqual()))
1498 : WHash()
1499 , WKeyEqual() {
1500 ROBIN_HOOD_TRACE(this)
1501 }
1502
1503 // Creates an empty hash map. Nothing is allocated yet, this happens at the first insert.
1504 // This tremendously speeds up ctor & dtor of a map that never receives an element. The
1505 // penalty is payed at the first insert, and not before. Lookup of this empty map works
1506 // because everybody points to DummyInfoByte::b. parameter bucket_count is dictated by the
1507 // standard, but we can ignore it.
1508 explicit Table(
1509 size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/, const Hash& h = Hash{},
1510 const KeyEqual& equal = KeyEqual{}) noexcept(noexcept(Hash(h)) && noexcept(KeyEqual(equal)))
1511 : WHash(h)
1512 , WKeyEqual(equal) {
1513 ROBIN_HOOD_TRACE(this)
1514 }
1515
1516 template <typename Iter>
1517 Table(Iter first, Iter last, size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0,
1518 const Hash& h = Hash{}, const KeyEqual& equal = KeyEqual{})
1519 : WHash(h)
1520 , WKeyEqual(equal) {
1521 ROBIN_HOOD_TRACE(this)
1522 insert(first, last);
1523 }
1524
1525 Table(std::initializer_list<value_type> initlist,
1526 size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash& h = Hash{},
1527 const KeyEqual& equal = KeyEqual{})
1528 : WHash(h)
1529 , WKeyEqual(equal) {
1530 ROBIN_HOOD_TRACE(this)
1531 insert(initlist.begin(), initlist.end());
1532 }
1533
1534 Table(Table&& o) noexcept
1535 : WHash(std::move(static_cast<WHash&>(o)))
1536 , WKeyEqual(std::move(static_cast<WKeyEqual&>(o)))
1537 , DataPool(std::move(static_cast<DataPool&>(o))) {
1538 ROBIN_HOOD_TRACE(this)
1539 if (o.mMask) {
1540 mHashMultiplier = std::move(o.mHashMultiplier);
1541 mKeyVals = std::move(o.mKeyVals);
1542 mInfo = std::move(o.mInfo);
1543 mNumElements = std::move(o.mNumElements);
1544 mMask = std::move(o.mMask);
1545 mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed);
1546 mInfoInc = std::move(o.mInfoInc);
1547 mInfoHashShift = std::move(o.mInfoHashShift);
1548 // set other's mask to 0 so its destructor won't do anything
1549 o.init();
1550 }
1551 }
1552
1553 Table& operator=(Table&& o) noexcept {
1554 ROBIN_HOOD_TRACE(this)
1555 if (&o != this) {
1556 if (o.mMask) {
1557 // only move stuff if the other map actually has some data
1558 destroy();
1559 mHashMultiplier = std::move(o.mHashMultiplier);
1560 mKeyVals = std::move(o.mKeyVals);
1561 mInfo = std::move(o.mInfo);
1562 mNumElements = std::move(o.mNumElements);
1563 mMask = std::move(o.mMask);
1564 mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed);
1565 mInfoInc = std::move(o.mInfoInc);
1566 mInfoHashShift = std::move(o.mInfoHashShift);
1567 WHash::operator=(std::move(static_cast<WHash&>(o)));
1568 WKeyEqual::operator=(std::move(static_cast<WKeyEqual&>(o)));
1569 DataPool::operator=(std::move(static_cast<DataPool&>(o)));
1570
1571 o.init();
1572
1573 } else {
1574 // nothing in the other map => just clear us.
1575 clear();
1576 }
1577 }
1578 return *this;
1579 }
1580
1581 Table(const Table& o)
1582 : WHash(static_cast<const WHash&>(o))
1583 , WKeyEqual(static_cast<const WKeyEqual&>(o))
1584 , DataPool(static_cast<const DataPool&>(o)) {
1585 ROBIN_HOOD_TRACE(this)
1586 if (!o.empty()) {
1587 // not empty: create an exact copy. it is also possible to just iterate through all
1588 // elements and insert them, but copying is probably faster.
1589
1590 auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1);
1591 auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
1592
1593 ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal("
1594 << numElementsWithBuffer << ")")
1595 mHashMultiplier = o.mHashMultiplier;
1596 mKeyVals = static_cast<Node*>(
1597 detail::assertNotNull<std::bad_alloc>(std::malloc(numBytesTotal)));
1598 // no need for calloc because clonData does memcpy
1599 mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
1600 mNumElements = o.mNumElements;
1601 mMask = o.mMask;
1602 mMaxNumElementsAllowed = o.mMaxNumElementsAllowed;
1603 mInfoInc = o.mInfoInc;
1604 mInfoHashShift = o.mInfoHashShift;
1605 cloneData(o);
1606 }
1607 }
1608
1609 // Creates a copy of the given map. Copy constructor of each entry is used.
1610 // Not sure why clang-tidy thinks this doesn't handle self assignment, it does
1611 // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp)
1612 Table& operator=(Table const& o) {
1613 ROBIN_HOOD_TRACE(this)
1614 if (&o == this) {
1615 // prevent assigning of itself
1616 return *this;
1617 }
1618
1619 // we keep using the old allocator and not assign the new one, because we want to keep
1620 // the memory available. when it is the same size.
1621 if (o.empty()) {
1622 if (0 == mMask) {
1623 // nothing to do, we are empty too
1624 return *this;
1625 }
1626
1627 // not empty: destroy what we have there
1628 // clear also resets mInfo to 0, that's sometimes not necessary.
1629 destroy();
1630 init();
1631 WHash::operator=(static_cast<const WHash&>(o));
1632 WKeyEqual::operator=(static_cast<const WKeyEqual&>(o));
1633 DataPool::operator=(static_cast<DataPool const&>(o));
1634
1635 return *this;
1636 }
1637
1638 // clean up old stuff
1639 Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}.nodes(*this);
1640
1641 if (mMask != o.mMask) {
1642 // no luck: we don't have the same array size allocated, so we need to realloc.
1643 if (0 != mMask) {
1644 // only deallocate if we actually have data!
1645 ROBIN_HOOD_LOG("std::free")
1646 std::free(mKeyVals);
1647 }
1648
1649 auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1);
1650 auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
1651 ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal("
1652 << numElementsWithBuffer << ")")
1653 mKeyVals = static_cast<Node*>(
1654 detail::assertNotNull<std::bad_alloc>(std::malloc(numBytesTotal)));
1655
1656 // no need for calloc here because cloneData performs a memcpy.
1657 mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
1658 // sentinel is set in cloneData
1659 }
1660 WHash::operator=(static_cast<const WHash&>(o));
1661 WKeyEqual::operator=(static_cast<const WKeyEqual&>(o));
1662 DataPool::operator=(static_cast<DataPool const&>(o));
1663 mHashMultiplier = o.mHashMultiplier;
1664 mNumElements = o.mNumElements;
1665 mMask = o.mMask;
1666 mMaxNumElementsAllowed = o.mMaxNumElementsAllowed;
1667 mInfoInc = o.mInfoInc;
1668 mInfoHashShift = o.mInfoHashShift;
1669 cloneData(o);
1670
1671 return *this;
1672 }
1673
1674 // Swaps everything between the two maps.
1675 void swap(Table& o) {
1676 ROBIN_HOOD_TRACE(this)
1677 using std::swap;
1678 swap(o, *this);
1679 }
1680
1681 // Clears all data, without resizing.
1682 void clear() {
1683 ROBIN_HOOD_TRACE(this)
1684 if (empty()) {
1685 // don't do anything! also important because we don't want to write to
1686 // DummyInfoByte::b, even though we would just write 0 to it.
1687 return;
1688 }
1689
1690 Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}.nodes(*this);
1691
1692 auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
1693 // clear everything, then set the sentinel again
1694 uint8_t const z = 0;
1695 std::fill(mInfo, mInfo + calcNumBytesInfo(numElementsWithBuffer), z);
1696 mInfo[numElementsWithBuffer] = 1;
1697
1698 mInfoInc = InitialInfoInc;
1699 mInfoHashShift = InitialInfoHashShift;
1700 }
1701
1702 // Destroys the map and all it's contents.
1704 ROBIN_HOOD_TRACE(this)
1705 destroy();
1706 }
1707
1708 // Checks if both tables contain the same entries. Order is irrelevant.
1709 bool operator==(const Table& other) const {
1710 ROBIN_HOOD_TRACE(this)
1711 if (other.size() != size()) {
1712 return false;
1713 }
1714 for (auto const& otherEntry : other) {
1715 if (!has(otherEntry)) {
1716 return false;
1717 }
1718 }
1719
1720 return true;
1721 }
1722
1723 bool operator!=(const Table& other) const {
1724 ROBIN_HOOD_TRACE(this)
1725 return !operator==(other);
1726 }
1727
1728 template <typename Q = mapped_type>
1729 typename std::enable_if<!std::is_void<Q>::value, Q&>::type operator[](const key_type& key) {
1730 ROBIN_HOOD_TRACE(this)
1731 auto idxAndState = insertKeyPrepareEmptySpot(key);
1732 switch (idxAndState.second) {
1733 case InsertionState::key_found:
1734 break;
1735
1736 case InsertionState::new_node:
1737 ::new (static_cast<void*>(&mKeyVals[idxAndState.first]))
1738 Node(*this, std::piecewise_construct, std::forward_as_tuple(key),
1739 std::forward_as_tuple());
1740 break;
1741
1742 case InsertionState::overwrite_node:
1743 mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct,
1744 std::forward_as_tuple(key), std::forward_as_tuple());
1745 break;
1746
1747 case InsertionState::overflow_error:
1748 throwOverflowError();
1749 }
1750
1751 return mKeyVals[idxAndState.first].getSecond();
1752 }
1753
1754 template <typename Q = mapped_type>
1755 typename std::enable_if<!std::is_void<Q>::value, Q&>::type operator[](key_type&& key) {
1756 ROBIN_HOOD_TRACE(this)
1757 auto idxAndState = insertKeyPrepareEmptySpot(key);
1758 switch (idxAndState.second) {
1759 case InsertionState::key_found:
1760 break;
1761
1762 case InsertionState::new_node:
1763 ::new (static_cast<void*>(&mKeyVals[idxAndState.first]))
1764 Node(*this, std::piecewise_construct, std::forward_as_tuple(std::move(key)),
1765 std::forward_as_tuple());
1766 break;
1767
1768 case InsertionState::overwrite_node:
1769 mKeyVals[idxAndState.first] =
1770 Node(*this, std::piecewise_construct, std::forward_as_tuple(std::move(key)),
1771 std::forward_as_tuple());
1772 break;
1773
1774 case InsertionState::overflow_error:
1775 throwOverflowError();
1776 }
1777
1778 return mKeyVals[idxAndState.first].getSecond();
1779 }
1780
1781 template <typename Iter>
1782 void insert(Iter first, Iter last) {
1783 for (; first != last; ++first) {
1784 // value_type ctor needed because this might be called with std::pair's
1785 insert(value_type(*first));
1786 }
1787 }
1788
1789 void insert(std::initializer_list<value_type> ilist) {
1790 for (auto&& vt : ilist) {
1791 insert(std::move(vt));
1792 }
1793 }
1794
1795 template <typename... Args>
1796 std::pair<iterator, bool> emplace(Args&&... args) {
1797 ROBIN_HOOD_TRACE(this)
1798 Node n{*this, std::forward<Args>(args)...};
1799 auto idxAndState = insertKeyPrepareEmptySpot(getFirstConst(n));
1800 switch (idxAndState.second) {
1801 case InsertionState::key_found:
1802 n.destroy(*this);
1803 break;
1804
1805 case InsertionState::new_node:
1806 ::new (static_cast<void*>(&mKeyVals[idxAndState.first])) Node(*this, std::move(n));
1807 break;
1808
1809 case InsertionState::overwrite_node:
1810 mKeyVals[idxAndState.first] = std::move(n);
1811 break;
1812
1813 case InsertionState::overflow_error:
1814 n.destroy(*this);
1815 throwOverflowError();
1816 break;
1817 }
1818
1819 return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first),
1820 InsertionState::key_found != idxAndState.second);
1821 }
1822
1823 template <typename... Args>
1824 iterator emplace_hint(const_iterator position, Args&&... args) {
1825 (void)position;
1826 return emplace(std::forward<Args>(args)...).first;
1827 }
1828
1829 template <typename... Args>
1830 std::pair<iterator, bool> try_emplace(const key_type& key, Args&&... args) {
1831 return try_emplace_impl(key, std::forward<Args>(args)...);
1832 }
1833
1834 template <typename... Args>
1835 std::pair<iterator, bool> try_emplace(key_type&& key, Args&&... args) {
1836 return try_emplace_impl(std::move(key), std::forward<Args>(args)...);
1837 }
1838
1839 template <typename... Args>
1840 iterator try_emplace(const_iterator hint, const key_type& key, Args&&... args) {
1841 (void)hint;
1842 return try_emplace_impl(key, std::forward<Args>(args)...).first;
1843 }
1844
1845 template <typename... Args>
1846 iterator try_emplace(const_iterator hint, key_type&& key, Args&&... args) {
1847 (void)hint;
1848 return try_emplace_impl(std::move(key), std::forward<Args>(args)...).first;
1849 }
1850
1851 template <typename Mapped>
1852 std::pair<iterator, bool> insert_or_assign(const key_type& key, Mapped&& obj) {
1853 return insertOrAssignImpl(key, std::forward<Mapped>(obj));
1854 }
1855
1856 template <typename Mapped>
1857 std::pair<iterator, bool> insert_or_assign(key_type&& key, Mapped&& obj) {
1858 return insertOrAssignImpl(std::move(key), std::forward<Mapped>(obj));
1859 }
1860
1861 template <typename Mapped>
1862 iterator insert_or_assign(const_iterator hint, const key_type& key, Mapped&& obj) {
1863 (void)hint;
1864 return insertOrAssignImpl(key, std::forward<Mapped>(obj)).first;
1865 }
1866
1867 template <typename Mapped>
1869 (void)hint;
1870 return insertOrAssignImpl(std::move(key), std::forward<Mapped>(obj)).first;
1871 }
1872
1873 std::pair<iterator, bool> insert(const value_type& keyval) {
1874 ROBIN_HOOD_TRACE(this)
1875 return emplace(keyval);
1876 }
1877
1879 (void)hint;
1880 return emplace(keyval).first;
1881 }
1882
1883 std::pair<iterator, bool> insert(value_type&& keyval) {
1884 return emplace(std::move(keyval));
1885 }
1886
1888 (void)hint;
1889 return emplace(std::move(keyval)).first;
1890 }
1891
1892 // Returns 1 if key is found, 0 otherwise.
1893 size_t count(const key_type& key) const { // NOLINT(modernize-use-nodiscard)
1894 ROBIN_HOOD_TRACE(this)
1895 auto kv = mKeyVals + findIdx(key);
1896 if (kv != reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
1897 return 1;
1898 }
1899 return 0;
1900 }
1901
1902 template <typename OtherKey, typename Self_ = Self>
1903 // NOLINTNEXTLINE(modernize-use-nodiscard)
1904 typename std::enable_if<Self_::is_transparent, size_t>::type count(const OtherKey& key) const {
1905 ROBIN_HOOD_TRACE(this)
1906 auto kv = mKeyVals + findIdx(key);
1907 if (kv != reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
1908 return 1;
1909 }
1910 return 0;
1911 }
1912
1913 bool contains(const key_type& key) const { // NOLINT(modernize-use-nodiscard)
1914 return 1U == count(key);
1915 }
1916
1917 template <typename OtherKey, typename Self_ = Self>
1918 // NOLINTNEXTLINE(modernize-use-nodiscard)
1919 typename std::enable_if<Self_::is_transparent, bool>::type contains(const OtherKey& key) const {
1920 return 1U == count(key);
1921 }
1922
1923 // Returns a reference to the value found for key.
1924 // Throws std::out_of_range if element cannot be found
1925 template <typename Q = mapped_type>
1926 // NOLINTNEXTLINE(modernize-use-nodiscard)
1927 typename std::enable_if<!std::is_void<Q>::value, Q&>::type at(key_type const& key) {
1928 ROBIN_HOOD_TRACE(this)
1929 auto kv = mKeyVals + findIdx(key);
1930 if (kv == reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
1931 doThrow<std::out_of_range>("key not found");
1932 }
1933 return kv->getSecond();
1934 }
1935
1936 // Returns a reference to the value found for key.
1937 // Throws std::out_of_range if element cannot be found
1938 template <typename Q = mapped_type>
1939 // NOLINTNEXTLINE(modernize-use-nodiscard)
1940 typename std::enable_if<!std::is_void<Q>::value, Q const&>::type at(key_type const& key) const {
1941 ROBIN_HOOD_TRACE(this)
1942 auto kv = mKeyVals + findIdx(key);
1943 if (kv == reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
1944 doThrow<std::out_of_range>("key not found");
1945 }
1946 return kv->getSecond();
1947 }
1948
1949 const_iterator find(const key_type& key) const { // NOLINT(modernize-use-nodiscard)
1950 ROBIN_HOOD_TRACE(this)
1951 const size_t idx = findIdx(key);
1952 return const_iterator{mKeyVals + idx, mInfo + idx};
1953 }
1954
1955 template <typename OtherKey>
1956 const_iterator find(const OtherKey& key, is_transparent_tag /*unused*/) const {
1957 ROBIN_HOOD_TRACE(this)
1958 const size_t idx = findIdx(key);
1959 return const_iterator{mKeyVals + idx, mInfo + idx};
1960 }
1961
1962 template <typename OtherKey, typename Self_ = Self>
1963 typename std::enable_if<Self_::is_transparent, // NOLINT(modernize-use-nodiscard)
1964 const_iterator>::type // NOLINT(modernize-use-nodiscard)
1965 find(const OtherKey& key) const { // NOLINT(modernize-use-nodiscard)
1966 ROBIN_HOOD_TRACE(this)
1967 const size_t idx = findIdx(key);
1968 return const_iterator{mKeyVals + idx, mInfo + idx};
1969 }
1970
1971 iterator find(const key_type& key) {
1972 ROBIN_HOOD_TRACE(this)
1973 const size_t idx = findIdx(key);
1974 return iterator{mKeyVals + idx, mInfo + idx};
1975 }
1976
1977 template <typename OtherKey>
1978 iterator find(const OtherKey& key, is_transparent_tag /*unused*/) {
1979 ROBIN_HOOD_TRACE(this)
1980 const size_t idx = findIdx(key);
1981 return iterator{mKeyVals + idx, mInfo + idx};
1982 }
1983
1984 template <typename OtherKey, typename Self_ = Self>
1985 typename std::enable_if<Self_::is_transparent, iterator>::type find(const OtherKey& key) {
1986 ROBIN_HOOD_TRACE(this)
1987 const size_t idx = findIdx(key);
1988 return iterator{mKeyVals + idx, mInfo + idx};
1989 }
1990
1992 ROBIN_HOOD_TRACE(this)
1993 if (empty()) {
1994 return end();
1995 }
1996 return iterator(mKeyVals, mInfo, fast_forward_tag{});
1997 }
1998 const_iterator begin() const { // NOLINT(modernize-use-nodiscard)
1999 ROBIN_HOOD_TRACE(this)
2000 return cbegin();
2001 }
2002 const_iterator cbegin() const { // NOLINT(modernize-use-nodiscard)
2003 ROBIN_HOOD_TRACE(this)
2004 if (empty()) {
2005 return cend();
2006 }
2007 return const_iterator(mKeyVals, mInfo, fast_forward_tag{});
2008 }
2009
2011 ROBIN_HOOD_TRACE(this)
2012 // no need to supply valid info pointer: end() must not be dereferenced, and only node
2013 // pointer is compared.
2014 return iterator{reinterpret_cast_no_cast_align_warning<Node*>(mInfo), nullptr};
2015 }
2016 const_iterator end() const { // NOLINT(modernize-use-nodiscard)
2017 ROBIN_HOOD_TRACE(this)
2018 return cend();
2019 }
2020 const_iterator cend() const { // NOLINT(modernize-use-nodiscard)
2021 ROBIN_HOOD_TRACE(this)
2022 return const_iterator{reinterpret_cast_no_cast_align_warning<Node*>(mInfo), nullptr};
2023 }
2024
2026 ROBIN_HOOD_TRACE(this)
2027 // its safe to perform const cast here
2028 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast)
2029 return erase(iterator{const_cast<Node*>(pos.mKeyVals), const_cast<uint8_t*>(pos.mInfo)});
2030 }
2031
2032 // Erases element at pos, returns iterator to the next element.
2034 ROBIN_HOOD_TRACE(this)
2035 // we assume that pos always points to a valid entry, and not end().
2036 auto const idx = static_cast<size_t>(pos.mKeyVals - mKeyVals);
2037
2038 shiftDown(idx);
2039 --mNumElements;
2040
2041 if (*pos.mInfo) {
2042 // we've backward shifted, return this again
2043 return pos;
2044 }
2045
2046 // no backward shift, return next element
2047 return ++pos;
2048 }
2049
2050 size_t erase(const key_type& key) {
2051 ROBIN_HOOD_TRACE(this)
2052 size_t idx{};
2053 InfoType info{};
2054 keyToIdx(key, &idx, &info);
2055
2056 // check while info matches with the source idx
2057 do {
2058 if (info == mInfo[idx] && WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) {
2059 shiftDown(idx);
2060 --mNumElements;
2061 return 1;
2062 }
2063 next(&info, &idx);
2064 } while (info <= mInfo[idx]);
2065
2066 // nothing found to delete
2067 return 0;
2068 }
2069
2070 // reserves space for the specified number of elements. Makes sure the old data fits.
2071 // exactly the same as reserve(c).
2072 void rehash(size_t c) {
2073 // forces a reserve
2074 reserve(c, true);
2075 }
2076
2077 // reserves space for the specified number of elements. Makes sure the old data fits.
2078 // Exactly the same as rehash(c). Use rehash(0) to shrink to fit.
2079 void reserve(size_t c) {
2080 // reserve, but don't force rehash
2081 reserve(c, false);
2082 }
2083
2084 // If possible reallocates the map to a smaller one. This frees the underlying table.
2085 // Does not do anything if load_factor is too large for decreasing the table's size.
2086 void compact() {
2087 ROBIN_HOOD_TRACE(this)
2088 auto newSize = InitialNumElements;
2089 while (calcMaxNumElementsAllowed(newSize) < mNumElements && newSize != 0) {
2090 newSize *= 2;
2091 }
2092 if (ROBIN_HOOD_UNLIKELY(newSize == 0)) {
2093 throwOverflowError();
2094 }
2095
2096 ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1")
2097
2098 // only actually do anything when the new size is bigger than the old one. This prevents to
2099 // continuously allocate for each reserve() call.
2100 if (newSize < mMask + 1) {
2101 rehashPowerOfTwo(newSize, true);
2102 }
2103 }
2104
2105 size_type size() const noexcept { // NOLINT(modernize-use-nodiscard)
2106 ROBIN_HOOD_TRACE(this)
2107 return mNumElements;
2108 }
2109
2110 size_type max_size() const noexcept { // NOLINT(modernize-use-nodiscard)
2111 ROBIN_HOOD_TRACE(this)
2112 return static_cast<size_type>(-1);
2113 }
2114
2115 ROBIN_HOOD(NODISCARD) bool empty() const noexcept {
2116 ROBIN_HOOD_TRACE(this)
2117 return 0 == mNumElements;
2118 }
2119
2120 float max_load_factor() const noexcept { // NOLINT(modernize-use-nodiscard)
2121 ROBIN_HOOD_TRACE(this)
2122 return MaxLoadFactor100 / 100.0F;
2123 }
2124
2125 // Average number of elements per bucket. Since we allow only 1 per bucket
2126 float load_factor() const noexcept { // NOLINT(modernize-use-nodiscard)
2127 ROBIN_HOOD_TRACE(this)
2128 return static_cast<float>(size()) / static_cast<float>(mMask + 1);
2129 }
2130
2131 ROBIN_HOOD(NODISCARD) size_t mask() const noexcept {
2132 ROBIN_HOOD_TRACE(this)
2133 return mMask;
2134 }
2135
2136 ROBIN_HOOD(NODISCARD) size_t calcMaxNumElementsAllowed(size_t maxElements) const noexcept {
2137 if (ROBIN_HOOD_LIKELY(maxElements <= (std::numeric_limits<size_t>::max)() / 100)) {
2138 return maxElements * MaxLoadFactor100 / 100;
2139 }
2140
2141 // we might be a bit inprecise, but since maxElements is quite large that doesn't matter
2142 return (maxElements / 100) * MaxLoadFactor100;
2143 }
2144
2145 ROBIN_HOOD(NODISCARD) size_t calcNumBytesInfo(size_t numElements) const noexcept {
2146 // we add a uint64_t, which houses the sentinel (first byte) and padding so we can load
2147 // 64bit types.
2148 return numElements + sizeof(uint64_t);
2149 }
2150
2151 ROBIN_HOOD(NODISCARD)
2152 size_t calcNumElementsWithBuffer(size_t numElements) const noexcept {
2153 auto maxNumElementsAllowed = calcMaxNumElementsAllowed(numElements);
2154 return numElements + (std::min)(maxNumElementsAllowed, (static_cast<size_t>(0xFF)));
2155 }
2156
2157 // calculation only allowed for 2^n values
2158 ROBIN_HOOD(NODISCARD) size_t calcNumBytesTotal(size_t numElements) const {
2159#if ROBIN_HOOD(BITNESS) == 64
2160 return numElements * sizeof(Node) + calcNumBytesInfo(numElements);
2161#else
2162 // make sure we're doing 64bit operations, so we are at least safe against 32bit overflows.
2163 auto const ne = static_cast<uint64_t>(numElements);
2164 auto const s = static_cast<uint64_t>(sizeof(Node));
2165 auto const infos = static_cast<uint64_t>(calcNumBytesInfo(numElements));
2166
2167 auto const total64 = ne * s + infos;
2168 auto const total = static_cast<size_t>(total64);
2169
2170 if (ROBIN_HOOD_UNLIKELY(static_cast<uint64_t>(total) != total64)) {
2171 throwOverflowError();
2172 }
2173 return total;
2174#endif
2175 }
2176
2177private:
2178 template <typename Q = mapped_type>
2179 ROBIN_HOOD(NODISCARD)
2180 typename std::enable_if<!std::is_void<Q>::value, bool>::type has(const value_type& e) const {
2181 ROBIN_HOOD_TRACE(this)
2182 auto it = find(e.first);
2183 return it != end() && it->second == e.second;
2184 }
2185
2186 template <typename Q = mapped_type>
2187 ROBIN_HOOD(NODISCARD)
2188 typename std::enable_if<std::is_void<Q>::value, bool>::type has(const value_type& e) const {
2189 ROBIN_HOOD_TRACE(this)
2190 return find(e) != end();
2191 }
2192
2193 void reserve(size_t c, bool forceRehash) {
2194 ROBIN_HOOD_TRACE(this)
2195 auto const minElementsAllowed = (std::max)(c, mNumElements);
2196 auto newSize = InitialNumElements;
2197 while (calcMaxNumElementsAllowed(newSize) < minElementsAllowed && newSize != 0) {
2198 newSize *= 2;
2199 }
2200 if (ROBIN_HOOD_UNLIKELY(newSize == 0)) {
2201 throwOverflowError();
2202 }
2203
2204 ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1")
2205
2206 // only actually do anything when the new size is bigger than the old one. This prevents to
2207 // continuously allocate for each reserve() call.
2208 if (forceRehash || newSize > mMask + 1) {
2209 rehashPowerOfTwo(newSize, false);
2210 }
2211 }
2212
2213 // reserves space for at least the specified number of elements.
2214 // only works if numBuckets if power of two
2215 // True on success, false otherwise
2216 void rehashPowerOfTwo(size_t numBuckets, bool forceFree) {
2217 ROBIN_HOOD_TRACE(this)
2218
2219 Node* const oldKeyVals = mKeyVals;
2220 uint8_t const* const oldInfo = mInfo;
2221
2222 const size_t oldMaxElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
2223
2224 // resize operation: move stuff
2225 initData(numBuckets);
2226 if (oldMaxElementsWithBuffer > 1) {
2227 for (size_t i = 0; i < oldMaxElementsWithBuffer; ++i) {
2228 if (oldInfo[i] != 0) {
2229 // might throw an exception, which is really bad since we are in the middle of
2230 // moving stuff.
2231 insert_move(std::move(oldKeyVals[i]));
2232 // destroy the node but DON'T destroy the data.
2233 oldKeyVals[i].~Node();
2234 }
2235 }
2236
2237 // this check is not necessary as it's guarded by the previous if, but it helps
2238 // silence g++'s overeager "attempt to free a non-heap object 'map'
2239 // [-Werror=free-nonheap-object]" warning.
2240 if (oldKeyVals != reinterpret_cast_no_cast_align_warning<Node*>(&mMask)) {
2241 // don't destroy old data: put it into the pool instead
2242 if (forceFree) {
2243 std::free(oldKeyVals);
2244 } else {
2245 DataPool::addOrFree(oldKeyVals, calcNumBytesTotal(oldMaxElementsWithBuffer));
2246 }
2247 }
2248 }
2249 }
2250
2251 ROBIN_HOOD(NOINLINE) void throwOverflowError() const {
2252#if ROBIN_HOOD(HAS_EXCEPTIONS)
2253 throw std::overflow_error("robin_hood::map overflow");
2254#else
2255 abort();
2256#endif
2257 }
2258
2259 template <typename OtherKey, typename... Args>
2260 std::pair<iterator, bool> try_emplace_impl(OtherKey&& key, Args&&... args) {
2261 ROBIN_HOOD_TRACE(this)
2262 auto idxAndState = insertKeyPrepareEmptySpot(key);
2263 switch (idxAndState.second) {
2264 case InsertionState::key_found:
2265 break;
2266
2267 case InsertionState::new_node:
2268 ::new (static_cast<void*>(&mKeyVals[idxAndState.first])) Node(
2269 *this, std::piecewise_construct, std::forward_as_tuple(std::forward<OtherKey>(key)),
2270 std::forward_as_tuple(std::forward<Args>(args)...));
2271 break;
2272
2273 case InsertionState::overwrite_node:
2274 mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct,
2275 std::forward_as_tuple(std::forward<OtherKey>(key)),
2276 std::forward_as_tuple(std::forward<Args>(args)...));
2277 break;
2278
2279 case InsertionState::overflow_error:
2280 throwOverflowError();
2281 break;
2282 }
2283
2284 return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first),
2285 InsertionState::key_found != idxAndState.second);
2286 }
2287
2288 template <typename OtherKey, typename Mapped>
2289 std::pair<iterator, bool> insertOrAssignImpl(OtherKey&& key, Mapped&& obj) {
2290 ROBIN_HOOD_TRACE(this)
2291 auto idxAndState = insertKeyPrepareEmptySpot(key);
2292 switch (idxAndState.second) {
2293 case InsertionState::key_found:
2294 mKeyVals[idxAndState.first].getSecond() = std::forward<Mapped>(obj);
2295 break;
2296
2297 case InsertionState::new_node:
2298 ::new (static_cast<void*>(&mKeyVals[idxAndState.first])) Node(
2299 *this, std::piecewise_construct, std::forward_as_tuple(std::forward<OtherKey>(key)),
2300 std::forward_as_tuple(std::forward<Mapped>(obj)));
2301 break;
2302
2303 case InsertionState::overwrite_node:
2304 mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct,
2305 std::forward_as_tuple(std::forward<OtherKey>(key)),
2306 std::forward_as_tuple(std::forward<Mapped>(obj)));
2307 break;
2308
2309 case InsertionState::overflow_error:
2310 throwOverflowError();
2311 break;
2312 }
2313
2314 return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first),
2315 InsertionState::key_found != idxAndState.second);
2316 }
2317
2318 void initData(size_t max_elements) {
2319 mNumElements = 0;
2320 mMask = max_elements - 1;
2321 mMaxNumElementsAllowed = calcMaxNumElementsAllowed(max_elements);
2322
2323 auto const numElementsWithBuffer = calcNumElementsWithBuffer(max_elements);
2324
2325 // malloc & zero mInfo. Faster than calloc everything.
2326 auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
2327 ROBIN_HOOD_LOG("std::calloc " << numBytesTotal << " = calcNumBytesTotal("
2328 << numElementsWithBuffer << ")")
2329 mKeyVals = reinterpret_cast<Node*>(
2330 detail::assertNotNull<std::bad_alloc>(std::malloc(numBytesTotal)));
2331 mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
2332 std::memset(mInfo, 0, numBytesTotal - numElementsWithBuffer * sizeof(Node));
2333
2334 // set sentinel
2335 mInfo[numElementsWithBuffer] = 1;
2336
2337 mInfoInc = InitialInfoInc;
2338 mInfoHashShift = InitialInfoHashShift;
2339 }
2340
2341 enum class InsertionState { overflow_error, key_found, new_node, overwrite_node };
2342
2343 // Finds key, and if not already present prepares a spot where to pot the key & value.
2344 // This potentially shifts nodes out of the way, updates mInfo and number of inserted
2345 // elements, so the only operation left to do is create/assign a new node at that spot.
2346 template <typename OtherKey>
2347 std::pair<size_t, InsertionState> insertKeyPrepareEmptySpot(OtherKey&& key) {
2348 for (int i = 0; i < 256; ++i) {
2349 size_t idx{};
2350 InfoType info{};
2351 keyToIdx(key, &idx, &info);
2352 nextWhileLess(&info, &idx);
2353
2354 // while we potentially have a match
2355 while (info == mInfo[idx]) {
2356 if (WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) {
2357 // key already exists, do NOT insert.
2358 // see http://en.cppreference.com/w/cpp/container/unordered_map/insert
2359 return std::make_pair(idx, InsertionState::key_found);
2360 }
2361 next(&info, &idx);
2362 }
2363
2364 // unlikely that this evaluates to true
2365 if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) {
2366 if (!increase_size()) {
2367 return std::make_pair(size_t(0), InsertionState::overflow_error);
2368 }
2369 continue;
2370 }
2371
2372 // key not found, so we are now exactly where we want to insert it.
2373 auto const insertion_idx = idx;
2374 auto const insertion_info = info;
2375 if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) {
2376 mMaxNumElementsAllowed = 0;
2377 }
2378
2379 // find an empty spot
2380 while (0 != mInfo[idx]) {
2381 next(&info, &idx);
2382 }
2383
2384 if (idx != insertion_idx) {
2385 shiftUp(idx, insertion_idx);
2386 }
2387 // put at empty spot
2388 mInfo[insertion_idx] = static_cast<uint8_t>(insertion_info);
2389 ++mNumElements;
2390 return std::make_pair(insertion_idx, idx == insertion_idx
2391 ? InsertionState::new_node
2392 : InsertionState::overwrite_node);
2393 }
2394
2395 // enough attempts failed, so finally give up.
2396 return std::make_pair(size_t(0), InsertionState::overflow_error);
2397 }
2398
2399 bool try_increase_info() {
2400 ROBIN_HOOD_LOG("mInfoInc=" << mInfoInc << ", numElements=" << mNumElements
2401 << ", maxNumElementsAllowed="
2402 << calcMaxNumElementsAllowed(mMask + 1))
2403 if (mInfoInc <= 2) {
2404 // need to be > 2 so that shift works (otherwise undefined behavior!)
2405 return false;
2406 }
2407 // we got space left, try to make info smaller
2408 mInfoInc = static_cast<uint8_t>(mInfoInc >> 1U);
2409
2410 // remove one bit of the hash, leaving more space for the distance info.
2411 // This is extremely fast because we can operate on 8 bytes at once.
2412 ++mInfoHashShift;
2413 auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
2414
2415 for (size_t i = 0; i < numElementsWithBuffer; i += 8) {
2416 auto val = unaligned_load<uint64_t>(mInfo + i);
2417 val = (val >> 1U) & UINT64_C(0x7f7f7f7f7f7f7f7f);
2418 std::memcpy(mInfo + i, &val, sizeof(val));
2419 }
2420 // update sentinel, which might have been cleared out!
2421 mInfo[numElementsWithBuffer] = 1;
2422
2423 mMaxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1);
2424 return true;
2425 }
2426
2427 // True if resize was possible, false otherwise
2428 bool increase_size() {
2429 // nothing allocated yet? just allocate InitialNumElements
2430 if (0 == mMask) {
2431 initData(InitialNumElements);
2432 return true;
2433 }
2434
2435 auto const maxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1);
2436 if (mNumElements < maxNumElementsAllowed && try_increase_info()) {
2437 return true;
2438 }
2439
2440 ROBIN_HOOD_LOG("mNumElements=" << mNumElements << ", maxNumElementsAllowed="
2441 << maxNumElementsAllowed << ", load="
2442 << (static_cast<double>(mNumElements) * 100.0 /
2443 (static_cast<double>(mMask) + 1)))
2444
2445 if (mNumElements * 2 < calcMaxNumElementsAllowed(mMask + 1)) {
2446 // we have to resize, even though there would still be plenty of space left!
2447 // Try to rehash instead. Delete freed memory so we don't steadyily increase mem in case
2448 // we have to rehash a few times
2449 nextHashMultiplier();
2450 rehashPowerOfTwo(mMask + 1, true);
2451 } else {
2452 // we've reached the capacity of the map, so the hash seems to work nice. Keep using it.
2453 rehashPowerOfTwo((mMask + 1) * 2, false);
2454 }
2455 return true;
2456 }
2457
2458 void nextHashMultiplier() {
2459 // adding an *even* number, so that the multiplier will always stay odd. This is necessary
2460 // so that the hash stays a mixing function (and thus doesn't have any information loss).
2461 mHashMultiplier += UINT64_C(0xc4ceb9fe1a85ec54);
2462 }
2463
2464 void destroy() {
2465 if (0 == mMask) {
2466 // don't deallocate!
2467 return;
2468 }
2469
2470 Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}
2471 .nodesDoNotDeallocate(*this);
2472
2473 // This protection against not deleting mMask shouldn't be needed as it's sufficiently
2474 // protected with the 0==mMask check, but I have this anyways because g++ 7 otherwise
2475 // reports a compile error: attempt to free a non-heap object 'fm'
2476 // [-Werror=free-nonheap-object]
2477 if (mKeyVals != reinterpret_cast_no_cast_align_warning<Node*>(&mMask)) {
2478 ROBIN_HOOD_LOG("std::free")
2479 std::free(mKeyVals);
2480 }
2481 }
2482
2483 void init() noexcept {
2484 mKeyVals = reinterpret_cast_no_cast_align_warning<Node*>(&mMask);
2485 mInfo = reinterpret_cast<uint8_t*>(&mMask);
2486 mNumElements = 0;
2487 mMask = 0;
2488 mMaxNumElementsAllowed = 0;
2489 mInfoInc = InitialInfoInc;
2490 mInfoHashShift = InitialInfoHashShift;
2491 }
2492
2493 // members are sorted so no padding occurs
2494 uint64_t mHashMultiplier = UINT64_C(0xc4ceb9fe1a85ec53); // 8 byte 8
2495 Node* mKeyVals = reinterpret_cast_no_cast_align_warning<Node*>(&mMask); // 8 byte 16
2496 uint8_t* mInfo = reinterpret_cast<uint8_t*>(&mMask); // 8 byte 24
2497 size_t mNumElements = 0; // 8 byte 32
2498 size_t mMask = 0; // 8 byte 40
2499 size_t mMaxNumElementsAllowed = 0; // 8 byte 48
2500 InfoType mInfoInc = InitialInfoInc; // 4 byte 52
2501 InfoType mInfoHashShift = InitialInfoHashShift; // 4 byte 56
2502 // 16 byte 56 if NodeAllocator
2503};
2504
2505} // namespace detail
2506
2507// map
2508
2509template <typename Key, typename T, typename Hash = hash<Key>,
2510 typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
2512
2513template <typename Key, typename T, typename Hash = hash<Key>,
2514 typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
2516
2517template <typename Key, typename T, typename Hash = hash<Key>,
2518 typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
2520 detail::Table<sizeof(robin_hood::pair<Key, T>) <= sizeof(size_t) * 6 &&
2521 std::is_nothrow_move_constructible<robin_hood::pair<Key, T>>::value &&
2522 std::is_nothrow_move_assignable<robin_hood::pair<Key, T>>::value,
2523 MaxLoadFactor100, Key, T, Hash, KeyEqual>;
2524
2525// set
2526
2527template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
2528 size_t MaxLoadFactor100 = 80>
2530
2531template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
2532 size_t MaxLoadFactor100 = 80>
2534
2535template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
2536 size_t MaxLoadFactor100 = 80>
2537using unordered_set = detail::Table<sizeof(Key) <= sizeof(size_t) * 6 &&
2538 std::is_nothrow_move_constructible<Key>::value &&
2539 std::is_nothrow_move_assignable<Key>::value,
2540 MaxLoadFactor100, Key, void, Hash, KeyEqual>;
2541
2542} // namespace robin_hood
2543
2544#endif
static constexpr std::size_t size() noexcept
Definition robin_hood.h:245
T value_type
Definition robin_hood.h:243
Definition robin_hood.h:367
~BulkPoolAllocator() noexcept
Definition robin_hood.h:399
BulkPoolAllocator & operator=(BulkPoolAllocator &&o) noexcept
Definition robin_hood.h:383
void swap(BulkPoolAllocator< T, MinNumAllocs, MaxNumAllocs > &other) noexcept
Definition robin_hood.h:451
void reset() noexcept
Definition robin_hood.h:404
void deallocate(T *obj) noexcept
Definition robin_hood.h:431
BulkPoolAllocator & operator=(const BulkPoolAllocator &ROBIN_HOOD_UNUSED(o)) noexcept
Definition robin_hood.h:394
BulkPoolAllocator(BulkPoolAllocator &&o) noexcept
Definition robin_hood.h:376
T * allocate()
Definition robin_hood.h:417
void addOrFree(void *ptr, const size_t numBytes) noexcept
Definition robin_hood.h:439
Definition robin_hood.h:916
void rehash(size_t c)
Definition robin_hood.h:2072
std::enable_if< Self_::is_transparent, const_iterator >::type find(const OtherKey &key) const
Definition robin_hood.h:1965
iterator find(const key_type &key)
Definition robin_hood.h:1971
iterator erase(const_iterator pos)
Definition robin_hood.h:2025
const_iterator cend() const
Definition robin_hood.h:2020
static constexpr bool is_map
Definition robin_hood.h:919
Hash hasher
Definition robin_hood.h:930
std::pair< iterator, bool > try_emplace(const key_type &key, Args &&... args)
Definition robin_hood.h:1830
Iter< true > const_iterator
Definition robin_hood.h:1495
std::pair< iterator, bool > try_emplace(key_type &&key, Args &&... args)
Definition robin_hood.h:1835
Table(size_t ROBIN_HOOD_UNUSED(bucket_count), const Hash &h=Hash{}, const KeyEqual &equal=KeyEqual{}) noexcept(noexcept(Hash(h)) &&noexcept(KeyEqual(equal)))
Definition robin_hood.h:1508
void swap(Table &o)
Definition robin_hood.h:1675
const_iterator end() const
Definition robin_hood.h:2016
Table(Iter first, Iter last, size_t ROBIN_HOOD_UNUSED(bucket_count)=0, const Hash &h=Hash{}, const KeyEqual &equal=KeyEqual{})
Definition robin_hood.h:1517
Table & operator=(Table const &o)
Definition robin_hood.h:1612
void compact()
Definition robin_hood.h:2086
ROBIN_HOOD(NODISCARD) size_t calcMaxNumElementsAllowed(size_t maxElements) const noexcept
Definition robin_hood.h:2136
iterator insert(const_iterator hint, value_type &&keyval)
Definition robin_hood.h:1887
std::pair< iterator, bool > insert(value_type &&keyval)
Definition robin_hood.h:1883
Iter< false > iterator
Definition robin_hood.h:1494
bool operator==(const Table &other) const
Definition robin_hood.h:1709
std::pair< iterator, bool > insert_or_assign(key_type &&key, Mapped &&obj)
Definition robin_hood.h:1857
std::pair< iterator, bool > insert_or_assign(const key_type &key, Mapped &&obj)
Definition robin_hood.h:1852
ROBIN_HOOD(NODISCARD) bool empty() const noexcept
Definition robin_hood.h:2115
void clear()
Definition robin_hood.h:1682
ROBIN_HOOD(NODISCARD) size_t calcNumBytesInfo(size_t numElements) const noexcept
Definition robin_hood.h:2145
void reserve(size_t c)
Definition robin_hood.h:2079
bool operator!=(const Table &other) const
Definition robin_hood.h:1723
Key key_type
Definition robin_hood.h:924
const_iterator find(const key_type &key) const
Definition robin_hood.h:1949
iterator end()
Definition robin_hood.h:2010
bool contains(const key_type &key) const
Definition robin_hood.h:1913
iterator find(const OtherKey &key, is_transparent_tag)
Definition robin_hood.h:1978
size_t size_type
Definition robin_hood.h:929
size_t erase(const key_type &key)
Definition robin_hood.h:2050
std::enable_if<!std::is_void< Q >::value, Q & >::type operator[](const key_type &key)
Definition robin_hood.h:1729
const_iterator begin() const
Definition robin_hood.h:1998
size_t count(const key_type &key) const
Definition robin_hood.h:1893
iterator try_emplace(const_iterator hint, const key_type &key, Args &&... args)
Definition robin_hood.h:1840
const_iterator cbegin() const
Definition robin_hood.h:2002
std::enable_if<!std::is_void< Q >::value, Qconst & >::type at(key_type const &key) const
Definition robin_hood.h:1940
~Table()
Definition robin_hood.h:1703
size_type size() const noexcept
Definition robin_hood.h:2105
size_t calcNumElementsWithBuffer(size_t numElements) const noexcept
Definition robin_hood.h:2152
KeyEqual key_equal
Definition robin_hood.h:931
void insert(std::initializer_list< value_type > ilist)
Definition robin_hood.h:1789
iterator insert_or_assign(const_iterator hint, const key_type &key, Mapped &&obj)
Definition robin_hood.h:1862
iterator emplace_hint(const_iterator position, Args &&... args)
Definition robin_hood.h:1824
typename std::conditional< is_set, Key, robin_hood::pair< typename std::conditional< is_flat, Key, Key const >::type, T > >::type value_type
Definition robin_hood.h:928
float load_factor() const noexcept
Definition robin_hood.h:2126
std::enable_if< Self_::is_transparent, iterator >::type find(const OtherKey &key)
Definition robin_hood.h:1985
float max_load_factor() const noexcept
Definition robin_hood.h:2120
ROBIN_HOOD(NODISCARD) size_t calcNumBytesTotal(size_t numElements) const
Definition robin_hood.h:2158
ROBIN_HOOD(NODISCARD) size_t mask() const noexcept
Definition robin_hood.h:2131
std::pair< iterator, bool > insert(const value_type &keyval)
Definition robin_hood.h:1873
size_type max_size() const noexcept
Definition robin_hood.h:2110
iterator insert(const_iterator hint, const value_type &keyval)
Definition robin_hood.h:1878
static constexpr bool is_flat
Definition robin_hood.h:918
std::pair< iterator, bool > emplace(Args &&... args)
Definition robin_hood.h:1796
T mapped_type
Definition robin_hood.h:925
Table() noexcept(noexcept(Hash()) &&noexcept(KeyEqual()))
Definition robin_hood.h:1497
iterator begin()
Definition robin_hood.h:1991
Table & operator=(Table &&o) noexcept
Definition robin_hood.h:1553
const_iterator find(const OtherKey &key, is_transparent_tag) const
Definition robin_hood.h:1956
std::enable_if< Self_::is_transparent, bool >::type contains(const OtherKey &key) const
Definition robin_hood.h:1919
iterator try_emplace(const_iterator hint, key_type &&key, Args &&... args)
Definition robin_hood.h:1846
std::enable_if< Self_::is_transparent, size_t >::type count(const OtherKey &key) const
Definition robin_hood.h:1904
Table(const Table &o)
Definition robin_hood.h:1581
iterator insert_or_assign(const_iterator hint, key_type &&key, Mapped &&obj)
Definition robin_hood.h:1868
static constexpr bool is_transparent
Definition robin_hood.h:921
static constexpr bool is_set
Definition robin_hood.h:920
iterator erase(iterator pos)
Definition robin_hood.h:2033
std::enable_if<!std::is_void< Q >::value, Q & >::type operator[](key_type &&key)
Definition robin_hood.h:1755
void insert(Iter first, Iter last)
Definition robin_hood.h:1782
std::enable_if<!std::is_void< Q >::value, Q & >::type at(key_type const &key)
Definition robin_hood.h:1927
typename detail_::IntSeqImpl< T, 0, N,(N - 0)==1 >::TResult make_integer_sequence
Definition robin_hood.h:292
make_index_sequence< sizeof...(T)> index_sequence_for
Definition robin_hood.h:298
make_integer_sequence< std::size_t, N > make_index_sequence
Definition robin_hood.h:295
T reinterpret_cast_no_cast_align_warning(void *ptr) noexcept
Definition robin_hood.h:322
T unaligned_load(void const *ptr) noexcept
Definition robin_hood.h:355
T * assertNotNull(T *t, Args &&... args)
Definition robin_hood.h:347
uint32_t SizeT
Definition robin_hood.h:310
T rotr(T x, unsigned k)
Definition robin_hood.h:314
Definition robin_hood.h:228
constexpr bool operator==(pair< A, B > const &x, pair< A, B > const &y)
Definition robin_hood.h:666
constexpr bool operator<=(pair< A, B > const &x, pair< A, B > const &y)
Definition robin_hood.h:684
constexpr bool operator>(pair< A, B > const &x, pair< A, B > const &y)
Definition robin_hood.h:680
constexpr bool operator!=(pair< A, B > const &x, pair< A, B > const &y)
Definition robin_hood.h:670
size_t hash_int(uint64_t x) noexcept
Definition robin_hood.h:748
void swap(pair< A, B > &a, pair< A, B > &b) noexcept(noexcept(std::declval< pair< A, B > & >().swap(std::declval< pair< A, B > & >())))
Definition robin_hood.h:660
size_t hash_bytes(void const *ptr, size_t len) noexcept
Definition robin_hood.h:692
constexpr bool operator<(pair< A, B > const &x, pair< A, B > const &y) noexcept(noexcept(std::declval< A const & >()< std::declval< A const & >()) &&noexcept(std::declval< B const & >()< std::declval< B const & >()))
Definition robin_hood.h:674
constexpr bool operator>=(pair< A, B > const &x, pair< A, B > const &y)
Definition robin_hood.h:688
#define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x)
Definition robin_hood.h:160
#define ROBIN_HOOD_COUNT_LEADING_ZEROES(x)
Definition robin_hood.h:159
#define ROBIN_HOOD_UNUSED(identifier)
Definition robin_hood.h:100
#define ROBIN_HOOD_COUNT(x)
Definition robin_hood.h:92
#define ROBIN_HOOD_TRACE(x)
Definition robin_hood.h:70
#define ROBIN_HOOD_HASH_INT(T)
Definition robin_hood.h:818
#define ROBIN_HOOD(x)
Definition robin_hood.h:97
#define ROBIN_HOOD_UNLIKELY(condition)
Definition robin_hood.h:182
#define ROBIN_HOOD_LIKELY(condition)
Definition robin_hood.h:181
#define ROBIN_HOOD_LOG(x)
Definition robin_hood.h:61
Definition robin_hood.h:238
typename IntSeqCombiner< typename IntSeqImpl< TValue, Begin, Begin+(End - Begin)/2,(End - Begin)/2==1 >::TResult, typename IntSeqImpl< TValue, Begin+(End - Begin)/2, End,(End - Begin+1)/2==1 >::TResult >::TResult TResult
Definition robin_hood.h:271
void addOrFree(void *ptr, size_t ROBIN_HOOD_UNUSED(numBytes)) noexcept
Definition robin_hood.h:548
Definition robin_hood.h:541
Definition robin_hood.h:868
WrapHash(T const &o) noexcept(noexcept(T(std::declval< T const & >())))
Definition robin_hood.h:870
Definition robin_hood.h:875
WrapKeyEqual(T const &o) noexcept(noexcept(T(std::declval< T const & >())))
Definition robin_hood.h:877
Definition robin_hood.h:859
Definition robin_hood.h:563
static const bool value
Definition robin_hood.h:564
Definition robin_hood.h:854
void type
Definition robin_hood.h:855
size_t operator()(Enum e) const noexcept
Definition robin_hood.h:812
size_t operator()(T *ptr) const noexcept
Definition robin_hood.h:791
size_t operator()(std::basic_string< CharT > const &str) const noexcept
Definition robin_hood.h:775
size_t operator()(std::shared_ptr< T > const &ptr) const noexcept
Definition robin_hood.h:805
size_t operator()(std::unique_ptr< T > const &ptr) const noexcept
Definition robin_hood.h:798
Definition robin_hood.h:763
size_t operator()(T const &obj) const noexcept(noexcept(std::declval< std::hash< T > >().operator()(std::declval< T const & >())))
Definition robin_hood.h:764
Definition robin_hood.h:576
Definition robin_hood.h:582
constexpr pair(std::piecewise_construct_t, std::tuple< U1... > a, std::tuple< U2... > b) noexcept(noexcept(pair(std::declval< std::tuple< U1... > & >(), std::declval< std::tuple< U2... > & >(), ROBIN_HOOD_STD::index_sequence_for< U1... >(), ROBIN_HOOD_STD::index_sequence_for< U2... >())))
Definition robin_hood.h:622
void swap(pair< T1, T2 > &o) noexcept((detail::swappable::nothrow< T1 >::value) &&(detail::swappable::nothrow< T2 >::value))
Definition robin_hood.h:648
constexpr pair(std::pair< T1, T2 > &&o) noexcept(noexcept(T1(std::move(std::declval< T1 && >()))) &&noexcept(T2(std::move(std::declval< T2 && >()))))
Definition robin_hood.h:600
pair(std::tuple< U1... > &a, std::tuple< U2... > &b, ROBIN_HOOD_STD::index_sequence< I1... >, ROBIN_HOOD_STD::index_sequence< I2... >) noexcept(noexcept(T1(std::forward< U1 >(std::get< I1 >(std::declval< std::tuple< U1... > & >()))...)) &&noexcept(T2(std::forward< U2 >(std::get< I2 >(std::declval< std::tuple< U2... > & >()))...)))
Definition robin_hood.h:634
T2 second
Definition robin_hood.h:656
constexpr pair(T1 &&a, T2 &&b) noexcept(noexcept(T1(std::move(std::declval< T1 && >()))) &&noexcept(T2(std::move(std::declval< T2 && >()))))
Definition robin_hood.h:605
T1 first
Definition robin_hood.h:655
constexpr pair(U1 &&a, U2 &&b) noexcept(noexcept(T1(std::forward< U1 >(std::declval< U1 && >()))) &&noexcept(T2(std::forward< U2 >(std::declval< U2 && >()))))
Definition robin_hood.h:611
T2 second_type
Definition robin_hood.h:584
T1 first_type
Definition robin_hood.h:583
constexpr pair(std::pair< T1, T2 > const &o) noexcept(noexcept(T1(std::declval< T1 const & >())) &&noexcept(T2(std::declval< T2 const & >())))
Definition robin_hood.h:594
constexpr pair() noexcept(noexcept(U1()) &&noexcept(U2()))
Definition robin_hood.h:589