uvgVPCCenc 1.1.0
uvgVPCCenc is an open-source real-time V-PCC encoder library written in C++ from scratch.
Loading...
Searching...
No Matches
robin_hood.h
Go to the documentation of this file.
1// ______ _____ ______ _________
2// ______________ ___ /_ ___(_)_______ ___ /_ ______ ______ ______ /
3// __ ___/_ __ \__ __ \__ / __ __ \ __ __ \_ __ \_ __ \_ __ /
4// _ / / /_/ /_ /_/ /_ / _ / / / _ / / // /_/ // /_/ // /_/ /
5// /_/ \____/ /_.___/ /_/ /_/ /_/ ________/_/ /_/ \____/ \____/ \__,_/
6// _/_____/
7//
8// Fast & memory efficient hashtable based on robin hood hashing for C++11/14/17/20
9// https://github.com/martinus/robin-hood-hashing
10//
11// Licensed under the MIT License <http://opensource.org/licenses/MIT>.
12// SPDX-License-Identifier: MIT
13// Copyright (c) 2018-2021 Martin Ankerl <http://martin.ankerl.com>
14//
15// Permission is hereby granted, free of charge, to any person obtaining a copy
16// of this software and associated documentation files (the "Software"), to deal
17// in the Software without restriction, including without limitation the rights
18// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
19// copies of the Software, and to permit persons to whom the Software is
20// furnished to do so, subject to the following conditions:
21//
22// The above copyright notice and this permission notice shall be included in all
23// copies or substantial portions of the Software.
24//
25// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
26// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
28// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
30// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31// SOFTWARE.
32
33#ifndef ROBIN_HOOD_H_INCLUDED
34#define ROBIN_HOOD_H_INCLUDED
35
36// see https://semver.org/
37#define ROBIN_HOOD_VERSION_MAJOR 3 // for incompatible API changes
38#define ROBIN_HOOD_VERSION_MINOR 11 // for adding functionality in a backwards-compatible manner
39#define ROBIN_HOOD_VERSION_PATCH 5 // for backwards-compatible bug fixes
40
41#include <algorithm>
42#include <cstdlib>
43#include <cstring>
44#include <functional>
45#include <limits>
46#include <memory> // only to support hash of smart pointers
47#include <stdexcept>
48#include <string>
49#include <type_traits>
50#include <utility>
51#if __cplusplus >= 201703L
52# include <string_view>
53#endif
54
55// #define ROBIN_HOOD_LOG_ENABLED
56#ifdef ROBIN_HOOD_LOG_ENABLED
57# include <iostream>
58# define ROBIN_HOOD_LOG(...) \
59 std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl;
60#else
61# define ROBIN_HOOD_LOG(x)
62#endif
63
64// #define ROBIN_HOOD_TRACE_ENABLED
65#ifdef ROBIN_HOOD_TRACE_ENABLED
66# include <iostream>
67# define ROBIN_HOOD_TRACE(...) \
68 std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl;
69#else
70# define ROBIN_HOOD_TRACE(x)
71#endif
72
73// #define ROBIN_HOOD_COUNT_ENABLED
74#ifdef ROBIN_HOOD_COUNT_ENABLED
75# include <iostream>
76# define ROBIN_HOOD_COUNT(x) ++counts().x;
77namespace robin_hood {
78struct Counts {
79 uint64_t shiftUp{};
80 uint64_t shiftDown{};
81};
82inline std::ostream& operator<<(std::ostream& os, Counts const& c) {
83 return os << c.shiftUp << " shiftUp" << std::endl << c.shiftDown << " shiftDown" << std::endl;
84}
85
86static Counts& counts() {
87 static Counts counts{};
88 return counts;
89}
90} // namespace robin_hood
91#else
92# define ROBIN_HOOD_COUNT(x)
93#endif
94
95// all non-argument macros should use this facility. See
96// https://www.fluentcpp.com/2019/05/28/better-macros-better-flags/
97#define ROBIN_HOOD(x) ROBIN_HOOD_PRIVATE_DEFINITION_##x()
98
99// mark unused members with this macro
100#define ROBIN_HOOD_UNUSED(identifier)
101
102// bitness
103#if SIZE_MAX == UINT32_MAX
104# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 32
105#elif SIZE_MAX == UINT64_MAX
106# define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 64
107#else
108# error Unsupported bitness
109#endif
110
111// endianess
112#ifdef _MSC_VER
113# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() 1
114# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() 0
115#else
116# define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() \
117 (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
118# define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
119#endif
120
121// inline
122#ifdef _MSC_VER
123# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __declspec(noinline)
124#else
125# define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __attribute__((noinline))
126#endif
127
128// exceptions
129#if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND)
130# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 0
131#else
132# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 1
133#endif
134
135// count leading/trailing bits
136#if !defined(ROBIN_HOOD_DISABLE_INTRINSICS)
137# ifdef _MSC_VER
138# if ROBIN_HOOD(BITNESS) == 32
139# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward
140# else
141# define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward64
142# endif
143# include <intrin.h>
144# pragma intrinsic(ROBIN_HOOD(BITSCANFORWARD))
145# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) \
146 [](size_t mask) noexcept -> int { \
147 unsigned long index; \
148 return ROBIN_HOOD(BITSCANFORWARD)(&index, mask) ? static_cast<int>(index) \
149 : ROBIN_HOOD(BITNESS); \
150 }(x)
151# else
152# if ROBIN_HOOD(BITNESS) == 32
153# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzl
154# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzl
155# else
156# define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzll
157# define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzll
158# endif
159# define ROBIN_HOOD_COUNT_LEADING_ZEROES(x) ((x) ? ROBIN_HOOD(CLZ)(x) : ROBIN_HOOD(BITNESS))
160# define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) ((x) ? ROBIN_HOOD(CTZ)(x) : ROBIN_HOOD(BITNESS))
161# endif
162#endif
163
164// fallthrough
165#ifndef __has_cpp_attribute // For backwards compatibility
166# define __has_cpp_attribute(x) 0
167#endif
168#if __has_cpp_attribute(clang::fallthrough)
169# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[clang::fallthrough]]
170#elif __has_cpp_attribute(gnu::fallthrough)
171# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[gnu::fallthrough]]
172#else
173# define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH()
174#endif
175
176// likely/unlikely
177#ifdef _MSC_VER
178# define ROBIN_HOOD_LIKELY(condition) condition
179# define ROBIN_HOOD_UNLIKELY(condition) condition
180#else
181# define ROBIN_HOOD_LIKELY(condition) __builtin_expect(condition, 1)
182# define ROBIN_HOOD_UNLIKELY(condition) __builtin_expect(condition, 0)
183#endif
184
185// detect if native wchar_t type is availiable in MSVC
186#ifdef _MSC_VER
187# ifdef _NATIVE_WCHAR_T_DEFINED
188# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1
189# else
190# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 0
191# endif
192#else
193# define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1
194#endif
195
196// detect if MSVC supports the pair(std::piecewise_construct_t,...) consructor being constexpr
197#ifdef _MSC_VER
198# if _MSC_VER <= 1900
199# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 1
200# else
201# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0
202# endif
203#else
204# define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0
205#endif
206
207// workaround missing "is_trivially_copyable" in g++ < 5.0
208// See https://stackoverflow.com/a/31798726/48181
209#if defined(__GNUC__) && __GNUC__ < 5
210# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) __has_trivial_copy(__VA_ARGS__)
211#else
212# define ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value
213#endif
214
215#if defined(__clang__)
216#pragma clang diagnostic push
217#pragma clang diagnostic ignored "-Wdeprecated-builtins"
218#endif
219
220// helpers for C++ versions, see https://gcc.gnu.org/onlinedocs/cpp/Standard-Predefined-Macros.html
221#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX() __cplusplus
222#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX98() 199711L
223#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX11() 201103L
224#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX14() 201402L
225#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX17() 201703L
226
227#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17)
228# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() [[nodiscard]]
229#else
230# define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD()
231#endif
232
233namespace robin_hood {
234
235#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14)
236# define ROBIN_HOOD_STD std
237#else
238
239// c++11 compatibility layer
240namespace ROBIN_HOOD_STD {
241template <class T>
243 : std::integral_constant<std::size_t, alignof(typename std::remove_all_extents<T>::type)> {};
244
245template <class T, T... Ints>
247public:
248 using value_type = T;
249 static_assert(std::is_integral<value_type>::value, "not integral type");
250 static constexpr std::size_t size() noexcept {
251 return sizeof...(Ints);
252 }
253};
254template <std::size_t... Inds>
255using index_sequence = integer_sequence<std::size_t, Inds...>;
256
257namespace detail_ {
258template <class T, T Begin, T End, bool>
260 using TValue = T;
261 static_assert(std::is_integral<TValue>::value, "not integral type");
262 static_assert(Begin >= 0 && Begin < End, "unexpected argument (Begin<0 || Begin<=End)");
263
264 template <class, class>
266
267 template <TValue... Inds0, TValue... Inds1>
269 using TResult = integer_sequence<TValue, Inds0..., Inds1...>;
270 };
271
272 using TResult =
273 typename IntSeqCombiner<typename IntSeqImpl<TValue, Begin, Begin + (End - Begin) / 2,
274 (End - Begin) / 2 == 1>::TResult,
275 typename IntSeqImpl<TValue, Begin + (End - Begin) / 2, End,
276 (End - Begin + 1) / 2 == 1>::TResult>::TResult;
277};
278
279template <class T, T Begin>
280struct IntSeqImpl<T, Begin, Begin, false> {
281 using TValue = T;
282 static_assert(std::is_integral<TValue>::value, "not integral type");
283 static_assert(Begin >= 0, "unexpected argument (Begin<0)");
285};
286
287template <class T, T Begin, T End>
288struct IntSeqImpl<T, Begin, End, true> {
289 using TValue = T;
290 static_assert(std::is_integral<TValue>::value, "not integral type");
291 static_assert(Begin >= 0, "unexpected argument (Begin<0)");
293};
294} // namespace detail_
295
296template <class T, T N>
297using make_integer_sequence = typename detail_::IntSeqImpl<T, 0, N, (N - 0) == 1>::TResult;
298
299template <std::size_t N>
301
302template <class... T>
304
305} // namespace ROBIN_HOOD_STD
306
307#endif
308
309namespace detail {
310
311// make sure we static_cast to the correct type for hash_int
312#if ROBIN_HOOD(BITNESS) == 64
313using SizeT = uint64_t;
314#else
315using SizeT = uint32_t;
316#endif
317
318template <typename T>
319T rotr(T x, unsigned k) {
320 return (x >> k) | (x << (8U * sizeof(T) - k));
321}
322
323// This cast gets rid of warnings like "cast from 'uint8_t*' {aka 'unsigned char*'} to
324// 'uint64_t*' {aka 'long unsigned int*'} increases required alignment of target type". Use with
325// care!
326template <typename T>
327inline T reinterpret_cast_no_cast_align_warning(void* ptr) noexcept {
328 return reinterpret_cast<T>(ptr);
329}
330
331template <typename T>
332inline T reinterpret_cast_no_cast_align_warning(void const* ptr) noexcept {
333 return reinterpret_cast<T>(ptr);
334}
335
336// make sure this is not inlined as it is slow and dramatically enlarges code, thus making other
337// inlinings more difficult. Throws are also generally the slow path.
338template <typename E, typename... Args>
339[[noreturn]] ROBIN_HOOD(NOINLINE)
340#if ROBIN_HOOD(HAS_EXCEPTIONS)
341 void doThrow(Args&&... args) {
342 // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay)
343 throw E(std::forward<Args>(args)...);
344}
345#else
346 void doThrow(Args&&... ROBIN_HOOD_UNUSED(args) /*unused*/) {
347 abort();
348}
349#endif
350
351template <typename E, typename T, typename... Args>
352T* assertNotNull(T* t, Args&&... args) {
353 if (ROBIN_HOOD_UNLIKELY(nullptr == t)) {
354 doThrow<E>(std::forward<Args>(args)...);
355 }
356 return t;
357}
358
359template <typename T>
360inline T unaligned_load(void const* ptr) noexcept {
361 // using memcpy so we don't get into unaligned load problems.
362 // compiler should optimize this very well anyways.
363 T t;
364 std::memcpy(&t, ptr, sizeof(T));
365 return t;
366}
367
368// Allocates bulks of memory for objects of type T. This deallocates the memory in the destructor,
369// and keeps a linked list of the allocated memory around. Overhead per allocation is the size of a
370// pointer.
371template <typename T, size_t MinNumAllocs = 4, size_t MaxNumAllocs = 256>
373public:
374 BulkPoolAllocator() noexcept = default;
375
376 // does not copy anything, just creates a new allocator.
378 : mHead(nullptr)
379 , mListForFree(nullptr) {}
380
382 : mHead(o.mHead)
383 , mListForFree(o.mListForFree) {
384 o.mListForFree = nullptr;
385 o.mHead = nullptr;
386 }
387
389 reset();
390 mHead = o.mHead;
391 mListForFree = o.mListForFree;
392 o.mListForFree = nullptr;
393 o.mHead = nullptr;
394 return *this;
395 }
396
398 // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp)
399 operator=(const BulkPoolAllocator& ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept {
400 // does not do anything
401 return *this;
402 }
403
405 reset();
406 }
407
408 // Deallocates all allocated memory.
409 void reset() noexcept {
410 while (mListForFree) {
411 T* tmp = *mListForFree;
412 ROBIN_HOOD_LOG("std::free")
413 std::free(mListForFree);
414 mListForFree = reinterpret_cast_no_cast_align_warning<T**>(tmp);
415 }
416 mHead = nullptr;
417 }
418
419 // allocates, but does NOT initialize. Use in-place new constructor, e.g.
420 // T* obj = pool.allocate();
421 // ::new (static_cast<void*>(obj)) T();
422 T* allocate() {
423 T* tmp = mHead;
424 if (!tmp) {
425 tmp = performAllocation();
426 }
427
428 mHead = *reinterpret_cast_no_cast_align_warning<T**>(tmp);
429 return tmp;
430 }
431
432 // does not actually deallocate but puts it in store.
433 // make sure you have already called the destructor! e.g. with
434 // obj->~T();
435 // pool.deallocate(obj);
436 void deallocate(T* obj) noexcept {
437 *reinterpret_cast_no_cast_align_warning<T**>(obj) = mHead;
438 mHead = obj;
439 }
440
441 // Adds an already allocated block of memory to the allocator. This allocator is from now on
442 // responsible for freeing the data (with free()). If the provided data is not large enough to
443 // make use of, it is immediately freed. Otherwise it is reused and freed in the destructor.
444 void addOrFree(void* ptr, const size_t numBytes) noexcept {
445 // calculate number of available elements in ptr
446 if (numBytes < ALIGNMENT + ALIGNED_SIZE) {
447 // not enough data for at least one element. Free and return.
448 ROBIN_HOOD_LOG("std::free")
449 std::free(ptr);
450 } else {
451 ROBIN_HOOD_LOG("add to buffer")
452 add(ptr, numBytes);
453 }
454 }
455
457 using std::swap;
458 swap(mHead, other.mHead);
459 swap(mListForFree, other.mListForFree);
460 }
461
462private:
463 // iterates the list of allocated memory to calculate how many to alloc next.
464 // Recalculating this each time saves us a size_t member.
465 // This ignores the fact that memory blocks might have been added manually with addOrFree. In
466 // practice, this should not matter much.
467 ROBIN_HOOD(NODISCARD) size_t calcNumElementsToAlloc() const noexcept {
468 auto tmp = mListForFree;
469 size_t numAllocs = MinNumAllocs;
470
471 while (numAllocs * 2 <= MaxNumAllocs && tmp) {
472 auto x = reinterpret_cast<T***>(tmp);
473 tmp = *x;
474 numAllocs *= 2;
475 }
476
477 return numAllocs;
478 }
479
480 // WARNING: Underflow if numBytes < ALIGNMENT! This is guarded in addOrFree().
481 void add(void* ptr, const size_t numBytes) noexcept {
482 const size_t numElements = (numBytes - ALIGNMENT) / ALIGNED_SIZE;
483
484 auto data = reinterpret_cast<T**>(ptr);
485
486 // link free list
487 auto x = reinterpret_cast<T***>(data);
488 *x = mListForFree;
489 mListForFree = data;
490
491 // create linked list for newly allocated data
492 auto* const headT =
493 reinterpret_cast_no_cast_align_warning<T*>(reinterpret_cast<char*>(ptr) + ALIGNMENT);
494
495 auto* const head = reinterpret_cast<char*>(headT);
496
497 // Visual Studio compiler automatically unrolls this loop, which is pretty cool
498 for (size_t i = 0; i < numElements; ++i) {
499 *reinterpret_cast_no_cast_align_warning<char**>(head + i * ALIGNED_SIZE) =
500 head + (i + 1) * ALIGNED_SIZE;
501 }
502
503 // last one points to 0
504 *reinterpret_cast_no_cast_align_warning<T**>(head + (numElements - 1) * ALIGNED_SIZE) =
505 mHead;
506 mHead = headT;
507 }
508
509 // Called when no memory is available (mHead == 0).
510 // Don't inline this slow path.
511 ROBIN_HOOD(NOINLINE) T* performAllocation() {
512 size_t const numElementsToAlloc = calcNumElementsToAlloc();
513
514 // alloc new memory: [prev |T, T, ... T]
515 size_t const bytes = ALIGNMENT + ALIGNED_SIZE * numElementsToAlloc;
516 ROBIN_HOOD_LOG("std::malloc " << bytes << " = " << ALIGNMENT << " + " << ALIGNED_SIZE
517 << " * " << numElementsToAlloc)
518 add(assertNotNull<std::bad_alloc>(std::malloc(bytes)), bytes);
519 return mHead;
520 }
521
522 // enforce byte alignment of the T's
523#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14)
524 static constexpr size_t ALIGNMENT =
525 (std::max)(std::alignment_of<T>::value, std::alignment_of<T*>::value);
526#else
527 static const size_t ALIGNMENT =
528 (ROBIN_HOOD_STD::alignment_of<T>::value > ROBIN_HOOD_STD::alignment_of<T*>::value)
529 ? ROBIN_HOOD_STD::alignment_of<T>::value
530 : +ROBIN_HOOD_STD::alignment_of<T*>::value; // the + is for walkarround
531#endif
532
533 static constexpr size_t ALIGNED_SIZE = ((sizeof(T) - 1) / ALIGNMENT + 1) * ALIGNMENT;
534
535 static_assert(MinNumAllocs >= 1, "MinNumAllocs");
536 static_assert(MaxNumAllocs >= MinNumAllocs, "MaxNumAllocs");
537 static_assert(ALIGNED_SIZE >= sizeof(T*), "ALIGNED_SIZE");
538 static_assert(0 == (ALIGNED_SIZE % sizeof(T*)), "ALIGNED_SIZE mod");
539 static_assert(ALIGNMENT >= sizeof(T*), "ALIGNMENT");
540
541 T* mHead{nullptr};
542 T** mListForFree{nullptr};
543};
544
545template <typename T, size_t MinSize, size_t MaxSize, bool IsFlat>
547
548// dummy allocator that does nothing
549template <typename T, size_t MinSize, size_t MaxSize>
550struct NodeAllocator<T, MinSize, MaxSize, true> {
551
552 // we are not using the data, so just free it.
553 void addOrFree(void* ptr, size_t ROBIN_HOOD_UNUSED(numBytes) /*unused*/) noexcept {
554 ROBIN_HOOD_LOG("std::free")
555 std::free(ptr);
556 }
557};
558
559template <typename T, size_t MinSize, size_t MaxSize>
560struct NodeAllocator<T, MinSize, MaxSize, false> : public BulkPoolAllocator<T, MinSize, MaxSize> {};
561
562// c++14 doesn't have is_nothrow_swappable, and clang++ 6.0.1 doesn't like it either, so I'm making
563// my own here.
564namespace swappable {
565#if ROBIN_HOOD(CXX) < ROBIN_HOOD(CXX17)
566using std::swap;
567template <typename T>
568struct nothrow {
569 static const bool value = noexcept(swap(std::declval<T&>(), std::declval<T&>()));
570};
571#else
572template <typename T>
573struct nothrow {
574 static const bool value = std::is_nothrow_swappable<T>::value;
575};
576#endif
577} // namespace swappable
578
579} // namespace detail
580
582
583// A custom pair implementation is used in the map because std::pair is not is_trivially_copyable,
584// which means it would not be allowed to be used in std::memcpy. This struct is copyable, which is
585// also tested.
586template <typename T1, typename T2>
587struct pair {
588 using first_type = T1;
589 using second_type = T2;
590
591 template <typename U1 = T1, typename U2 = T2,
592 typename = typename std::enable_if<std::is_default_constructible<U1>::value &&
593 std::is_default_constructible<U2>::value>::type>
594 constexpr pair() noexcept(noexcept(U1()) && noexcept(U2()))
595 : first()
596 , second() {}
597
598 // pair constructors are explicit so we don't accidentally call this ctor when we don't have to.
599 explicit constexpr pair(std::pair<T1, T2> const& o) noexcept(
600 noexcept(T1(std::declval<T1 const&>())) && noexcept(T2(std::declval<T2 const&>())))
601 : first(o.first)
602 , second(o.second) {}
603
604 // pair constructors are explicit so we don't accidentally call this ctor when we don't have to.
605 explicit constexpr pair(std::pair<T1, T2>&& o) noexcept(noexcept(
606 T1(std::move(std::declval<T1&&>()))) && noexcept(T2(std::move(std::declval<T2&&>()))))
607 : first(std::move(o.first))
608 , second(std::move(o.second)) {}
609
610 constexpr pair(T1&& a, T2&& b) noexcept(noexcept(
611 T1(std::move(std::declval<T1&&>()))) && noexcept(T2(std::move(std::declval<T2&&>()))))
612 : first(std::move(a))
613 , second(std::move(b)) {}
614
615 template <typename U1, typename U2>
616 constexpr pair(U1&& a, U2&& b) noexcept(noexcept(T1(std::forward<U1>(
617 std::declval<U1&&>()))) && noexcept(T2(std::forward<U2>(std::declval<U2&&>()))))
618 : first(std::forward<U1>(a))
619 , second(std::forward<U2>(b)) {}
620
621 template <typename... U1, typename... U2>
622 // MSVC 2015 produces error "C2476: ‘constexpr’ constructor does not initialize all members"
623 // if this constructor is constexpr
624#if !ROBIN_HOOD(BROKEN_CONSTEXPR)
625 constexpr
626#endif
627 pair(std::piecewise_construct_t /*unused*/, std::tuple<U1...> a,
628 std::tuple<U2...>
629 b) noexcept(noexcept(pair(std::declval<std::tuple<U1...>&>(),
630 std::declval<std::tuple<U2...>&>(),
635 }
636
637 // constructor called from the std::piecewise_construct_t ctor
638 template <typename... U1, size_t... I1, typename... U2, size_t... I2>
639 pair(std::tuple<U1...>& a, std::tuple<U2...>& b, ROBIN_HOOD_STD::index_sequence<I1...> /*unused*/, ROBIN_HOOD_STD::index_sequence<I2...> /*unused*/) noexcept(
640 noexcept(T1(std::forward<U1>(std::get<I1>(
641 std::declval<std::tuple<
642 U1...>&>()))...)) && noexcept(T2(std::
643 forward<U2>(std::get<I2>(
644 std::declval<std::tuple<U2...>&>()))...)))
645 : first(std::forward<U1>(std::get<I1>(a))...)
646 , second(std::forward<U2>(std::get<I2>(b))...) {
647 // make visual studio compiler happy about warning about unused a & b.
648 // Visual studio's pair implementation disables warning 4100.
649 (void)a;
650 (void)b;
651 }
652
655 using std::swap;
656 swap(first, o.first);
657 swap(second, o.second);
658 }
659
660 T1 first; // NOLINT(misc-non-private-member-variables-in-classes)
661 T2 second; // NOLINT(misc-non-private-member-variables-in-classes)
662};
663
664template <typename A, typename B>
665inline void swap(pair<A, B>& a, pair<A, B>& b) noexcept(
666 noexcept(std::declval<pair<A, B>&>().swap(std::declval<pair<A, B>&>()))) {
667 a.swap(b);
668}
669
670template <typename A, typename B>
671inline constexpr bool operator==(pair<A, B> const& x, pair<A, B> const& y) {
672 return (x.first == y.first) && (x.second == y.second);
673}
674template <typename A, typename B>
675inline constexpr bool operator!=(pair<A, B> const& x, pair<A, B> const& y) {
676 return !(x == y);
677}
678template <typename A, typename B>
679inline constexpr bool operator<(pair<A, B> const& x, pair<A, B> const& y) noexcept(noexcept(
680 std::declval<A const&>() < std::declval<A const&>()) && noexcept(std::declval<B const&>() <
681 std::declval<B const&>())) {
682 return x.first < y.first || (!(y.first < x.first) && x.second < y.second);
683}
684template <typename A, typename B>
685inline constexpr bool operator>(pair<A, B> const& x, pair<A, B> const& y) {
686 return y < x;
687}
688template <typename A, typename B>
689inline constexpr bool operator<=(pair<A, B> const& x, pair<A, B> const& y) {
690 return !(x > y);
691}
692template <typename A, typename B>
693inline constexpr bool operator>=(pair<A, B> const& x, pair<A, B> const& y) {
694 return !(x < y);
695}
696
697inline size_t hash_bytes(void const* ptr, size_t len) noexcept {
698 static constexpr uint64_t m = UINT64_C(0xc6a4a7935bd1e995);
699 static constexpr uint64_t seed = UINT64_C(0xe17a1465);
700 static constexpr unsigned int r = 47;
701
702 auto const* const data64 = static_cast<uint64_t const*>(ptr);
703 uint64_t h = seed ^ (len * m);
704
705 size_t const n_blocks = len / 8;
706 for (size_t i = 0; i < n_blocks; ++i) {
707 auto k = detail::unaligned_load<uint64_t>(data64 + i);
708
709 k *= m;
710 k ^= k >> r;
711 k *= m;
712
713 h ^= k;
714 h *= m;
715 }
716
717 auto const* const data8 = reinterpret_cast<uint8_t const*>(data64 + n_blocks);
718 switch (len & 7U) {
719 case 7:
720 h ^= static_cast<uint64_t>(data8[6]) << 48U;
721 ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
722 case 6:
723 h ^= static_cast<uint64_t>(data8[5]) << 40U;
724 ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
725 case 5:
726 h ^= static_cast<uint64_t>(data8[4]) << 32U;
727 ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
728 case 4:
729 h ^= static_cast<uint64_t>(data8[3]) << 24U;
730 ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
731 case 3:
732 h ^= static_cast<uint64_t>(data8[2]) << 16U;
733 ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
734 case 2:
735 h ^= static_cast<uint64_t>(data8[1]) << 8U;
736 ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
737 case 1:
738 h ^= static_cast<uint64_t>(data8[0]);
739 h *= m;
740 ROBIN_HOOD(FALLTHROUGH); // FALLTHROUGH
741 default:
742 break;
743 }
744
745 h ^= h >> r;
746
747 // not doing the final step here, because this will be done by keyToIdx anyways
748 // h *= m;
749 // h ^= h >> r;
750 return static_cast<size_t>(h);
751}
752
753inline size_t hash_int(uint64_t x) noexcept {
754 // tried lots of different hashes, let's stick with murmurhash3. It's simple, fast, well tested,
755 // and doesn't need any special 128bit operations.
756 x ^= x >> 33U;
757 x *= UINT64_C(0xff51afd7ed558ccd);
758 x ^= x >> 33U;
759
760 // not doing the final step here, because this will be done by keyToIdx anyways
761 // x *= UINT64_C(0xc4ceb9fe1a85ec53);
762 // x ^= x >> 33U;
763 return static_cast<size_t>(x);
764}
765
766// A thin wrapper around std::hash, performing an additional simple mixing step of the result.
767template <typename T, typename Enable = void>
768struct hash : public std::hash<T> {
769 size_t operator()(T const& obj) const
770 noexcept(noexcept(std::declval<std::hash<T>>().operator()(std::declval<T const&>()))) {
771 // call base hash
772 auto result = std::hash<T>::operator()(obj);
773 // return mixed of that, to be save against identity has
774 return hash_int(static_cast<detail::SizeT>(result));
775 }
776};
777
778template <typename CharT>
779struct hash<std::basic_string<CharT>> {
780 size_t operator()(std::basic_string<CharT> const& str) const noexcept {
781 return hash_bytes(str.data(), sizeof(CharT) * str.size());
782 }
783};
784
785#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17)
786template <typename CharT>
787struct hash<std::basic_string_view<CharT>> {
788 size_t operator()(std::basic_string_view<CharT> const& sv) const noexcept {
789 return hash_bytes(sv.data(), sizeof(CharT) * sv.size());
790 }
791};
792#endif
793
794template <class T>
795struct hash<T*> {
796 size_t operator()(T* ptr) const noexcept {
797 return hash_int(reinterpret_cast<detail::SizeT>(ptr));
798 }
799};
800
801template <class T>
802struct hash<std::unique_ptr<T>> {
803 size_t operator()(std::unique_ptr<T> const& ptr) const noexcept {
804 return hash_int(reinterpret_cast<detail::SizeT>(ptr.get()));
805 }
806};
807
808template <class T>
809struct hash<std::shared_ptr<T>> {
810 size_t operator()(std::shared_ptr<T> const& ptr) const noexcept {
811 return hash_int(reinterpret_cast<detail::SizeT>(ptr.get()));
812 }
813};
814
815template <typename Enum>
816struct hash<Enum, typename std::enable_if<std::is_enum<Enum>::value>::type> {
817 size_t operator()(Enum e) const noexcept {
818 using Underlying = typename std::underlying_type<Enum>::type;
819 return hash<Underlying>{}(static_cast<Underlying>(e));
820 }
821};
822
823#define ROBIN_HOOD_HASH_INT(T) \
824 template <> \
825 struct hash<T> { \
826 size_t operator()(T const& obj) const noexcept { \
827 return hash_int(static_cast<uint64_t>(obj)); \
828 } \
829 }
830
831#if defined(__GNUC__) && !defined(__clang__)
832# pragma GCC diagnostic push
833# pragma GCC diagnostic ignored "-Wuseless-cast"
834#endif
835// see https://en.cppreference.com/w/cpp/utility/hash
839ROBIN_HOOD_HASH_INT(unsigned char);
842#if ROBIN_HOOD(HAS_NATIVE_WCHART)
844#endif
846ROBIN_HOOD_HASH_INT(unsigned short);
851ROBIN_HOOD_HASH_INT(unsigned long);
852ROBIN_HOOD_HASH_INT(unsigned long long);
853#if defined(__GNUC__) && !defined(__clang__)
854# pragma GCC diagnostic pop
855#endif
856namespace detail {
857
858template <typename T>
859struct void_type {
860 using type = void;
861};
862
863template <typename T, typename = void>
864struct has_is_transparent : public std::false_type {};
865
866template <typename T>
867struct has_is_transparent<T, typename void_type<typename T::is_transparent>::type>
868 : public std::true_type {};
869
870// using wrapper classes for hash and key_equal prevents the diamond problem when the same type
871// is used. see https://stackoverflow.com/a/28771920/48181
872template <typename T>
873struct WrapHash : public T {
874 WrapHash() = default;
875 explicit WrapHash(T const& o) noexcept(noexcept(T(std::declval<T const&>())))
876 : T(o) {}
877};
878
879template <typename T>
880struct WrapKeyEqual : public T {
881 WrapKeyEqual() = default;
882 explicit WrapKeyEqual(T const& o) noexcept(noexcept(T(std::declval<T const&>())))
883 : T(o) {}
884};
885
886// A highly optimized hashmap implementation, using the Robin Hood algorithm.
887//
888// In most cases, this map should be usable as a drop-in replacement for std::unordered_map, but
889// be about 2x faster in most cases and require much less allocations.
890//
891// This implementation uses the following memory layout:
892//
893// [Node, Node, ... Node | info, info, ... infoSentinel ]
894//
895// * Node: either a DataNode that directly has the std::pair<key, val> as member,
896// or a DataNode with a pointer to std::pair<key,val>. Which DataNode representation to use
897// depends on how fast the swap() operation is. Heuristically, this is automatically choosen
898// based on sizeof(). there are always 2^n Nodes.
899//
900// * info: Each Node in the map has a corresponding info byte, so there are 2^n info bytes.
901// Each byte is initialized to 0, meaning the corresponding Node is empty. Set to 1 means the
902// corresponding node contains data. Set to 2 means the corresponding Node is filled, but it
903// actually belongs to the previous position and was pushed out because that place is already
904// taken.
905//
906// * infoSentinel: Sentinel byte set to 1, so that iterator's ++ can stop at end() without the
907// need for a idx variable.
908//
909// According to STL, order of templates has effect on throughput. That's why I've moved the
910// boolean to the front.
911// https://www.reddit.com/r/cpp/comments/ahp6iu/compile_time_binary_size_reductions_and_cs_future/eeguck4/
912template <bool IsFlat, size_t MaxLoadFactor100, typename Key, typename T, typename Hash,
913 typename KeyEqual>
914class Table
915 : public WrapHash<Hash>,
916 public WrapKeyEqual<KeyEqual>,
918 typename std::conditional<
919 std::is_void<T>::value, Key,
920 robin_hood::pair<typename std::conditional<IsFlat, Key, Key const>::type, T>>::type,
921 4, 16384, IsFlat> {
922public:
923 static constexpr bool is_flat = IsFlat;
924 static constexpr bool is_map = !std::is_void<T>::value;
925 static constexpr bool is_set = !is_map;
926 static constexpr bool is_transparent =
928
929 using key_type = Key;
930 using mapped_type = T;
931 using value_type = typename std::conditional<
932 is_set, Key,
934 using size_type = size_t;
935 using hasher = Hash;
936 using key_equal = KeyEqual;
938
939private:
940 static_assert(MaxLoadFactor100 > 10 && MaxLoadFactor100 < 100,
941 "MaxLoadFactor100 needs to be >10 && < 100");
942
943 using WHash = WrapHash<Hash>;
945
946 // configuration defaults
947
948 // make sure we have 8 elements, needed to quickly rehash mInfo
949 static constexpr size_t InitialNumElements = sizeof(uint64_t);
950 static constexpr uint32_t InitialInfoNumBits = 5;
951 static constexpr uint8_t InitialInfoInc = 1U << InitialInfoNumBits;
952 static constexpr size_t InfoMask = InitialInfoInc - 1U;
953 static constexpr uint8_t InitialInfoHashShift = 0;
955
956 // type needs to be wider than uint8_t.
957 using InfoType = uint32_t;
958
959 // DataNode ////////////////////////////////////////////////////////
960
961 // Primary template for the data node. We have special implementations for small and big
962 // objects. For large objects it is assumed that swap() is fairly slow, so we allocate these
963 // on the heap so swap merely swaps a pointer.
964 template <typename M, bool>
965 class DataNode {};
966
967 // Small: just allocate on the stack.
968 template <typename M>
969 class DataNode<M, true> final {
970 public:
971 template <typename... Args>
972 explicit DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, Args&&... args) noexcept(
973 noexcept(value_type(std::forward<Args>(args)...)))
974 : mData(std::forward<Args>(args)...) {}
975
976 DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode<M, true>&& n) noexcept(
977 std::is_nothrow_move_constructible<value_type>::value)
978 : mData(std::move(n.mData)) {}
979
980 // doesn't do anything
981 void destroy(M& ROBIN_HOOD_UNUSED(map) /*unused*/) noexcept {}
982 void destroyDoNotDeallocate() noexcept {}
983
984 value_type const* operator->() const noexcept {
985 return &mData;
986 }
987 value_type* operator->() noexcept {
988 return &mData;
989 }
990
991 const value_type& operator*() const noexcept {
992 return mData;
993 }
994
995 value_type& operator*() noexcept {
996 return mData;
997 }
998
999 template <typename VT = value_type>
1000 ROBIN_HOOD(NODISCARD)
1001 typename std::enable_if<is_map, typename VT::first_type&>::type getFirst() noexcept {
1002 return mData.first;
1003 }
1004 template <typename VT = value_type>
1005 ROBIN_HOOD(NODISCARD)
1006 typename std::enable_if<is_set, VT&>::type getFirst() noexcept {
1007 return mData;
1008 }
1009
1010 template <typename VT = value_type>
1011 ROBIN_HOOD(NODISCARD)
1012 typename std::enable_if<is_map, typename VT::first_type const&>::type
1013 getFirst() const noexcept {
1014 return mData.first;
1015 }
1016 template <typename VT = value_type>
1017 ROBIN_HOOD(NODISCARD)
1018 typename std::enable_if<is_set, VT const&>::type getFirst() const noexcept {
1019 return mData;
1020 }
1021
1022 template <typename MT = mapped_type>
1023 ROBIN_HOOD(NODISCARD)
1024 typename std::enable_if<is_map, MT&>::type getSecond() noexcept {
1025 return mData.second;
1026 }
1027
1028 template <typename MT = mapped_type>
1029 ROBIN_HOOD(NODISCARD)
1030 typename std::enable_if<is_set, MT const&>::type getSecond() const noexcept {
1031 return mData.second;
1032 }
1033
1034 void swap(DataNode<M, true>& o) noexcept(
1035 noexcept(std::declval<value_type>().swap(std::declval<value_type>()))) {
1036 mData.swap(o.mData);
1037 }
1038
1039 private:
1040 value_type mData;
1041 };
1042
1043 // big object: allocate on heap.
1044 template <typename M>
1045 class DataNode<M, false> {
1046 public:
1047 template <typename... Args>
1048 explicit DataNode(M& map, Args&&... args)
1049 : mData(map.allocate()) {
1050 ::new (static_cast<void*>(mData)) value_type(std::forward<Args>(args)...);
1051 }
1052
1053 DataNode(M& ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode<M, false>&& n) noexcept
1054 : mData(std::move(n.mData)) {}
1055
1056 void destroy(M& map) noexcept {
1057 // don't deallocate, just put it into list of datapool.
1058 mData->~value_type();
1059 map.deallocate(mData);
1060 }
1061
1062 void destroyDoNotDeallocate() noexcept {
1063 mData->~value_type();
1064 }
1065
1066 value_type const* operator->() const noexcept {
1067 return mData;
1068 }
1069
1070 value_type* operator->() noexcept {
1071 return mData;
1072 }
1073
1074 const value_type& operator*() const {
1075 return *mData;
1076 }
1077
1078 value_type& operator*() {
1079 return *mData;
1080 }
1081
1082 template <typename VT = value_type>
1083 ROBIN_HOOD(NODISCARD)
1084 typename std::enable_if<is_map, typename VT::first_type&>::type getFirst() noexcept {
1085 return mData->first;
1086 }
1087 template <typename VT = value_type>
1088 ROBIN_HOOD(NODISCARD)
1089 typename std::enable_if<is_set, VT&>::type getFirst() noexcept {
1090 return *mData;
1091 }
1092
1093 template <typename VT = value_type>
1094 ROBIN_HOOD(NODISCARD)
1095 typename std::enable_if<is_map, typename VT::first_type const&>::type
1096 getFirst() const noexcept {
1097 return mData->first;
1098 }
1099 template <typename VT = value_type>
1100 ROBIN_HOOD(NODISCARD)
1101 typename std::enable_if<is_set, VT const&>::type getFirst() const noexcept {
1102 return *mData;
1103 }
1104
1105 template <typename MT = mapped_type>
1106 ROBIN_HOOD(NODISCARD)
1107 typename std::enable_if<is_map, MT&>::type getSecond() noexcept {
1108 return mData->second;
1109 }
1110
1111 template <typename MT = mapped_type>
1112 ROBIN_HOOD(NODISCARD)
1113 typename std::enable_if<is_map, MT const&>::type getSecond() const noexcept {
1114 return mData->second;
1115 }
1116
1117 void swap(DataNode<M, false>& o) noexcept {
1118 using std::swap;
1119 swap(mData, o.mData);
1120 }
1121
1122 private:
1123 value_type* mData;
1124 };
1125
1126 using Node = DataNode<Self, IsFlat>;
1127
1128 // helpers for insertKeyPrepareEmptySpot: extract first entry (only const required)
1129 ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(Node const& n) const noexcept {
1130 return n.getFirst();
1131 }
1132
1133 // in case we have void mapped_type, we are not using a pair, thus we just route k through.
1134 // No need to disable this because it's just not used if not applicable.
1135 ROBIN_HOOD(NODISCARD) key_type const& getFirstConst(key_type const& k) const noexcept {
1136 return k;
1137 }
1138
1139 // in case we have non-void mapped_type, we have a standard robin_hood::pair
1140 template <typename Q = mapped_type>
1141 ROBIN_HOOD(NODISCARD)
1142 typename std::enable_if<!std::is_void<Q>::value, key_type const&>::type
1143 getFirstConst(value_type const& vt) const noexcept {
1144 return vt.first;
1145 }
1146
1147 // Cloner //////////////////////////////////////////////////////////
1148
1149 template <typename M, bool UseMemcpy>
1150 struct Cloner;
1151
1152 // fast path: Just copy data, without allocating anything.
1153 template <typename M>
1154 struct Cloner<M, true> {
1155 void operator()(M const& source, M& target) const {
1156 auto const* const src = reinterpret_cast<char const*>(source.mKeyVals);
1157 auto* tgt = reinterpret_cast<char*>(target.mKeyVals);
1158 auto const numElementsWithBuffer = target.calcNumElementsWithBuffer(target.mMask + 1);
1159 std::copy(src, src + target.calcNumBytesTotal(numElementsWithBuffer), tgt);
1160 }
1161 };
1162
1163 template <typename M>
1164 struct Cloner<M, false> {
1165 void operator()(M const& s, M& t) const {
1166 auto const numElementsWithBuffer = t.calcNumElementsWithBuffer(t.mMask + 1);
1167 std::copy(s.mInfo, s.mInfo + t.calcNumBytesInfo(numElementsWithBuffer), t.mInfo);
1168
1169 for (size_t i = 0; i < numElementsWithBuffer; ++i) {
1170 if (t.mInfo[i]) {
1171 ::new (static_cast<void*>(t.mKeyVals + i)) Node(t, *s.mKeyVals[i]);
1172 }
1173 }
1174 }
1175 };
1176
1177 // Destroyer ///////////////////////////////////////////////////////
1178
1179 template <typename M, bool IsFlatAndTrivial>
1180 struct Destroyer {};
1181
1182 template <typename M>
1183 struct Destroyer<M, true> {
1184 void nodes(M& m) const noexcept {
1185 m.mNumElements = 0;
1186 }
1187
1188 void nodesDoNotDeallocate(M& m) const noexcept {
1189 m.mNumElements = 0;
1190 }
1191 };
1192
1193 template <typename M>
1194 struct Destroyer<M, false> {
1195 void nodes(M& m) const noexcept {
1196 m.mNumElements = 0;
1197 // clear also resets mInfo to 0, that's sometimes not necessary.
1198 auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1);
1199
1200 for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) {
1201 if (0 != m.mInfo[idx]) {
1202 Node& n = m.mKeyVals[idx];
1203 n.destroy(m);
1204 n.~Node();
1205 }
1206 }
1207 }
1208
1209 void nodesDoNotDeallocate(M& m) const noexcept {
1210 m.mNumElements = 0;
1211 // clear also resets mInfo to 0, that's sometimes not necessary.
1212 auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1);
1213 for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) {
1214 if (0 != m.mInfo[idx]) {
1215 Node& n = m.mKeyVals[idx];
1216 n.destroyDoNotDeallocate();
1217 n.~Node();
1218 }
1219 }
1220 }
1221 };
1222
1223 // Iter ////////////////////////////////////////////////////////////
1224
1225 struct fast_forward_tag {};
1226
1227 // generic iterator for both const_iterator and iterator.
1228 template <bool IsConst>
1229 // NOLINTNEXTLINE(hicpp-special-member-functions,cppcoreguidelines-special-member-functions)
1230 class Iter {
1231 private:
1232 using NodePtr = typename std::conditional<IsConst, Node const*, Node*>::type;
1233
1234 public:
1235 using difference_type = std::ptrdiff_t;
1236 using value_type = typename Self::value_type;
1237 using reference = typename std::conditional<IsConst, value_type const&, value_type&>::type;
1238 using pointer = typename std::conditional<IsConst, value_type const*, value_type*>::type;
1239 using iterator_category = std::forward_iterator_tag;
1240
1241 // default constructed iterator can be compared to itself, but WON'T return true when
1242 // compared to end().
1243 Iter() = default;
1244
1245 // Rule of zero: nothing specified. The conversion constructor is only enabled for
1246 // iterator to const_iterator, so it doesn't accidentally work as a copy ctor.
1247
1248 // Conversion constructor from iterator to const_iterator.
1249 template <bool OtherIsConst,
1250 typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
1251 // NOLINTNEXTLINE(hicpp-explicit-conversions)
1252 Iter(Iter<OtherIsConst> const& other) noexcept
1253 : mKeyVals(other.mKeyVals)
1254 , mInfo(other.mInfo) {}
1255
1256 Iter(NodePtr valPtr, uint8_t const* infoPtr) noexcept
1257 : mKeyVals(valPtr)
1258 , mInfo(infoPtr) {}
1259
1260 Iter(NodePtr valPtr, uint8_t const* infoPtr,
1261 fast_forward_tag ROBIN_HOOD_UNUSED(tag) /*unused*/) noexcept
1262 : mKeyVals(valPtr)
1263 , mInfo(infoPtr) {
1264 fastForward();
1265 }
1266
1267 template <bool OtherIsConst,
1268 typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
1269 Iter& operator=(Iter<OtherIsConst> const& other) noexcept {
1270 mKeyVals = other.mKeyVals;
1271 mInfo = other.mInfo;
1272 return *this;
1273 }
1274
1275 // prefix increment. Undefined behavior if we are at end()!
1276 Iter& operator++() noexcept {
1277 mInfo++;
1278 mKeyVals++;
1279 fastForward();
1280 return *this;
1281 }
1282
1283 Iter operator++(int) noexcept {
1284 Iter tmp = *this;
1285 ++(*this);
1286 return tmp;
1287 }
1288
1289 reference operator*() const {
1290 return **mKeyVals;
1291 }
1292
1293 pointer operator->() const {
1294 return &**mKeyVals;
1295 }
1296
1297 template <bool O>
1298 bool operator==(Iter<O> const& o) const noexcept {
1299 return mKeyVals == o.mKeyVals;
1300 }
1301
1302 template <bool O>
1303 bool operator!=(Iter<O> const& o) const noexcept {
1304 return mKeyVals != o.mKeyVals;
1305 }
1306
1307 private:
1308 // fast forward to the next non-free info byte
1309 // I've tried a few variants that don't depend on intrinsics, but unfortunately they are
1310 // quite a bit slower than this one. So I've reverted that change again. See map_benchmark.
1311 void fastForward() noexcept {
1312 size_t n = 0;
1313 while (0U == (n = detail::unaligned_load<size_t>(mInfo))) {
1314 mInfo += sizeof(size_t);
1315 mKeyVals += sizeof(size_t);
1316 }
1317#if defined(ROBIN_HOOD_DISABLE_INTRINSICS)
1318 // we know for certain that within the next 8 bytes we'll find a non-zero one.
1319 if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load<uint32_t>(mInfo))) {
1320 mInfo += 4;
1321 mKeyVals += 4;
1322 }
1323 if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load<uint16_t>(mInfo))) {
1324 mInfo += 2;
1325 mKeyVals += 2;
1326 }
1327 if (ROBIN_HOOD_UNLIKELY(0U == *mInfo)) {
1328 mInfo += 1;
1329 mKeyVals += 1;
1330 }
1331#else
1332# if ROBIN_HOOD(LITTLE_ENDIAN)
1333 auto inc = ROBIN_HOOD_COUNT_TRAILING_ZEROES(n) / 8;
1334# else
1335 auto inc = ROBIN_HOOD_COUNT_LEADING_ZEROES(n) / 8;
1336# endif
1337 mInfo += inc;
1338 mKeyVals += inc;
1339#endif
1340 }
1341
1342 friend class Table<IsFlat, MaxLoadFactor100, key_type, mapped_type, hasher, key_equal>;
1343 NodePtr mKeyVals{nullptr};
1344 uint8_t const* mInfo{nullptr};
1345 };
1346
1348
1349 // highly performance relevant code.
1350 // Lower bits are used for indexing into the array (2^n size)
1351 // The upper 1-5 bits need to be a reasonable good hash, to save comparisons.
1352 template <typename HashKey>
1353 void keyToIdx(HashKey&& key, size_t* idx, InfoType* info) const {
1354 // In addition to whatever hash is used, add another mul & shift so we get better hashing.
1355 // This serves as a bad hash prevention, if the given data is
1356 // badly mixed.
1357 auto h = static_cast<uint64_t>(WHash::operator()(key));
1358
1359 h *= mHashMultiplier;
1360 h ^= h >> 33U;
1361
1362 // the lower InitialInfoNumBits are reserved for info.
1363 *info = mInfoInc + static_cast<InfoType>((h & InfoMask) >> mInfoHashShift);
1364 *idx = (static_cast<size_t>(h) >> InitialInfoNumBits) & mMask;
1365 }
1366
1367 // forwards the index by one, wrapping around at the end
1368 void next(InfoType* info, size_t* idx) const noexcept {
1369 *idx = *idx + 1;
1370 *info += mInfoInc;
1371 }
1372
1373 void nextWhileLess(InfoType* info, size_t* idx) const noexcept {
1374 // unrolling this by hand did not bring any speedups.
1375 while (*info < mInfo[*idx]) {
1376 next(info, idx);
1377 }
1378 }
1379
1380 // Shift everything up by one element. Tries to move stuff around.
1381 void
1382 shiftUp(size_t startIdx,
1383 size_t const insertion_idx) noexcept(std::is_nothrow_move_assignable<Node>::value) {
1384 auto idx = startIdx;
1385 ::new (static_cast<void*>(mKeyVals + idx)) Node(std::move(mKeyVals[idx - 1]));
1386 while (--idx != insertion_idx) {
1387 mKeyVals[idx] = std::move(mKeyVals[idx - 1]);
1388 }
1389
1390 idx = startIdx;
1391 while (idx != insertion_idx) {
1392 ROBIN_HOOD_COUNT(shiftUp)
1393 mInfo[idx] = static_cast<uint8_t>(mInfo[idx - 1] + mInfoInc);
1394 if (ROBIN_HOOD_UNLIKELY(mInfo[idx] + mInfoInc > 0xFF)) {
1395 mMaxNumElementsAllowed = 0;
1396 }
1397 --idx;
1398 }
1399 }
1400
1401 void shiftDown(size_t idx) noexcept(std::is_nothrow_move_assignable<Node>::value) {
1402 // until we find one that is either empty or has zero offset.
1403 // TODO(martinus) we don't need to move everything, just the last one for the same
1404 // bucket.
1405 mKeyVals[idx].destroy(*this);
1406
1407 // until we find one that is either empty or has zero offset.
1408 while (mInfo[idx + 1] >= 2 * mInfoInc) {
1409 ROBIN_HOOD_COUNT(shiftDown)
1410 mInfo[idx] = static_cast<uint8_t>(mInfo[idx + 1] - mInfoInc);
1411 mKeyVals[idx] = std::move(mKeyVals[idx + 1]);
1412 ++idx;
1413 }
1414
1415 mInfo[idx] = 0;
1416 // don't destroy, we've moved it
1417 // mKeyVals[idx].destroy(*this);
1418 mKeyVals[idx].~Node();
1419 }
1420
1421 // copy of find(), except that it returns iterator instead of const_iterator.
1422 template <typename Other>
1423 ROBIN_HOOD(NODISCARD)
1424 size_t findIdx(Other const& key) const {
1425 size_t idx{};
1426 InfoType info{};
1427 keyToIdx(key, &idx, &info);
1428
1429 do {
1430 // unrolling this twice gives a bit of a speedup. More unrolling did not help.
1431 if (info == mInfo[idx] &&
1432 ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) {
1433 return idx;
1434 }
1435 next(&info, &idx);
1436 if (info == mInfo[idx] &&
1437 ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) {
1438 return idx;
1439 }
1440 next(&info, &idx);
1441 } while (info <= mInfo[idx]);
1442
1443 // nothing found!
1444 return mMask == 0 ? 0
1445 : static_cast<size_t>(std::distance(
1446 mKeyVals, reinterpret_cast_no_cast_align_warning<Node*>(mInfo)));
1447 }
1448
1449 void cloneData(const Table& o) {
1450 Cloner<Table, IsFlat && ROBIN_HOOD_IS_TRIVIALLY_COPYABLE(Node)>()(o, *this);
1451 }
1452
1453 // inserts a keyval that is guaranteed to be new, e.g. when the hashmap is resized.
1454 // @return True on success, false if something went wrong
1455 void insert_move(Node&& keyval) {
1456 // we don't retry, fail if overflowing
1457 // don't need to check max num elements
1458 if (0 == mMaxNumElementsAllowed && !try_increase_info()) {
1459 throwOverflowError();
1460 }
1461
1462 size_t idx{};
1463 InfoType info{};
1464 keyToIdx(keyval.getFirst(), &idx, &info);
1465
1466 // skip forward. Use <= because we are certain that the element is not there.
1467 while (info <= mInfo[idx]) {
1468 idx = idx + 1;
1469 info += mInfoInc;
1470 }
1471
1472 // key not found, so we are now exactly where we want to insert it.
1473 auto const insertion_idx = idx;
1474 auto const insertion_info = static_cast<uint8_t>(info);
1475 if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) {
1476 mMaxNumElementsAllowed = 0;
1477 }
1478
1479 // find an empty spot
1480 while (0 != mInfo[idx]) {
1481 next(&info, &idx);
1482 }
1483
1484 auto& l = mKeyVals[insertion_idx];
1485 if (idx == insertion_idx) {
1486 ::new (static_cast<void*>(&l)) Node(std::move(keyval));
1487 } else {
1488 shiftUp(idx, insertion_idx);
1489 l = std::move(keyval);
1490 }
1491
1492 // put at empty spot
1493 mInfo[insertion_idx] = insertion_info;
1494
1495 ++mNumElements;
1496 }
1497
1498public:
1499 using iterator = Iter<false>;
1500 using const_iterator = Iter<true>;
1501
1502 Table() noexcept(noexcept(Hash()) && noexcept(KeyEqual()))
1503 : WHash()
1504 , WKeyEqual() {
1505 ROBIN_HOOD_TRACE(this)
1506 }
1507
1508 // Creates an empty hash map. Nothing is allocated yet, this happens at the first insert.
1509 // This tremendously speeds up ctor & dtor of a map that never receives an element. The
1510 // penalty is payed at the first insert, and not before. Lookup of this empty map works
1511 // because everybody points to DummyInfoByte::b. parameter bucket_count is dictated by the
1512 // standard, but we can ignore it.
1513 explicit Table(
1514 size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/, const Hash& h = Hash{},
1515 const KeyEqual& equal = KeyEqual{}) noexcept(noexcept(Hash(h)) && noexcept(KeyEqual(equal)))
1516 : WHash(h)
1517 , WKeyEqual(equal) {
1518 ROBIN_HOOD_TRACE(this)
1519 }
1520
1521 template <typename Iter>
1522 Table(Iter first, Iter last, size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0,
1523 const Hash& h = Hash{}, const KeyEqual& equal = KeyEqual{})
1524 : WHash(h)
1525 , WKeyEqual(equal) {
1526 ROBIN_HOOD_TRACE(this)
1527 insert(first, last);
1528 }
1529
1530 Table(std::initializer_list<value_type> initlist,
1531 size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0, const Hash& h = Hash{},
1532 const KeyEqual& equal = KeyEqual{})
1533 : WHash(h)
1534 , WKeyEqual(equal) {
1535 ROBIN_HOOD_TRACE(this)
1536 insert(initlist.begin(), initlist.end());
1537 }
1538
1539 Table(Table&& o) noexcept
1540 : WHash(std::move(static_cast<WHash&>(o)))
1541 , WKeyEqual(std::move(static_cast<WKeyEqual&>(o)))
1542 , DataPool(std::move(static_cast<DataPool&>(o))) {
1543 ROBIN_HOOD_TRACE(this)
1544 if (o.mMask) {
1545 mHashMultiplier = std::move(o.mHashMultiplier);
1546 mKeyVals = std::move(o.mKeyVals);
1547 mInfo = std::move(o.mInfo);
1548 mNumElements = std::move(o.mNumElements);
1549 mMask = std::move(o.mMask);
1550 mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed);
1551 mInfoInc = std::move(o.mInfoInc);
1552 mInfoHashShift = std::move(o.mInfoHashShift);
1553 // set other's mask to 0 so its destructor won't do anything
1554 o.init();
1555 }
1556 }
1557
1558 Table& operator=(Table&& o) noexcept {
1559 ROBIN_HOOD_TRACE(this)
1560 if (&o != this) {
1561 if (o.mMask) {
1562 // only move stuff if the other map actually has some data
1563 destroy();
1564 mHashMultiplier = std::move(o.mHashMultiplier);
1565 mKeyVals = std::move(o.mKeyVals);
1566 mInfo = std::move(o.mInfo);
1567 mNumElements = std::move(o.mNumElements);
1568 mMask = std::move(o.mMask);
1569 mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed);
1570 mInfoInc = std::move(o.mInfoInc);
1571 mInfoHashShift = std::move(o.mInfoHashShift);
1572 WHash::operator=(std::move(static_cast<WHash&>(o)));
1573 WKeyEqual::operator=(std::move(static_cast<WKeyEqual&>(o)));
1574 DataPool::operator=(std::move(static_cast<DataPool&>(o)));
1575
1576 o.init();
1577
1578 } else {
1579 // nothing in the other map => just clear us.
1580 clear();
1581 }
1582 }
1583 return *this;
1584 }
1585
1586 Table(const Table& o)
1587 : WHash(static_cast<const WHash&>(o))
1588 , WKeyEqual(static_cast<const WKeyEqual&>(o))
1589 , DataPool(static_cast<const DataPool&>(o)) {
1590 ROBIN_HOOD_TRACE(this)
1591 if (!o.empty()) {
1592 // not empty: create an exact copy. it is also possible to just iterate through all
1593 // elements and insert them, but copying is probably faster.
1594
1595 auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1);
1596 auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
1597
1598 ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal("
1599 << numElementsWithBuffer << ")")
1600 mHashMultiplier = o.mHashMultiplier;
1601 mKeyVals = static_cast<Node*>(
1602 detail::assertNotNull<std::bad_alloc>(std::malloc(numBytesTotal)));
1603 // no need for calloc because clonData does memcpy
1604 mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
1605 mNumElements = o.mNumElements;
1606 mMask = o.mMask;
1607 mMaxNumElementsAllowed = o.mMaxNumElementsAllowed;
1608 mInfoInc = o.mInfoInc;
1609 mInfoHashShift = o.mInfoHashShift;
1610 cloneData(o);
1611 }
1612 }
1613
1614 // Creates a copy of the given map. Copy constructor of each entry is used.
1615 // Not sure why clang-tidy thinks this doesn't handle self assignment, it does
1616 // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp)
1617 Table& operator=(Table const& o) {
1618 ROBIN_HOOD_TRACE(this)
1619 if (&o == this) {
1620 // prevent assigning of itself
1621 return *this;
1622 }
1623
1624 // we keep using the old allocator and not assign the new one, because we want to keep
1625 // the memory available. when it is the same size.
1626 if (o.empty()) {
1627 if (0 == mMask) {
1628 // nothing to do, we are empty too
1629 return *this;
1630 }
1631
1632 // not empty: destroy what we have there
1633 // clear also resets mInfo to 0, that's sometimes not necessary.
1634 destroy();
1635 init();
1636 WHash::operator=(static_cast<const WHash&>(o));
1637 WKeyEqual::operator=(static_cast<const WKeyEqual&>(o));
1638 DataPool::operator=(static_cast<DataPool const&>(o));
1639
1640 return *this;
1641 }
1642
1643 // clean up old stuff
1644 Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}.nodes(*this);
1645
1646 if (mMask != o.mMask) {
1647 // no luck: we don't have the same array size allocated, so we need to realloc.
1648 if (0 != mMask) {
1649 // only deallocate if we actually have data!
1650 ROBIN_HOOD_LOG("std::free")
1651 std::free(mKeyVals);
1652 }
1653
1654 auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1);
1655 auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
1656 ROBIN_HOOD_LOG("std::malloc " << numBytesTotal << " = calcNumBytesTotal("
1657 << numElementsWithBuffer << ")")
1658 mKeyVals = static_cast<Node*>(
1659 detail::assertNotNull<std::bad_alloc>(std::malloc(numBytesTotal)));
1660
1661 // no need for calloc here because cloneData performs a memcpy.
1662 mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
1663 // sentinel is set in cloneData
1664 }
1665 WHash::operator=(static_cast<const WHash&>(o));
1666 WKeyEqual::operator=(static_cast<const WKeyEqual&>(o));
1667 DataPool::operator=(static_cast<DataPool const&>(o));
1668 mHashMultiplier = o.mHashMultiplier;
1669 mNumElements = o.mNumElements;
1670 mMask = o.mMask;
1671 mMaxNumElementsAllowed = o.mMaxNumElementsAllowed;
1672 mInfoInc = o.mInfoInc;
1673 mInfoHashShift = o.mInfoHashShift;
1674 cloneData(o);
1675
1676 return *this;
1677 }
1678
1679 // Swaps everything between the two maps.
1680 void swap(Table& o) {
1681 ROBIN_HOOD_TRACE(this)
1682 using std::swap;
1683 swap(o, *this);
1684 }
1685
1686 // Clears all data, without resizing.
1687 void clear() {
1688 ROBIN_HOOD_TRACE(this)
1689 if (empty()) {
1690 // don't do anything! also important because we don't want to write to
1691 // DummyInfoByte::b, even though we would just write 0 to it.
1692 return;
1693 }
1694
1695 Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}.nodes(*this);
1696
1697 auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
1698 // clear everything, then set the sentinel again
1699 uint8_t const z = 0;
1700 std::fill(mInfo, mInfo + calcNumBytesInfo(numElementsWithBuffer), z);
1701 mInfo[numElementsWithBuffer] = 1;
1702
1703 mInfoInc = InitialInfoInc;
1704 mInfoHashShift = InitialInfoHashShift;
1705 }
1706
1707 // Destroys the map and all it's contents.
1709 ROBIN_HOOD_TRACE(this)
1710 destroy();
1711 }
1712
1713 // Checks if both tables contain the same entries. Order is irrelevant.
1714 bool operator==(const Table& other) const {
1715 ROBIN_HOOD_TRACE(this)
1716 if (other.size() != size()) {
1717 return false;
1718 }
1719 for (auto const& otherEntry : other) {
1720 if (!has(otherEntry)) {
1721 return false;
1722 }
1723 }
1724
1725 return true;
1726 }
1727
1728 bool operator!=(const Table& other) const {
1729 ROBIN_HOOD_TRACE(this)
1730 return !operator==(other);
1731 }
1732
1733 template <typename Q = mapped_type>
1734 typename std::enable_if<!std::is_void<Q>::value, Q&>::type operator[](const key_type& key) {
1735 ROBIN_HOOD_TRACE(this)
1736 auto idxAndState = insertKeyPrepareEmptySpot(key);
1737 switch (idxAndState.second) {
1738 case InsertionState::key_found:
1739 break;
1740
1741 case InsertionState::new_node:
1742 ::new (static_cast<void*>(&mKeyVals[idxAndState.first]))
1743 Node(*this, std::piecewise_construct, std::forward_as_tuple(key),
1744 std::forward_as_tuple());
1745 break;
1746
1747 case InsertionState::overwrite_node:
1748 mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct,
1749 std::forward_as_tuple(key), std::forward_as_tuple());
1750 break;
1751
1752 case InsertionState::overflow_error:
1753 throwOverflowError();
1754 }
1755
1756 return mKeyVals[idxAndState.first].getSecond();
1757 }
1758
1759 template <typename Q = mapped_type>
1760 typename std::enable_if<!std::is_void<Q>::value, Q&>::type operator[](key_type&& key) {
1761 ROBIN_HOOD_TRACE(this)
1762 auto idxAndState = insertKeyPrepareEmptySpot(key);
1763 switch (idxAndState.second) {
1764 case InsertionState::key_found:
1765 break;
1766
1767 case InsertionState::new_node:
1768 ::new (static_cast<void*>(&mKeyVals[idxAndState.first]))
1769 Node(*this, std::piecewise_construct, std::forward_as_tuple(std::move(key)),
1770 std::forward_as_tuple());
1771 break;
1772
1773 case InsertionState::overwrite_node:
1774 mKeyVals[idxAndState.first] =
1775 Node(*this, std::piecewise_construct, std::forward_as_tuple(std::move(key)),
1776 std::forward_as_tuple());
1777 break;
1778
1779 case InsertionState::overflow_error:
1780 throwOverflowError();
1781 }
1782
1783 return mKeyVals[idxAndState.first].getSecond();
1784 }
1785
1786 template <typename Iter>
1787 void insert(Iter first, Iter last) {
1788 for (; first != last; ++first) {
1789 // value_type ctor needed because this might be called with std::pair's
1790 insert(value_type(*first));
1791 }
1792 }
1793
1794 void insert(std::initializer_list<value_type> ilist) {
1795 for (auto&& vt : ilist) {
1796 insert(std::move(vt));
1797 }
1798 }
1799
1800 template <typename... Args>
1801 std::pair<iterator, bool> emplace(Args&&... args) {
1802 ROBIN_HOOD_TRACE(this)
1803 Node n{*this, std::forward<Args>(args)...};
1804 auto idxAndState = insertKeyPrepareEmptySpot(getFirstConst(n));
1805 switch (idxAndState.second) {
1806 case InsertionState::key_found:
1807 n.destroy(*this);
1808 break;
1809
1810 case InsertionState::new_node:
1811 ::new (static_cast<void*>(&mKeyVals[idxAndState.first])) Node(*this, std::move(n));
1812 break;
1813
1814 case InsertionState::overwrite_node:
1815 mKeyVals[idxAndState.first] = std::move(n);
1816 break;
1817
1818 case InsertionState::overflow_error:
1819 n.destroy(*this);
1820 throwOverflowError();
1821 break;
1822 }
1823
1824 return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first),
1825 InsertionState::key_found != idxAndState.second);
1826 }
1827
1828 template <typename... Args>
1829 iterator emplace_hint(const_iterator position, Args&&... args) {
1830 (void)position;
1831 return emplace(std::forward<Args>(args)...).first;
1832 }
1833
1834 template <typename... Args>
1835 std::pair<iterator, bool> try_emplace(const key_type& key, Args&&... args) {
1836 return try_emplace_impl(key, std::forward<Args>(args)...);
1837 }
1838
1839 template <typename... Args>
1840 std::pair<iterator, bool> try_emplace(key_type&& key, Args&&... args) {
1841 return try_emplace_impl(std::move(key), std::forward<Args>(args)...);
1842 }
1843
1844 template <typename... Args>
1845 iterator try_emplace(const_iterator hint, const key_type& key, Args&&... args) {
1846 (void)hint;
1847 return try_emplace_impl(key, std::forward<Args>(args)...).first;
1848 }
1849
1850 template <typename... Args>
1851 iterator try_emplace(const_iterator hint, key_type&& key, Args&&... args) {
1852 (void)hint;
1853 return try_emplace_impl(std::move(key), std::forward<Args>(args)...).first;
1854 }
1855
1856 template <typename Mapped>
1857 std::pair<iterator, bool> insert_or_assign(const key_type& key, Mapped&& obj) {
1858 return insertOrAssignImpl(key, std::forward<Mapped>(obj));
1859 }
1860
1861 template <typename Mapped>
1862 std::pair<iterator, bool> insert_or_assign(key_type&& key, Mapped&& obj) {
1863 return insertOrAssignImpl(std::move(key), std::forward<Mapped>(obj));
1864 }
1865
1866 template <typename Mapped>
1867 iterator insert_or_assign(const_iterator hint, const key_type& key, Mapped&& obj) {
1868 (void)hint;
1869 return insertOrAssignImpl(key, std::forward<Mapped>(obj)).first;
1870 }
1871
1872 template <typename Mapped>
1874 (void)hint;
1875 return insertOrAssignImpl(std::move(key), std::forward<Mapped>(obj)).first;
1876 }
1877
1878 std::pair<iterator, bool> insert(const value_type& keyval) {
1879 ROBIN_HOOD_TRACE(this)
1880 return emplace(keyval);
1881 }
1882
1884 (void)hint;
1885 return emplace(keyval).first;
1886 }
1887
1888 std::pair<iterator, bool> insert(value_type&& keyval) {
1889 return emplace(std::move(keyval));
1890 }
1891
1893 (void)hint;
1894 return emplace(std::move(keyval)).first;
1895 }
1896
1897 // Returns 1 if key is found, 0 otherwise.
1898 size_t count(const key_type& key) const { // NOLINT(modernize-use-nodiscard)
1899 ROBIN_HOOD_TRACE(this)
1900 auto kv = mKeyVals + findIdx(key);
1901 if (kv != reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
1902 return 1;
1903 }
1904 return 0;
1905 }
1906
1907 template <typename OtherKey, typename Self_ = Self>
1908 // NOLINTNEXTLINE(modernize-use-nodiscard)
1909 typename std::enable_if<Self_::is_transparent, size_t>::type count(const OtherKey& key) const {
1910 ROBIN_HOOD_TRACE(this)
1911 auto kv = mKeyVals + findIdx(key);
1912 if (kv != reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
1913 return 1;
1914 }
1915 return 0;
1916 }
1917
1918 bool contains(const key_type& key) const { // NOLINT(modernize-use-nodiscard)
1919 return 1U == count(key);
1920 }
1921
1922 template <typename OtherKey, typename Self_ = Self>
1923 // NOLINTNEXTLINE(modernize-use-nodiscard)
1924 typename std::enable_if<Self_::is_transparent, bool>::type contains(const OtherKey& key) const {
1925 return 1U == count(key);
1926 }
1927
1928 // Returns a reference to the value found for key.
1929 // Throws std::out_of_range if element cannot be found
1930 template <typename Q = mapped_type>
1931 // NOLINTNEXTLINE(modernize-use-nodiscard)
1932 typename std::enable_if<!std::is_void<Q>::value, Q&>::type at(key_type const& key) {
1933 ROBIN_HOOD_TRACE(this)
1934 auto kv = mKeyVals + findIdx(key);
1935 if (kv == reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
1936 doThrow<std::out_of_range>("key not found");
1937 }
1938 return kv->getSecond();
1939 }
1940
1941 // Returns a reference to the value found for key.
1942 // Throws std::out_of_range if element cannot be found
1943 template <typename Q = mapped_type>
1944 // NOLINTNEXTLINE(modernize-use-nodiscard)
1945 typename std::enable_if<!std::is_void<Q>::value, Q const&>::type at(key_type const& key) const {
1946 ROBIN_HOOD_TRACE(this)
1947 auto kv = mKeyVals + findIdx(key);
1948 if (kv == reinterpret_cast_no_cast_align_warning<Node*>(mInfo)) {
1949 doThrow<std::out_of_range>("key not found");
1950 }
1951 return kv->getSecond();
1952 }
1953
1954 const_iterator find(const key_type& key) const { // NOLINT(modernize-use-nodiscard)
1955 ROBIN_HOOD_TRACE(this)
1956 const size_t idx = findIdx(key);
1957 return const_iterator{mKeyVals + idx, mInfo + idx};
1958 }
1959
1960 template <typename OtherKey>
1961 const_iterator find(const OtherKey& key, is_transparent_tag /*unused*/) const {
1962 ROBIN_HOOD_TRACE(this)
1963 const size_t idx = findIdx(key);
1964 return const_iterator{mKeyVals + idx, mInfo + idx};
1965 }
1966
1967 template <typename OtherKey, typename Self_ = Self>
1968 typename std::enable_if<Self_::is_transparent, // NOLINT(modernize-use-nodiscard)
1969 const_iterator>::type // NOLINT(modernize-use-nodiscard)
1970 find(const OtherKey& key) const { // NOLINT(modernize-use-nodiscard)
1971 ROBIN_HOOD_TRACE(this)
1972 const size_t idx = findIdx(key);
1973 return const_iterator{mKeyVals + idx, mInfo + idx};
1974 }
1975
1976 iterator find(const key_type& key) {
1977 ROBIN_HOOD_TRACE(this)
1978 const size_t idx = findIdx(key);
1979 return iterator{mKeyVals + idx, mInfo + idx};
1980 }
1981
1982 template <typename OtherKey>
1983 iterator find(const OtherKey& key, is_transparent_tag /*unused*/) {
1984 ROBIN_HOOD_TRACE(this)
1985 const size_t idx = findIdx(key);
1986 return iterator{mKeyVals + idx, mInfo + idx};
1987 }
1988
1989 template <typename OtherKey, typename Self_ = Self>
1990 typename std::enable_if<Self_::is_transparent, iterator>::type find(const OtherKey& key) {
1991 ROBIN_HOOD_TRACE(this)
1992 const size_t idx = findIdx(key);
1993 return iterator{mKeyVals + idx, mInfo + idx};
1994 }
1995
1997 ROBIN_HOOD_TRACE(this)
1998 if (empty()) {
1999 return end();
2000 }
2001 return iterator(mKeyVals, mInfo, fast_forward_tag{});
2002 }
2003 const_iterator begin() const { // NOLINT(modernize-use-nodiscard)
2004 ROBIN_HOOD_TRACE(this)
2005 return cbegin();
2006 }
2007 const_iterator cbegin() const { // NOLINT(modernize-use-nodiscard)
2008 ROBIN_HOOD_TRACE(this)
2009 if (empty()) {
2010 return cend();
2011 }
2012 return const_iterator(mKeyVals, mInfo, fast_forward_tag{});
2013 }
2014
2016 ROBIN_HOOD_TRACE(this)
2017 // no need to supply valid info pointer: end() must not be dereferenced, and only node
2018 // pointer is compared.
2019 return iterator{reinterpret_cast_no_cast_align_warning<Node*>(mInfo), nullptr};
2020 }
2021 const_iterator end() const { // NOLINT(modernize-use-nodiscard)
2022 ROBIN_HOOD_TRACE(this)
2023 return cend();
2024 }
2025 const_iterator cend() const { // NOLINT(modernize-use-nodiscard)
2026 ROBIN_HOOD_TRACE(this)
2027 return const_iterator{reinterpret_cast_no_cast_align_warning<Node*>(mInfo), nullptr};
2028 }
2029
2031 ROBIN_HOOD_TRACE(this)
2032 // its safe to perform const cast here
2033 // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast)
2034 return erase(iterator{const_cast<Node*>(pos.mKeyVals), const_cast<uint8_t*>(pos.mInfo)});
2035 }
2036
2037 // Erases element at pos, returns iterator to the next element.
2039 ROBIN_HOOD_TRACE(this)
2040 // we assume that pos always points to a valid entry, and not end().
2041 auto const idx = static_cast<size_t>(pos.mKeyVals - mKeyVals);
2042
2043 shiftDown(idx);
2044 --mNumElements;
2045
2046 if (*pos.mInfo) {
2047 // we've backward shifted, return this again
2048 return pos;
2049 }
2050
2051 // no backward shift, return next element
2052 return ++pos;
2053 }
2054
2055 size_t erase(const key_type& key) {
2056 ROBIN_HOOD_TRACE(this)
2057 size_t idx{};
2058 InfoType info{};
2059 keyToIdx(key, &idx, &info);
2060
2061 // check while info matches with the source idx
2062 do {
2063 if (info == mInfo[idx] && WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) {
2064 shiftDown(idx);
2065 --mNumElements;
2066 return 1;
2067 }
2068 next(&info, &idx);
2069 } while (info <= mInfo[idx]);
2070
2071 // nothing found to delete
2072 return 0;
2073 }
2074
2075 // reserves space for the specified number of elements. Makes sure the old data fits.
2076 // exactly the same as reserve(c).
2077 void rehash(size_t c) {
2078 // forces a reserve
2079 reserve(c, true);
2080 }
2081
2082 // reserves space for the specified number of elements. Makes sure the old data fits.
2083 // Exactly the same as rehash(c). Use rehash(0) to shrink to fit.
2084 void reserve(size_t c) {
2085 // reserve, but don't force rehash
2086 reserve(c, false);
2087 }
2088
2089 // If possible reallocates the map to a smaller one. This frees the underlying table.
2090 // Does not do anything if load_factor is too large for decreasing the table's size.
2091 void compact() {
2092 ROBIN_HOOD_TRACE(this)
2093 auto newSize = InitialNumElements;
2094 while (calcMaxNumElementsAllowed(newSize) < mNumElements && newSize != 0) {
2095 newSize *= 2;
2096 }
2097 if (ROBIN_HOOD_UNLIKELY(newSize == 0)) {
2098 throwOverflowError();
2099 }
2100
2101 ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1")
2102
2103 // only actually do anything when the new size is bigger than the old one. This prevents to
2104 // continuously allocate for each reserve() call.
2105 if (newSize < mMask + 1) {
2106 rehashPowerOfTwo(newSize, true);
2107 }
2108 }
2109
2110 size_type size() const noexcept { // NOLINT(modernize-use-nodiscard)
2111 ROBIN_HOOD_TRACE(this)
2112 return mNumElements;
2113 }
2114
2115 size_type max_size() const noexcept { // NOLINT(modernize-use-nodiscard)
2116 ROBIN_HOOD_TRACE(this)
2117 return static_cast<size_type>(-1);
2118 }
2119
2120 ROBIN_HOOD(NODISCARD) bool empty() const noexcept {
2121 ROBIN_HOOD_TRACE(this)
2122 return 0 == mNumElements;
2123 }
2124
2125 float max_load_factor() const noexcept { // NOLINT(modernize-use-nodiscard)
2126 ROBIN_HOOD_TRACE(this)
2127 return MaxLoadFactor100 / 100.0F;
2128 }
2129
2130 // Average number of elements per bucket. Since we allow only 1 per bucket
2131 float load_factor() const noexcept { // NOLINT(modernize-use-nodiscard)
2132 ROBIN_HOOD_TRACE(this)
2133 return static_cast<float>(size()) / static_cast<float>(mMask + 1);
2134 }
2135
2136 ROBIN_HOOD(NODISCARD) size_t mask() const noexcept {
2137 ROBIN_HOOD_TRACE(this)
2138 return mMask;
2139 }
2140
2141 ROBIN_HOOD(NODISCARD) size_t calcMaxNumElementsAllowed(size_t maxElements) const noexcept {
2142 if (ROBIN_HOOD_LIKELY(maxElements <= (std::numeric_limits<size_t>::max)() / 100)) {
2143 return maxElements * MaxLoadFactor100 / 100;
2144 }
2145
2146 // we might be a bit inprecise, but since maxElements is quite large that doesn't matter
2147 return (maxElements / 100) * MaxLoadFactor100;
2148 }
2149
2150 ROBIN_HOOD(NODISCARD) size_t calcNumBytesInfo(size_t numElements) const noexcept {
2151 // we add a uint64_t, which houses the sentinel (first byte) and padding so we can load
2152 // 64bit types.
2153 return numElements + sizeof(uint64_t);
2154 }
2155
2156 ROBIN_HOOD(NODISCARD)
2157 size_t calcNumElementsWithBuffer(size_t numElements) const noexcept {
2158 auto maxNumElementsAllowed = calcMaxNumElementsAllowed(numElements);
2159 return numElements + (std::min)(maxNumElementsAllowed, (static_cast<size_t>(0xFF)));
2160 }
2161
2162 // calculation only allowed for 2^n values
2163 ROBIN_HOOD(NODISCARD) size_t calcNumBytesTotal(size_t numElements) const {
2164#if ROBIN_HOOD(BITNESS) == 64
2165 return numElements * sizeof(Node) + calcNumBytesInfo(numElements);
2166#else
2167 // make sure we're doing 64bit operations, so we are at least safe against 32bit overflows.
2168 auto const ne = static_cast<uint64_t>(numElements);
2169 auto const s = static_cast<uint64_t>(sizeof(Node));
2170 auto const infos = static_cast<uint64_t>(calcNumBytesInfo(numElements));
2171
2172 auto const total64 = ne * s + infos;
2173 auto const total = static_cast<size_t>(total64);
2174
2175 if (ROBIN_HOOD_UNLIKELY(static_cast<uint64_t>(total) != total64)) {
2176 throwOverflowError();
2177 }
2178 return total;
2179#endif
2180 }
2181
2182private:
2183 template <typename Q = mapped_type>
2184 ROBIN_HOOD(NODISCARD)
2185 typename std::enable_if<!std::is_void<Q>::value, bool>::type has(const value_type& e) const {
2186 ROBIN_HOOD_TRACE(this)
2187 auto it = find(e.first);
2188 return it != end() && it->second == e.second;
2189 }
2190
2191 template <typename Q = mapped_type>
2192 ROBIN_HOOD(NODISCARD)
2193 typename std::enable_if<std::is_void<Q>::value, bool>::type has(const value_type& e) const {
2194 ROBIN_HOOD_TRACE(this)
2195 return find(e) != end();
2196 }
2197
2198 void reserve(size_t c, bool forceRehash) {
2199 ROBIN_HOOD_TRACE(this)
2200 auto const minElementsAllowed = (std::max)(c, mNumElements);
2201 auto newSize = InitialNumElements;
2202 while (calcMaxNumElementsAllowed(newSize) < minElementsAllowed && newSize != 0) {
2203 newSize *= 2;
2204 }
2205 if (ROBIN_HOOD_UNLIKELY(newSize == 0)) {
2206 throwOverflowError();
2207 }
2208
2209 ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1")
2210
2211 // only actually do anything when the new size is bigger than the old one. This prevents to
2212 // continuously allocate for each reserve() call.
2213 if (forceRehash || newSize > mMask + 1) {
2214 rehashPowerOfTwo(newSize, false);
2215 }
2216 }
2217
2218 // reserves space for at least the specified number of elements.
2219 // only works if numBuckets if power of two
2220 // True on success, false otherwise
2221 void rehashPowerOfTwo(size_t numBuckets, bool forceFree) {
2222 ROBIN_HOOD_TRACE(this)
2223
2224 Node* const oldKeyVals = mKeyVals;
2225 uint8_t const* const oldInfo = mInfo;
2226
2227 const size_t oldMaxElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
2228
2229 // resize operation: move stuff
2230 initData(numBuckets);
2231 if (oldMaxElementsWithBuffer > 1) {
2232 for (size_t i = 0; i < oldMaxElementsWithBuffer; ++i) {
2233 if (oldInfo[i] != 0) {
2234 // might throw an exception, which is really bad since we are in the middle of
2235 // moving stuff.
2236 insert_move(std::move(oldKeyVals[i]));
2237 // destroy the node but DON'T destroy the data.
2238 oldKeyVals[i].~Node();
2239 }
2240 }
2241
2242 // this check is not necessary as it's guarded by the previous if, but it helps
2243 // silence g++'s overeager "attempt to free a non-heap object 'map'
2244 // [-Werror=free-nonheap-object]" warning.
2245 if (oldKeyVals != reinterpret_cast_no_cast_align_warning<Node*>(&mMask)) {
2246 // don't destroy old data: put it into the pool instead
2247 if (forceFree) {
2248 std::free(oldKeyVals);
2249 } else {
2250 DataPool::addOrFree(oldKeyVals, calcNumBytesTotal(oldMaxElementsWithBuffer));
2251 }
2252 }
2253 }
2254 }
2255
2256 ROBIN_HOOD(NOINLINE) void throwOverflowError() const {
2257#if ROBIN_HOOD(HAS_EXCEPTIONS)
2258 throw std::overflow_error("robin_hood::map overflow");
2259#else
2260 abort();
2261#endif
2262 }
2263
2264 template <typename OtherKey, typename... Args>
2265 std::pair<iterator, bool> try_emplace_impl(OtherKey&& key, Args&&... args) {
2266 ROBIN_HOOD_TRACE(this)
2267 auto idxAndState = insertKeyPrepareEmptySpot(key);
2268 switch (idxAndState.second) {
2269 case InsertionState::key_found:
2270 break;
2271
2272 case InsertionState::new_node:
2273 ::new (static_cast<void*>(&mKeyVals[idxAndState.first])) Node(
2274 *this, std::piecewise_construct, std::forward_as_tuple(std::forward<OtherKey>(key)),
2275 std::forward_as_tuple(std::forward<Args>(args)...));
2276 break;
2277
2278 case InsertionState::overwrite_node:
2279 mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct,
2280 std::forward_as_tuple(std::forward<OtherKey>(key)),
2281 std::forward_as_tuple(std::forward<Args>(args)...));
2282 break;
2283
2284 case InsertionState::overflow_error:
2285 throwOverflowError();
2286 break;
2287 }
2288
2289 return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first),
2290 InsertionState::key_found != idxAndState.second);
2291 }
2292
2293 template <typename OtherKey, typename Mapped>
2294 std::pair<iterator, bool> insertOrAssignImpl(OtherKey&& key, Mapped&& obj) {
2295 ROBIN_HOOD_TRACE(this)
2296 auto idxAndState = insertKeyPrepareEmptySpot(key);
2297 switch (idxAndState.second) {
2298 case InsertionState::key_found:
2299 mKeyVals[idxAndState.first].getSecond() = std::forward<Mapped>(obj);
2300 break;
2301
2302 case InsertionState::new_node:
2303 ::new (static_cast<void*>(&mKeyVals[idxAndState.first])) Node(
2304 *this, std::piecewise_construct, std::forward_as_tuple(std::forward<OtherKey>(key)),
2305 std::forward_as_tuple(std::forward<Mapped>(obj)));
2306 break;
2307
2308 case InsertionState::overwrite_node:
2309 mKeyVals[idxAndState.first] = Node(*this, std::piecewise_construct,
2310 std::forward_as_tuple(std::forward<OtherKey>(key)),
2311 std::forward_as_tuple(std::forward<Mapped>(obj)));
2312 break;
2313
2314 case InsertionState::overflow_error:
2315 throwOverflowError();
2316 break;
2317 }
2318
2319 return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first),
2320 InsertionState::key_found != idxAndState.second);
2321 }
2322
2323 void initData(size_t max_elements) {
2324 mNumElements = 0;
2325 mMask = max_elements - 1;
2326 mMaxNumElementsAllowed = calcMaxNumElementsAllowed(max_elements);
2327
2328 auto const numElementsWithBuffer = calcNumElementsWithBuffer(max_elements);
2329
2330 // malloc & zero mInfo. Faster than calloc everything.
2331 auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
2332 ROBIN_HOOD_LOG("std::calloc " << numBytesTotal << " = calcNumBytesTotal("
2333 << numElementsWithBuffer << ")")
2334 mKeyVals = reinterpret_cast<Node*>(
2335 detail::assertNotNull<std::bad_alloc>(std::malloc(numBytesTotal)));
2336 mInfo = reinterpret_cast<uint8_t*>(mKeyVals + numElementsWithBuffer);
2337 std::memset(mInfo, 0, numBytesTotal - numElementsWithBuffer * sizeof(Node));
2338
2339 // set sentinel
2340 mInfo[numElementsWithBuffer] = 1;
2341
2342 mInfoInc = InitialInfoInc;
2343 mInfoHashShift = InitialInfoHashShift;
2344 }
2345
2346 enum class InsertionState { overflow_error, key_found, new_node, overwrite_node };
2347
2348 // Finds key, and if not already present prepares a spot where to pot the key & value.
2349 // This potentially shifts nodes out of the way, updates mInfo and number of inserted
2350 // elements, so the only operation left to do is create/assign a new node at that spot.
2351 template <typename OtherKey>
2352 std::pair<size_t, InsertionState> insertKeyPrepareEmptySpot(OtherKey&& key) {
2353 for (int i = 0; i < 256; ++i) {
2354 size_t idx{};
2355 InfoType info{};
2356 keyToIdx(key, &idx, &info);
2357 nextWhileLess(&info, &idx);
2358
2359 // while we potentially have a match
2360 while (info == mInfo[idx]) {
2361 if (WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) {
2362 // key already exists, do NOT insert.
2363 // see http://en.cppreference.com/w/cpp/container/unordered_map/insert
2364 return std::make_pair(idx, InsertionState::key_found);
2365 }
2366 next(&info, &idx);
2367 }
2368
2369 // unlikely that this evaluates to true
2370 if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) {
2371 if (!increase_size()) {
2372 return std::make_pair(size_t(0), InsertionState::overflow_error);
2373 }
2374 continue;
2375 }
2376
2377 // key not found, so we are now exactly where we want to insert it.
2378 auto const insertion_idx = idx;
2379 auto const insertion_info = info;
2380 if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) {
2381 mMaxNumElementsAllowed = 0;
2382 }
2383
2384 // find an empty spot
2385 while (0 != mInfo[idx]) {
2386 next(&info, &idx);
2387 }
2388
2389 if (idx != insertion_idx) {
2390 shiftUp(idx, insertion_idx);
2391 }
2392 // put at empty spot
2393 mInfo[insertion_idx] = static_cast<uint8_t>(insertion_info);
2394 ++mNumElements;
2395 return std::make_pair(insertion_idx, idx == insertion_idx
2396 ? InsertionState::new_node
2397 : InsertionState::overwrite_node);
2398 }
2399
2400 // enough attempts failed, so finally give up.
2401 return std::make_pair(size_t(0), InsertionState::overflow_error);
2402 }
2403
2404 bool try_increase_info() {
2405 ROBIN_HOOD_LOG("mInfoInc=" << mInfoInc << ", numElements=" << mNumElements
2406 << ", maxNumElementsAllowed="
2407 << calcMaxNumElementsAllowed(mMask + 1))
2408 if (mInfoInc <= 2) {
2409 // need to be > 2 so that shift works (otherwise undefined behavior!)
2410 return false;
2411 }
2412 // we got space left, try to make info smaller
2413 mInfoInc = static_cast<uint8_t>(mInfoInc >> 1U);
2414
2415 // remove one bit of the hash, leaving more space for the distance info.
2416 // This is extremely fast because we can operate on 8 bytes at once.
2417 ++mInfoHashShift;
2418 auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
2419
2420 for (size_t i = 0; i < numElementsWithBuffer; i += 8) {
2421 auto val = unaligned_load<uint64_t>(mInfo + i);
2422 val = (val >> 1U) & UINT64_C(0x7f7f7f7f7f7f7f7f);
2423 std::memcpy(mInfo + i, &val, sizeof(val));
2424 }
2425 // update sentinel, which might have been cleared out!
2426 mInfo[numElementsWithBuffer] = 1;
2427
2428 mMaxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1);
2429 return true;
2430 }
2431
2432 // True if resize was possible, false otherwise
2433 bool increase_size() {
2434 // nothing allocated yet? just allocate InitialNumElements
2435 if (0 == mMask) {
2436 initData(InitialNumElements);
2437 return true;
2438 }
2439
2440 auto const maxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1);
2441 if (mNumElements < maxNumElementsAllowed && try_increase_info()) {
2442 return true;
2443 }
2444
2445 ROBIN_HOOD_LOG("mNumElements=" << mNumElements << ", maxNumElementsAllowed="
2446 << maxNumElementsAllowed << ", load="
2447 << (static_cast<double>(mNumElements) * 100.0 /
2448 (static_cast<double>(mMask) + 1)))
2449
2450 if (mNumElements * 2 < calcMaxNumElementsAllowed(mMask + 1)) {
2451 // we have to resize, even though there would still be plenty of space left!
2452 // Try to rehash instead. Delete freed memory so we don't steadyily increase mem in case
2453 // we have to rehash a few times
2454 nextHashMultiplier();
2455 rehashPowerOfTwo(mMask + 1, true);
2456 } else {
2457 // we've reached the capacity of the map, so the hash seems to work nice. Keep using it.
2458 rehashPowerOfTwo((mMask + 1) * 2, false);
2459 }
2460 return true;
2461 }
2462
2463 void nextHashMultiplier() {
2464 // adding an *even* number, so that the multiplier will always stay odd. This is necessary
2465 // so that the hash stays a mixing function (and thus doesn't have any information loss).
2466 mHashMultiplier += UINT64_C(0xc4ceb9fe1a85ec54);
2467 }
2468
2469 void destroy() {
2470 if (0 == mMask) {
2471 // don't deallocate!
2472 return;
2473 }
2474
2475 Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}
2476 .nodesDoNotDeallocate(*this);
2477
2478 // This protection against not deleting mMask shouldn't be needed as it's sufficiently
2479 // protected with the 0==mMask check, but I have this anyways because g++ 7 otherwise
2480 // reports a compile error: attempt to free a non-heap object 'fm'
2481 // [-Werror=free-nonheap-object]
2482 if (mKeyVals != reinterpret_cast_no_cast_align_warning<Node*>(&mMask)) {
2483 ROBIN_HOOD_LOG("std::free")
2484 std::free(mKeyVals);
2485 }
2486 }
2487
2488 void init() noexcept {
2489 mKeyVals = reinterpret_cast_no_cast_align_warning<Node*>(&mMask);
2490 mInfo = reinterpret_cast<uint8_t*>(&mMask);
2491 mNumElements = 0;
2492 mMask = 0;
2493 mMaxNumElementsAllowed = 0;
2494 mInfoInc = InitialInfoInc;
2495 mInfoHashShift = InitialInfoHashShift;
2496 }
2497
2498 // members are sorted so no padding occurs
2499 uint64_t mHashMultiplier = UINT64_C(0xc4ceb9fe1a85ec53); // 8 byte 8
2500 Node* mKeyVals = reinterpret_cast_no_cast_align_warning<Node*>(&mMask); // 8 byte 16
2501 uint8_t* mInfo = reinterpret_cast<uint8_t*>(&mMask); // 8 byte 24
2502 size_t mNumElements = 0; // 8 byte 32
2503 size_t mMask = 0; // 8 byte 40
2504 size_t mMaxNumElementsAllowed = 0; // 8 byte 48
2505 InfoType mInfoInc = InitialInfoInc; // 4 byte 52
2506 InfoType mInfoHashShift = InitialInfoHashShift; // 4 byte 56
2507 // 16 byte 56 if NodeAllocator
2508};
2509
2510} // namespace detail
2511
2512
2513#if defined(__clang__)
2514#pragma clang diagnostic pop
2515#endif
2516
2517// map
2518
2519template <typename Key, typename T, typename Hash = hash<Key>,
2520 typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
2522
2523template <typename Key, typename T, typename Hash = hash<Key>,
2524 typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
2526
2527template <typename Key, typename T, typename Hash = hash<Key>,
2528 typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
2530 detail::Table<sizeof(robin_hood::pair<Key, T>) <= sizeof(size_t) * 6 &&
2531 std::is_nothrow_move_constructible<robin_hood::pair<Key, T>>::value &&
2532 std::is_nothrow_move_assignable<robin_hood::pair<Key, T>>::value,
2533 MaxLoadFactor100, Key, T, Hash, KeyEqual>;
2534
2535// set
2536
2537template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
2538 size_t MaxLoadFactor100 = 80>
2540
2541template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
2542 size_t MaxLoadFactor100 = 80>
2544
2545template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>,
2546 size_t MaxLoadFactor100 = 80>
2547using unordered_set = detail::Table<sizeof(Key) <= sizeof(size_t) * 6 &&
2548 std::is_nothrow_move_constructible<Key>::value &&
2549 std::is_nothrow_move_assignable<Key>::value,
2550 MaxLoadFactor100, Key, void, Hash, KeyEqual>;
2551
2552} // namespace robin_hood
2553
2554#endif
static constexpr std::size_t size() noexcept
Definition robin_hood.h:250
T value_type
Definition robin_hood.h:248
Definition robin_hood.h:372
~BulkPoolAllocator() noexcept
Definition robin_hood.h:404
BulkPoolAllocator & operator=(BulkPoolAllocator &&o) noexcept
Definition robin_hood.h:388
void swap(BulkPoolAllocator< T, MinNumAllocs, MaxNumAllocs > &other) noexcept
Definition robin_hood.h:456
void reset() noexcept
Definition robin_hood.h:409
void deallocate(T *obj) noexcept
Definition robin_hood.h:436
BulkPoolAllocator & operator=(const BulkPoolAllocator &ROBIN_HOOD_UNUSED(o)) noexcept
Definition robin_hood.h:399
BulkPoolAllocator(BulkPoolAllocator &&o) noexcept
Definition robin_hood.h:381
T * allocate()
Definition robin_hood.h:422
void addOrFree(void *ptr, const size_t numBytes) noexcept
Definition robin_hood.h:444
Definition robin_hood.h:921
void rehash(size_t c)
Definition robin_hood.h:2077
std::enable_if< Self_::is_transparent, const_iterator >::type find(const OtherKey &key) const
Definition robin_hood.h:1970
iterator find(const key_type &key)
Definition robin_hood.h:1976
iterator erase(const_iterator pos)
Definition robin_hood.h:2030
const_iterator cend() const
Definition robin_hood.h:2025
static constexpr bool is_map
Definition robin_hood.h:924
Hash hasher
Definition robin_hood.h:935
std::pair< iterator, bool > try_emplace(const key_type &key, Args &&... args)
Definition robin_hood.h:1835
Iter< true > const_iterator
Definition robin_hood.h:1500
std::pair< iterator, bool > try_emplace(key_type &&key, Args &&... args)
Definition robin_hood.h:1840
Table(size_t ROBIN_HOOD_UNUSED(bucket_count), const Hash &h=Hash{}, const KeyEqual &equal=KeyEqual{}) noexcept(noexcept(Hash(h)) &&noexcept(KeyEqual(equal)))
Definition robin_hood.h:1513
void swap(Table &o)
Definition robin_hood.h:1680
const_iterator end() const
Definition robin_hood.h:2021
Table(Iter first, Iter last, size_t ROBIN_HOOD_UNUSED(bucket_count)=0, const Hash &h=Hash{}, const KeyEqual &equal=KeyEqual{})
Definition robin_hood.h:1522
Table & operator=(Table const &o)
Definition robin_hood.h:1617
void compact()
Definition robin_hood.h:2091
ROBIN_HOOD(NODISCARD) size_t calcMaxNumElementsAllowed(size_t maxElements) const noexcept
Definition robin_hood.h:2141
iterator insert(const_iterator hint, value_type &&keyval)
Definition robin_hood.h:1892
std::pair< iterator, bool > insert(value_type &&keyval)
Definition robin_hood.h:1888
Iter< false > iterator
Definition robin_hood.h:1499
bool operator==(const Table &other) const
Definition robin_hood.h:1714
std::pair< iterator, bool > insert_or_assign(key_type &&key, Mapped &&obj)
Definition robin_hood.h:1862
std::pair< iterator, bool > insert_or_assign(const key_type &key, Mapped &&obj)
Definition robin_hood.h:1857
ROBIN_HOOD(NODISCARD) bool empty() const noexcept
Definition robin_hood.h:2120
void clear()
Definition robin_hood.h:1687
ROBIN_HOOD(NODISCARD) size_t calcNumBytesInfo(size_t numElements) const noexcept
Definition robin_hood.h:2150
void reserve(size_t c)
Definition robin_hood.h:2084
bool operator!=(const Table &other) const
Definition robin_hood.h:1728
Key key_type
Definition robin_hood.h:929
const_iterator find(const key_type &key) const
Definition robin_hood.h:1954
iterator end()
Definition robin_hood.h:2015
bool contains(const key_type &key) const
Definition robin_hood.h:1918
iterator find(const OtherKey &key, is_transparent_tag)
Definition robin_hood.h:1983
size_t size_type
Definition robin_hood.h:934
size_t erase(const key_type &key)
Definition robin_hood.h:2055
std::enable_if<!std::is_void< Q >::value, Q & >::type operator[](const key_type &key)
Definition robin_hood.h:1734
const_iterator begin() const
Definition robin_hood.h:2003
size_t count(const key_type &key) const
Definition robin_hood.h:1898
iterator try_emplace(const_iterator hint, const key_type &key, Args &&... args)
Definition robin_hood.h:1845
const_iterator cbegin() const
Definition robin_hood.h:2007
std::enable_if<!std::is_void< Q >::value, Qconst & >::type at(key_type const &key) const
Definition robin_hood.h:1945
~Table()
Definition robin_hood.h:1708
size_type size() const noexcept
Definition robin_hood.h:2110
size_t calcNumElementsWithBuffer(size_t numElements) const noexcept
Definition robin_hood.h:2157
KeyEqual key_equal
Definition robin_hood.h:936
void insert(std::initializer_list< value_type > ilist)
Definition robin_hood.h:1794
iterator insert_or_assign(const_iterator hint, const key_type &key, Mapped &&obj)
Definition robin_hood.h:1867
iterator emplace_hint(const_iterator position, Args &&... args)
Definition robin_hood.h:1829
typename std::conditional< is_set, Key, robin_hood::pair< typename std::conditional< is_flat, Key, Key const >::type, T > >::type value_type
Definition robin_hood.h:933
float load_factor() const noexcept
Definition robin_hood.h:2131
std::enable_if< Self_::is_transparent, iterator >::type find(const OtherKey &key)
Definition robin_hood.h:1990
float max_load_factor() const noexcept
Definition robin_hood.h:2125
ROBIN_HOOD(NODISCARD) size_t calcNumBytesTotal(size_t numElements) const
Definition robin_hood.h:2163
ROBIN_HOOD(NODISCARD) size_t mask() const noexcept
Definition robin_hood.h:2136
std::pair< iterator, bool > insert(const value_type &keyval)
Definition robin_hood.h:1878
size_type max_size() const noexcept
Definition robin_hood.h:2115
iterator insert(const_iterator hint, const value_type &keyval)
Definition robin_hood.h:1883
static constexpr bool is_flat
Definition robin_hood.h:923
std::pair< iterator, bool > emplace(Args &&... args)
Definition robin_hood.h:1801
T mapped_type
Definition robin_hood.h:930
Table() noexcept(noexcept(Hash()) &&noexcept(KeyEqual()))
Definition robin_hood.h:1502
iterator begin()
Definition robin_hood.h:1996
Table & operator=(Table &&o) noexcept
Definition robin_hood.h:1558
const_iterator find(const OtherKey &key, is_transparent_tag) const
Definition robin_hood.h:1961
std::enable_if< Self_::is_transparent, bool >::type contains(const OtherKey &key) const
Definition robin_hood.h:1924
iterator try_emplace(const_iterator hint, key_type &&key, Args &&... args)
Definition robin_hood.h:1851
std::enable_if< Self_::is_transparent, size_t >::type count(const OtherKey &key) const
Definition robin_hood.h:1909
Table(const Table &o)
Definition robin_hood.h:1586
iterator insert_or_assign(const_iterator hint, key_type &&key, Mapped &&obj)
Definition robin_hood.h:1873
static constexpr bool is_transparent
Definition robin_hood.h:926
static constexpr bool is_set
Definition robin_hood.h:925
iterator erase(iterator pos)
Definition robin_hood.h:2038
std::enable_if<!std::is_void< Q >::value, Q & >::type operator[](key_type &&key)
Definition robin_hood.h:1760
void insert(Iter first, Iter last)
Definition robin_hood.h:1787
std::enable_if<!std::is_void< Q >::value, Q & >::type at(key_type const &key)
Definition robin_hood.h:1932
typename detail_::IntSeqImpl< T, 0, N,(N - 0)==1 >::TResult make_integer_sequence
Definition robin_hood.h:297
make_index_sequence< sizeof...(T)> index_sequence_for
Definition robin_hood.h:303
make_integer_sequence< std::size_t, N > make_index_sequence
Definition robin_hood.h:300
T reinterpret_cast_no_cast_align_warning(void *ptr) noexcept
Definition robin_hood.h:327
T unaligned_load(void const *ptr) noexcept
Definition robin_hood.h:360
T * assertNotNull(T *t, Args &&... args)
Definition robin_hood.h:352
uint32_t SizeT
Definition robin_hood.h:315
T rotr(T x, unsigned k)
Definition robin_hood.h:319
Definition robin_hood.h:233
constexpr bool operator==(pair< A, B > const &x, pair< A, B > const &y)
Definition robin_hood.h:671
constexpr bool operator<=(pair< A, B > const &x, pair< A, B > const &y)
Definition robin_hood.h:689
constexpr bool operator>(pair< A, B > const &x, pair< A, B > const &y)
Definition robin_hood.h:685
constexpr bool operator!=(pair< A, B > const &x, pair< A, B > const &y)
Definition robin_hood.h:675
size_t hash_int(uint64_t x) noexcept
Definition robin_hood.h:753
void swap(pair< A, B > &a, pair< A, B > &b) noexcept(noexcept(std::declval< pair< A, B > & >().swap(std::declval< pair< A, B > & >())))
Definition robin_hood.h:665
size_t hash_bytes(void const *ptr, size_t len) noexcept
Definition robin_hood.h:697
constexpr bool operator<(pair< A, B > const &x, pair< A, B > const &y) noexcept(noexcept(std::declval< A const & >()< std::declval< A const & >()) &&noexcept(std::declval< B const & >()< std::declval< B const & >()))
Definition robin_hood.h:679
constexpr bool operator>=(pair< A, B > const &x, pair< A, B > const &y)
Definition robin_hood.h:693
Definition jobManagement.cpp:168
#define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x)
Definition robin_hood.h:160
#define ROBIN_HOOD_COUNT_LEADING_ZEROES(x)
Definition robin_hood.h:159
#define ROBIN_HOOD_UNUSED(identifier)
Definition robin_hood.h:100
#define ROBIN_HOOD_COUNT(x)
Definition robin_hood.h:92
#define ROBIN_HOOD_TRACE(x)
Definition robin_hood.h:70
#define ROBIN_HOOD_HASH_INT(T)
Definition robin_hood.h:823
#define ROBIN_HOOD(x)
Definition robin_hood.h:97
#define ROBIN_HOOD_UNLIKELY(condition)
Definition robin_hood.h:182
#define ROBIN_HOOD_LIKELY(condition)
Definition robin_hood.h:181
#define ROBIN_HOOD_LOG(x)
Definition robin_hood.h:61
Definition robin_hood.h:243
typename IntSeqCombiner< typename IntSeqImpl< TValue, Begin, Begin+(End - Begin)/2,(End - Begin)/2==1 >::TResult, typename IntSeqImpl< TValue, Begin+(End - Begin)/2, End,(End - Begin+1)/2==1 >::TResult >::TResult TResult
Definition robin_hood.h:276
void addOrFree(void *ptr, size_t ROBIN_HOOD_UNUSED(numBytes)) noexcept
Definition robin_hood.h:553
Definition robin_hood.h:546
Definition robin_hood.h:873
WrapHash(T const &o) noexcept(noexcept(T(std::declval< T const & >())))
Definition robin_hood.h:875
Definition robin_hood.h:880
WrapKeyEqual(T const &o) noexcept(noexcept(T(std::declval< T const & >())))
Definition robin_hood.h:882
Definition robin_hood.h:864
Definition robin_hood.h:568
static const bool value
Definition robin_hood.h:569
Definition robin_hood.h:859
void type
Definition robin_hood.h:860
size_t operator()(Enum e) const noexcept
Definition robin_hood.h:817
size_t operator()(T *ptr) const noexcept
Definition robin_hood.h:796
size_t operator()(std::basic_string< CharT > const &str) const noexcept
Definition robin_hood.h:780
size_t operator()(std::shared_ptr< T > const &ptr) const noexcept
Definition robin_hood.h:810
size_t operator()(std::unique_ptr< T > const &ptr) const noexcept
Definition robin_hood.h:803
Definition robin_hood.h:768
size_t operator()(T const &obj) const noexcept(noexcept(std::declval< std::hash< T > >().operator()(std::declval< T const & >())))
Definition robin_hood.h:769
Definition robin_hood.h:581
Definition robin_hood.h:587
constexpr pair(std::piecewise_construct_t, std::tuple< U1... > a, std::tuple< U2... > b) noexcept(noexcept(pair(std::declval< std::tuple< U1... > & >(), std::declval< std::tuple< U2... > & >(), ROBIN_HOOD_STD::index_sequence_for< U1... >(), ROBIN_HOOD_STD::index_sequence_for< U2... >())))
Definition robin_hood.h:627
void swap(pair< T1, T2 > &o) noexcept((detail::swappable::nothrow< T1 >::value) &&(detail::swappable::nothrow< T2 >::value))
Definition robin_hood.h:653
constexpr pair(std::pair< T1, T2 > &&o) noexcept(noexcept(T1(std::move(std::declval< T1 && >()))) &&noexcept(T2(std::move(std::declval< T2 && >()))))
Definition robin_hood.h:605
pair(std::tuple< U1... > &a, std::tuple< U2... > &b, ROBIN_HOOD_STD::index_sequence< I1... >, ROBIN_HOOD_STD::index_sequence< I2... >) noexcept(noexcept(T1(std::forward< U1 >(std::get< I1 >(std::declval< std::tuple< U1... > & >()))...)) &&noexcept(T2(std::forward< U2 >(std::get< I2 >(std::declval< std::tuple< U2... > & >()))...)))
Definition robin_hood.h:639
T2 second
Definition robin_hood.h:661
constexpr pair(T1 &&a, T2 &&b) noexcept(noexcept(T1(std::move(std::declval< T1 && >()))) &&noexcept(T2(std::move(std::declval< T2 && >()))))
Definition robin_hood.h:610
T1 first
Definition robin_hood.h:660
constexpr pair(U1 &&a, U2 &&b) noexcept(noexcept(T1(std::forward< U1 >(std::declval< U1 && >()))) &&noexcept(T2(std::forward< U2 >(std::declval< U2 && >()))))
Definition robin_hood.h:616
T2 second_type
Definition robin_hood.h:589
T1 first_type
Definition robin_hood.h:588
constexpr pair(std::pair< T1, T2 > const &o) noexcept(noexcept(T1(std::declval< T1 const & >())) &&noexcept(T2(std::declval< T2 const & >())))
Definition robin_hood.h:599
constexpr pair() noexcept(noexcept(U1()) &&noexcept(U2()))
Definition robin_hood.h:594