Skip to content

Commit ead8510

Browse files
Charlie FraschCharlie Frasch
Charlie Frasch
authored and
Charlie Frasch
committed
WIP
* added Fifo4b, Fifo5a * cleaned up all the fifos * correctly retest full or empty on cached cursor reload * refactored bench to add warmup and doNotOptimize
1 parent 17e8c63 commit ead8510

13 files changed

+545
-83
lines changed

‎CMakeLists.txt

+2
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,9 @@ add_fifo(fifo2)
4141
add_fifo(fifo3)
4242
add_fifo(fifo4)
4343
add_fifo(fifo4a)
44+
add_fifo(fifo4b)
4445
add_fifo(fifo5)
46+
add_fifo(fifo5a)
4547
add_fifo(boost_lockfree)
4648
add_fifo(rigtorp)
4749
target_compile_options(rigtorp PRIVATE -Wno-interference-size)

‎Fifo3.hpp

+7-4
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ class Fifo3 : private Alloc
2323

2424
~Fifo3() {
2525
while(not empty()) {
26-
ring_[popCursor_ % capacity_].~T();
26+
element(popCursor_)->~T();
2727
++popCursor_;
2828
}
2929
allocator_traits::deallocate(*this, ring_, capacity_);
@@ -57,7 +57,7 @@ class Fifo3 : private Alloc
5757
if (full(pushCursor, popCursor)) {
5858
return false;
5959
}
60-
new (&ring_[pushCursor % capacity_]) T(value);
60+
new (element(pushCursor)) T(value);
6161
pushCursor_.store(pushCursor + 1, std::memory_order_release);
6262
return true;
6363
}
@@ -70,8 +70,8 @@ class Fifo3 : private Alloc
7070
if (empty(pushCursor, popCursor)) {
7171
return false;
7272
}
73-
value = ring_[popCursor % capacity_];
74-
ring_[popCursor % capacity_].~T();
73+
value = *element(popCursor);
74+
element(popCursor)->~T();
7575
popCursor_.store(popCursor + 1, std::memory_order_release);
7676
return true;
7777
}
@@ -83,6 +83,9 @@ class Fifo3 : private Alloc
8383
static auto empty(size_type pushCursor, size_type popCursor) noexcept {
8484
return pushCursor == popCursor;
8585
}
86+
auto element(size_type cursor) noexcept {
87+
return &ring_[cursor % capacity_];
88+
}
8689

8790
private:
8891
size_type capacity_;

‎Fifo4.hpp

+13-10
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ class Fifo4 : private Alloc
2323

2424
~Fifo4() {
2525
while(not empty()) {
26-
ring_[popCursor_ % capacity_].~T();
26+
element(popCursor_)->~T();
2727
++popCursor_;
2828
}
2929
allocator_traits::deallocate(*this, ring_, capacity_);
@@ -54,12 +54,12 @@ class Fifo4 : private Alloc
5454
auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
5555
if (full(pushCursor, popCursorCached_)) {
5656
popCursorCached_ = popCursor_.load(std::memory_order_acquire);
57-
}
58-
if (full(pushCursor, popCursorCached_)) {
59-
return false;
57+
if (full(pushCursor, popCursorCached_)) {
58+
return false;
59+
}
6060
}
6161

62-
new (&ring_[pushCursor % capacity_]) T(value);
62+
new (element(pushCursor)) T(value);
6363
pushCursor_.store(pushCursor + 1, std::memory_order_release);
6464
return true;
6565
}
@@ -70,13 +70,13 @@ class Fifo4 : private Alloc
7070
auto popCursor = popCursor_.load(std::memory_order_relaxed);
7171
if (empty(pushCursorCached_, popCursor)) {
7272
pushCursorCached_ = pushCursor_.load(std::memory_order_acquire);
73-
}
74-
if (empty(pushCursorCached_, popCursor)) {
75-
return false;
73+
if (empty(pushCursorCached_, popCursor)) {
74+
return false;
75+
}
7676
}
7777

78-
value = ring_[popCursor % capacity_];
79-
ring_[popCursor % capacity_].~T();
78+
value = *element(popCursor);
79+
element(popCursor)->~T();
8080
popCursor_.store(popCursor + 1, std::memory_order_release);
8181
return true;
8282
}
@@ -88,6 +88,9 @@ class Fifo4 : private Alloc
8888
static auto empty(size_type pushCursor, size_type popCursor) noexcept {
8989
return pushCursor == popCursor;
9090
}
91+
auto element(size_type cursor) noexcept {
92+
return &ring_[cursor % capacity_];
93+
}
9194

9295
private:
9396
size_type capacity_;

‎Fifo4a.hpp

+14-11
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ class Fifo4a : private Alloc
2323

2424
~Fifo4a() {
2525
while(not empty()) {
26-
ring_[popCursor_ & mask_].~T();
26+
element(popCursor_)->~T();
2727
++popCursor_;
2828
}
2929
allocator_traits::deallocate(*this, ring_, capacity());
@@ -41,7 +41,7 @@ class Fifo4a : private Alloc
4141
/// Returns whether the container has no elements
4242
auto empty() const noexcept { return size() == 0; }
4343

44-
/// Returns whether the container has capacity_() elements
44+
/// Returns whether the container has capacity() elements
4545
auto full() const noexcept { return size() == capacity(); }
4646

4747
/// Returns the number of elements that can be held in the fifo
@@ -54,12 +54,12 @@ class Fifo4a : private Alloc
5454
auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
5555
if (full(pushCursor, popCursorCached_)) {
5656
popCursorCached_ = popCursor_.load(std::memory_order_acquire);
57-
}
58-
if (full(pushCursor, popCursorCached_)) {
59-
return false;
57+
if (full(pushCursor, popCursorCached_)) {
58+
return false;
59+
}
6060
}
6161

62-
new (&ring_[pushCursor & mask_]) T(value);
62+
new (element(pushCursor)) T(value);
6363
pushCursor_.store(pushCursor + 1, std::memory_order_release);
6464
return true;
6565
}
@@ -70,13 +70,13 @@ class Fifo4a : private Alloc
7070
auto popCursor = popCursor_.load(std::memory_order_relaxed);
7171
if (empty(pushCursorCached_, popCursor)) {
7272
pushCursorCached_ = pushCursor_.load(std::memory_order_acquire);
73-
}
74-
if (empty(pushCursorCached_, popCursor)) {
75-
return false;
73+
if (empty(pushCursorCached_, popCursor)) {
74+
return false;
75+
}
7676
}
7777

78-
value = ring_[popCursor & mask_];
79-
ring_[popCursor & mask_].~T();
78+
value = *element(popCursor);
79+
element(popCursor)->~T();
8080
popCursor_.store(popCursor + 1, std::memory_order_release);
8181
return true;
8282
}
@@ -88,6 +88,9 @@ class Fifo4a : private Alloc
8888
static auto empty(size_type pushCursor, size_type popCursor) noexcept {
8989
return pushCursor == popCursor;
9090
}
91+
auto element(size_type cursor) noexcept {
92+
return &ring_[cursor & mask_];
93+
}
9194

9295
private:
9396
size_type mask_;

‎Fifo4b.hpp

+126
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
#pragma once
2+
3+
#include <atomic>
4+
#include <cassert>
5+
#include <memory>
6+
#include <new>
7+
8+
9+
/// Threadsafe, efficient circular FIFO with cached cursors; constrained cursors
10+
template<typename T, typename Alloc = std::allocator<T>>
11+
class Fifo4b : private Alloc
12+
{
13+
public:
14+
using value_type = T;
15+
using allocator_traits = std::allocator_traits<Alloc>;
16+
using size_type = typename allocator_traits::size_type;
17+
18+
explicit Fifo4b(size_type capacity, Alloc const& alloc = Alloc{})
19+
: Alloc{alloc}
20+
, capacity_{capacity + 1}
21+
, ring_{allocator_traits::allocate(*this, capacity)}
22+
{}
23+
24+
~Fifo4b() {
25+
// TODO fix shouldn't matter for benchmark since it waits until
26+
// the fifo is empty
27+
// while(not empty()) {
28+
// ring_[popCursor_ & mask_].~T();
29+
// ++popCursor_;
30+
// }
31+
allocator_traits::deallocate(*this, ring_, capacity());
32+
}
33+
34+
/// Returns the number of elements in the fifo
35+
auto size() const noexcept {
36+
auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
37+
auto popCursor = popCursor_.load(std::memory_order_relaxed);
38+
39+
assert(popCursor <= pushCursor);
40+
return pushCursor - popCursor;
41+
}
42+
43+
/// Returns whether the container has no elements
44+
auto empty() const noexcept { return size() == 0; }
45+
46+
/// Returns whether the container has capacity() elements
47+
auto full() const noexcept { return size() == capacity(); }
48+
49+
/// Returns the number of elements that can be held in the fifo
50+
auto capacity() const noexcept { return capacity_ - 1; }
51+
52+
53+
/// Push one object onto the fifo.
54+
/// @return `true` if the operation is successful; `false` if fifo is full.
55+
auto push(T const& value) {
56+
auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
57+
auto nextPushCursor = pushCursor + 1;
58+
if (nextPushCursor == capacity_) {
59+
nextPushCursor = 0;
60+
}
61+
if (nextPushCursor == popCursorCached_) {
62+
popCursorCached_ = popCursor_.load(std::memory_order_acquire);
63+
if (nextPushCursor == popCursorCached_) {
64+
return false;
65+
}
66+
}
67+
68+
new (&ring_[pushCursor]) T(value);
69+
pushCursor_.store(nextPushCursor, std::memory_order_release);
70+
return true;
71+
}
72+
73+
/// Pop one object from the fifo.
74+
/// @return `true` if the pop operation is successful; `false` if fifo is empty.
75+
auto pop(T& value) {
76+
auto popCursor = popCursor_.load(std::memory_order_relaxed);
77+
if (popCursor == pushCursorCached_) {
78+
pushCursorCached_ = pushCursor_.load(std::memory_order_acquire);
79+
if (pushCursorCached_ == popCursor) {
80+
return false;
81+
}
82+
}
83+
84+
value = ring_[popCursor];
85+
ring_[popCursor].~T();
86+
auto nextPopCursor = popCursor + 1;
87+
if (nextPopCursor == capacity_) {
88+
nextPopCursor = 0;
89+
}
90+
popCursor_.store(nextPopCursor, std::memory_order_release);
91+
return true;
92+
}
93+
94+
private:
95+
auto full(size_type pushCursor, size_type popCursor) const noexcept {
96+
return (pushCursor - popCursor) == capacity();
97+
}
98+
static auto empty(size_type pushCursor, size_type popCursor) noexcept {
99+
return pushCursor == popCursor;
100+
}
101+
102+
private:
103+
size_type capacity_;
104+
T* ring_;
105+
106+
using CursorType = std::atomic<size_type>;
107+
static_assert(CursorType::is_always_lock_free);
108+
109+
// See Fifo3 for reason std::hardware_destructive_interference_size is not used directly
110+
static constexpr auto hardware_destructive_interference_size = size_type{64};
111+
112+
/// Loaded and stored by the push thread; loaded by the pop thread
113+
alignas(hardware_destructive_interference_size) CursorType pushCursor_;
114+
115+
/// Exclusive to the push thread
116+
alignas(hardware_destructive_interference_size) size_type popCursorCached_{};
117+
118+
/// Loaded and stored by the pop thread; loaded by the push thread
119+
alignas(hardware_destructive_interference_size) CursorType popCursor_;
120+
121+
/// Exclusive to the pop thread
122+
alignas(hardware_destructive_interference_size) size_type pushCursorCached_{};
123+
124+
// Padding to avoid false sharing with adjacent objects
125+
char padding_[hardware_destructive_interference_size - sizeof(size_type)];
126+
};

‎Fifo5.hpp

+13-14
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,9 @@ class Fifo5 : private Alloc
3030

3131
explicit Fifo5(size_type capacity, Alloc const& alloc = Alloc{})
3232
: Alloc{alloc}
33-
, mask_{capacity - 1}
34-
, ring_{allocator_traits::allocate(*this, capacity)} {
35-
assert((capacity & mask_) == 0);
36-
}
33+
, capacity_{capacity}
34+
, ring_{allocator_traits::allocate(*this, capacity)}
35+
{}
3736

3837
~Fifo5() {
3938
allocator_traits::deallocate(*this, ring_, capacity());
@@ -56,7 +55,7 @@ class Fifo5 : private Alloc
5655
auto full() const noexcept { return size() == capacity(); }
5756

5857
/// Returns the number of elements that can be held in the fifo
59-
auto capacity() const noexcept { return mask_ + 1; }
58+
auto capacity() const noexcept { return capacity_; }
6059

6160

6261
/// An RAII proxy object returned by push(). Allows the caller to
@@ -129,9 +128,9 @@ class Fifo5 : private Alloc
129128
auto pushCursor = pushCursor_.load(std::memory_order_relaxed);
130129
if (full(pushCursor, popCursorCached_)) {
131130
popCursorCached_ = popCursor_.load(std::memory_order_acquire);
132-
}
133-
if (full(pushCursor, popCursorCached_)) {
134-
return pusher_t{};
131+
if (full(pushCursor, popCursorCached_)) {
132+
return pusher_t{};
133+
}
135134
}
136135
return pusher_t(this, pushCursor);
137136
}
@@ -207,9 +206,9 @@ class Fifo5 : private Alloc
207206
auto popCursor = popCursor_.load(std::memory_order_relaxed);
208207
if (empty(pushCursorCached_, popCursor)) {
209208
pushCursorCached_ = pushCursor_.load(std::memory_order_acquire);
210-
}
211-
if (empty(pushCursorCached_, popCursor)) {
212-
return popper_t{};
209+
if (empty(pushCursorCached_, popCursor)) {
210+
return popper_t{};
211+
}
213212
}
214213
return popper_t(this, popCursor);
215214
};
@@ -234,11 +233,11 @@ class Fifo5 : private Alloc
234233
return pushCursor == popCursor;
235234
}
236235

237-
auto* element(size_type cursor) noexcept { return &ring_[cursor & mask_]; }
238-
auto const* element(size_type cursor) const noexcept { return &ring_[cursor & mask_]; }
236+
auto* element(size_type cursor) noexcept { return &ring_[cursor % capacity_]; }
237+
auto const* element(size_type cursor) const noexcept { return &ring_[cursor % capacity_]; }
239238

240239
private:
241-
size_type mask_;
240+
size_type capacity_;
242241
T* ring_;
243242

244243
using CursorType = std::atomic<size_type>;

0 commit comments

Comments
 (0)