Add Fifo5b

Author: Charlie Frasch

CMakeLists.txt

@@ -44,6 +44,7 @@ add_fifo(fifo4a)
 add_fifo(fifo4b)
 add_fifo(fifo5)
 add_fifo(fifo5a)
+add_fifo(fifo5b)
 add_fifo(boost_lockfree)
 add_fifo(rigtorp)
 target_compile_options(rigtorp PRIVATE -Wno-interference-size)

Fifo5.hpp

@@ -138,8 +138,7 @@ class Fifo5 : private Alloc
     /// Push one object onto the fifo.
     /// @return `true` if the operation is successful; `false` if fifo is full.
     auto push(T const& value) noexcept {
-        auto pusher = push();
-        if (pusher) {
+        if (auto pusher = push(); pusher) {
             pusher = value;
             return true;
         }
@@ -216,8 +215,7 @@ class Fifo5 : private Alloc
     /// Pop one object from the fifo.
     /// @return `true` if the pop operation is successful; `false` if fifo is empty.
     auto pop(T& value) noexcept {
-        auto popper = pop();
-        if (popper) {
+        if (auto popper = pop(); popper) {
             value = *popper;
             return true;
         }

Fifo5b.hpp

@@ -0,0 +1,259 @@
+#pragma once
+
+#include <atomic>
+#include <cassert>
+#include <cstring>
+#include <memory>
+#include <new>
+#include <type_traits>
+
+// For ValueSizeTraits
+#include "Fifo5.hpp"
+
+
+/// Like Fifo5a except uses atomic_ref
+template<typename T, typename Alloc = std::allocator<T>>
+    requires std::is_trivial_v<T>
+class Fifo5b : private Alloc
+{
+public:
+    using value_type = T;
+    using allocator_traits = std::allocator_traits<Alloc>;
+    using size_type = typename allocator_traits::size_type;
+
+    explicit Fifo5b(size_type capacity, Alloc const& alloc = Alloc{})
+        : Alloc{alloc}
+        , mask_{capacity - 1}
+        , ring_{allocator_traits::allocate(*this, capacity)} {
+        assert((capacity & mask_) == 0);
+    }
+
+    ~Fifo5b() {
+        allocator_traits::deallocate(*this, ring_, capacity());
+    }
+
+
+    /// Returns the number of elements in the fifo
+    auto size() const noexcept {
+        auto pushCursor = pushCursorRef_.load(std::memory_order_relaxed);
+        auto popCursor = popCursorRef_.load(std::memory_order_relaxed);
+
+        assert(popCursor <= pushCursor);
+        return pushCursor - popCursor;
+    }
+
+    /// Returns whether the container has no elements
+    auto empty() const noexcept { return size() == 0; }
+
+    /// Returns whether the container has capacity_() elements
+    auto full() const noexcept { return size() == capacity(); }
+
+    /// Returns the number of elements that can be held in the fifo
+    auto capacity() const noexcept { return mask_ + 1; }
+
+
+    /// An RAII proxy object returned by push(). Allows the caller to
+    /// manipulate value_type's members directly in the fifo's ring. The
+    /// actual push happens when the pusher goes out of scope.
+    class pusher_t
+    {
+    public:
+        pusher_t() = default;
+        explicit pusher_t(Fifo5b* fifo, size_type cursor) noexcept : fifo_{fifo}, cursor_{cursor} {}
+
+        pusher_t(pusher_t const&) = delete;
+        pusher_t& operator=(pusher_t const&) = delete;
+
+        pusher_t(pusher_t&& other) noexcept
+            : fifo_{std::move(other.fifo_)}
+            , cursor_{std::move(other.cursor_)} {
+            other.release();
+        }
+        pusher_t& operator=(pusher_t&& other) noexcept {
+            fifo_ = std::move(other.fifo_);
+            cursor_ = std::move(other.cursor_);
+            other.release();
+            return *this;
+        }
+
+        ~pusher_t() {
+            if (fifo_) {
+                fifo_->pushCursorRef_.store(cursor_ + 1, std::memory_order_release);
+            }
+        }
+
+        /// If called the actual push operation will not be called when the
+        /// pusher_t goes out of scope. Operations on the pusher_t instance
+        /// after release has been called are undefined.
+        void release() noexcept { fifo_ = {}; }
+
+        /// Return whether or not the pusher_t is active.
+        explicit operator bool() const noexcept { return fifo_; }
+
+        /// @name Direct access to the fifo's ring
+        ///@{
+        value_type* get() noexcept { return fifo_->element(cursor_); }
+        value_type const* get() const noexcept { return fifo_->element(cursor_); }
+
+        value_type& operator*() noexcept { return *get(); }
+        value_type const& operator*() const noexcept { return *get(); }
+
+        value_type* operator->() noexcept { return get(); }
+        value_type const* operator->() const noexcept { return get(); }
+        ///@}
+
+        /// Copy-assign a `value_type` to the pusher. Prefer to use this
+        /// form rather than assigning directly to a value_type&. It takes
+        /// advantage of ValueSizeTraits.
+        pusher_t& operator=(value_type const& value) noexcept {
+            std::memcpy(get(), std::addressof(value), ValueSizeTraits<value_type>::size(value));
+            return *this;
+        }
+
+    private:
+        Fifo5b* fifo_{};
+        size_type cursor_;
+    };
+    friend class pusher_t;
+
+    /// Optionally push one object onto a file via a pusher.
+    /// @return a pointer to pusher_t.
+    pusher_t push() noexcept {
+        auto pushCursor = pushCursor_;
+        if (full(pushCursor, popCursorCached_)) {
+            // popCursorCached_ = popCursor_.load(std::memory_order_acquire);
+            popCursorCached_ = popCursorRef_.load(std::memory_order_acquire);
+            if (full(pushCursor, popCursorCached_)) {
+                return pusher_t{};
+            }
+        }
+        return pusher_t(this, pushCursor);
+    }
+
+    /// Push one object onto the fifo.
+    /// @return `true` if the operation is successful; `false` if fifo is full.
+    auto push(T const& value) noexcept {
+        if (auto pusher = push(); pusher) {
+            pusher = value;
+            return true;
+        }
+        return false;
+    }
+
+    /// An RAII proxy object returned by pop(). Allows the caller to
+    /// manipulate value_type members directly in the fifo's ring. The
+    // /actual pop happens when the popper goes out of scope.
+    class popper_t
+    {
+    public:
+        popper_t() = default;
+        explicit popper_t(Fifo5b* fifo, size_type cursor) noexcept : fifo_{fifo}, cursor_{cursor} {}
+
+        popper_t(popper_t const&) = delete;
+        popper_t& operator=(popper_t const&) = delete;
+
+        popper_t(popper_t&& other) noexcept
+            : fifo_{std::move(other.fifo_)}
+            , cursor_{std::move(other.cursor_)} {
+            other.release();
+        }
+        popper_t& operator=(popper_t&& other) noexcept {
+            fifo_ = std::move(other.fifo_);
+            cursor_ = std::move(other.cursor_);
+            other.release();
+            return *this;
+        }
+
+        ~popper_t() {
+            if (fifo_) {
+                fifo_->popCursorRef_.store(cursor_ + 1, std::memory_order_release);
+            }
+        }
+
+        /// If called the actual pop operation will not be called when the
+        /// popper_t goes out of scope. Operations on the popper_t instance
+        /// after release has been called are undefined.
+        void release() noexcept { fifo_ = {}; }
+
+        /// Return whether or not the popper_t is active.
+        explicit operator bool() const noexcept { return fifo_; }
+
+        /// @name Direct access to the fifo's ring
+        ///@{
+        value_type* get() noexcept { return fifo_->element(cursor_); }
+        value_type const* get() const noexcept { return fifo_->element(cursor_); }
+
+        value_type& operator*() noexcept { return *get(); }
+        value_type const& operator*() const noexcept { return *get(); }
+
+        value_type* operator->() noexcept { return get(); }
+        value_type const* operator->() const noexcept { return get(); }
+        ///@}
+
+    private:
+        Fifo5b* fifo_{};
+        size_type cursor_;
+    };
+    friend popper_t;
+
+    auto pop() noexcept {
+        auto popCursor = popCursor_;
+        if (empty(pushCursorCached_, popCursor)) {
+            pushCursorCached_ = pushCursorRef_.load(std::memory_order_acquire);
+            if (empty(pushCursorCached_, popCursor)) {
+                return popper_t{};
+            }
+        }
+        return popper_t(this, popCursor);
+    };
+
+    /// Pop one object from the fifo.
+    /// @return `true` if the pop operation is successful; `false` if fifo is empty.
+    auto pop(T& value) noexcept {
+        if (auto popper = pop(); popper) {
+            value = *popper;
+            return true;
+        }
+        return false;
+    }
+
+private:
+    auto full(size_type pushCursor, size_type popCursor) const noexcept {
+        assert(popCursor <= pushCursor);
+        return (pushCursor - popCursor) == capacity();
+    }
+    static auto empty(size_type pushCursor, size_type popCursor) noexcept {
+        return pushCursor == popCursor;
+    }
+
+    auto* element(size_type cursor) noexcept { return &ring_[cursor & mask_]; }
+    auto const* element(size_type cursor) const noexcept { return &ring_[cursor & mask_]; }
+
+private:
+    size_type mask_;
+    T* ring_;
+
+    using CursorType = size_type;
+    using CursorRefType = std::atomic_ref<CursorType>;
+
+    // https://stackoverflow.com/questions/39680206/understanding-stdhardware-destructive-interference-size-and-stdhardware-cons
+    // See Fifo3.hpp for reason why std::hardware_destructive_interference_size is not used directly
+    static constexpr auto hardware_destructive_interference_size = size_type{64};
+
+    /// Loaded and stored by the push thread; loaded by the pop thread
+    alignas(hardware_destructive_interference_size) CursorType pushCursor_{};
+    CursorRefType pushCursorRef_{pushCursor_};
+
+    /// Exclusive to the push thread
+    alignas(hardware_destructive_interference_size) size_type popCursorCached_{};
+
+    /// Loaded and stored by the pop thread; loaded by the push thread
+    alignas(hardware_destructive_interference_size) CursorType popCursor_{};
+    CursorRefType popCursorRef_{popCursor_};
+
+    /// Exclusive to the pop thread
+    alignas(hardware_destructive_interference_size) size_type pushCursorCached_{};
+
+    // Padding to avoid false sharing with adjacent objects
+    char padding_[hardware_destructive_interference_size - sizeof(size_type)];
+};

bench.cpp

@@ -1,6 +1,8 @@
 #include "Fifo4.hpp"
 #include "Fifo4a.hpp"
 #include "Fifo5.hpp"
+#include "Fifo5a.hpp"
+#include "Fifo5b.hpp"
 #include "rigtorp.hpp"
 
 #include <benchmark/benchmark.h>
@@ -95,6 +97,8 @@ void BM_Fifo(benchmark::State& state) {
 BENCHMARK_TEMPLATE(BM_Fifo, Fifo4);
 BENCHMARK_TEMPLATE(BM_Fifo, Fifo4a);
 BENCHMARK_TEMPLATE(BM_Fifo, Fifo5);
+BENCHMARK_TEMPLATE(BM_Fifo, Fifo5a);
+BENCHMARK_TEMPLATE(BM_Fifo, Fifo5b);
 BENCHMARK_TEMPLATE(BM_Fifo, rigtorp::SPSCQueue);
 
 BENCHMARK_MAIN();

bench_all.cpp

@@ -5,6 +5,7 @@
 #include "Fifo4b.hpp"
 #include "Fifo5.hpp"
 #include "Fifo5a.hpp"
+#include "Fifo5b.hpp"
 #include "Mutex.hpp"
 #include "rigtorp.hpp"
 #include <boost/lockfree/spsc_queue.hpp>   // boost 1.74.0
@@ -35,6 +36,7 @@ void once(long iters, int cpu1, int cpu2) {
         Bench<Fifo4b<ValueT>>{}(iters, cpu1, cpu2) << "," << std::flush <<
         Bench<Fifo5<ValueT>>{}(iters, cpu1, cpu2) << "," << std::flush <<
         Bench<Fifo5a<ValueT>>{}(iters, cpu1, cpu2) << "," << std::flush <<
+        Bench<Fifo5b<ValueT>>{}(iters, cpu1, cpu2) << "," << std::flush <<
         Bench<rigtorp::SPSCQueue<ValueT>>{}(iters, cpu1, cpu2) << "," << std::flush <<
         Bench<boost_spsc_queue<ValueT>>{}(iters, cpu1, cpu2) << std::flush <<
         "\n";
@@ -52,7 +54,7 @@ int main(int argc, char* argv[]) {
 
     using value_type = std::int64_t;
 
-    std::cout << "Fifo3,Fifo4,Fifo4a,Fifo4b,Fifo5,Fifo5a,rigtorp,boost_spsc_queue" << std::endl;
+    std::cout << "Fifo3,Fifo4,Fifo4a,Fifo4b,Fifo5,Fifo5a,Fifo5b,rigtorp,boost_spsc_queue" << std::endl;
     // std::cout << "Fifo2,Mutex\n";
     for (auto rep = 0; rep < reps; ++rep) {
         once<value_type>(iters, cpu1, cpu2);

fifo5b.cpp

@@ -0,0 +1,7 @@
+#include "Fifo5b.hpp"
+#include "bench.hpp"
+
+int main(int argc, char* argv[]) {
+    bench<Fifo5b>("Fifo5b", argc, argv);
+}
+

run_all.sh

@@ -1,6 +1,6 @@
 #! /usr/bin/bash
 
-for bench in fifo2 fifo3 fifo4 fifo4a fifo4b fifo5 fifo5a rigtorp boost_lockfree mutex;
+for bench in fifo2 fifo3 fifo4 fifo4a fifo4b fifo5 fifo5a fifo5b rigtorp boost_lockfree mutex;
 do
 ./build/release/$bench 1 2
 done

unitTests.cpp

@@ -3,6 +3,8 @@
 #include "Fifo3.hpp"
 #include "Fifo4.hpp"
 #include "Fifo5.hpp"
+#include "Fifo5a.hpp"
+#include "Fifo5b.hpp"
 
 #include <gtest/gtest.h>
 
@@ -40,7 +42,8 @@ using FifoTypes = ::testing::Types<
     Fifo2<test_type>,
     Fifo3<test_type>,
     Fifo4<test_type>,
-    Fifo5<test_type>
+    Fifo5<test_type>,
+    Fifo5b<test_type>
     >;
 TYPED_TEST_SUITE(FifoTest, FifoTypes);