src/common/c11_atomics/c11_atomics_benchmarks.cpp

   1 #include "c11_atomics_benchmarks.h"
   2 #include <atomic>
   3 #include <cstdint>
   4 #include <memory>
   5 #include <sstream>
   6 #include <string_view>
   7 #include <type_traits>
   8
   9 template <typename T> struct IntTypeName;
  10
  11 template <typename T>
  12 inline constexpr std::string_view int_type_name = IntTypeName<T>::name;
  13
  14 #define INT_TYPE_NAME(sz, ui, uint)                                           \
  15     template <> struct IntTypeName<std::uint##sz##_t> final                   \
  16     {                                                                         \
  17         static constexpr std::string_view name = #ui #sz;                     \
  18     };
  19
  20 INT_TYPE_NAME(8, u, uint)
  21 INT_TYPE_NAME(16, u, uint)
  22 INT_TYPE_NAME(32, u, uint)
  23 INT_TYPE_NAME(64, u, uint)
  24 INT_TYPE_NAME(8, i, int)
  25 INT_TYPE_NAME(16, i, int)
  26 INT_TYPE_NAME(32, i, int)
  27 INT_TYPE_NAME(64, i, int)
  28
  29 template <std::memory_order order> struct MemoryOrderName;
  30
  31 template <std::memory_order order>
  32 inline constexpr std::string_view memory_order_name =
  33     MemoryOrderName<order>::name;
  34
  35 #define MEMORY_ORDER_NAME(order)                                              \
  36     template <> struct MemoryOrderName<std::memory_order_##order> final       \
  37     {                                                                         \
  38         static constexpr std::string_view name = #order;                      \
  39     };
  40
  41 MEMORY_ORDER_NAME(relaxed)
  42 MEMORY_ORDER_NAME(acquire)
  43 MEMORY_ORDER_NAME(release)
  44 MEMORY_ORDER_NAME(acq_rel)
  45 MEMORY_ORDER_NAME(seq_cst)
  46
  47 template <typename T> using Buf = std::shared_ptr<std::vector<std::atomic<T>>>;
  48
  49 template <typename Fn, typename T, typename... NameParts>
  50 static void push_atomic_bench(std::vector<Benchmark> &benches,
  51                               const Config &config, Buf<T> buf, Fn fn,
  52                               NameParts &&...name_parts)
  53 {
  54     auto log2_stride = config.log2_stride;
  55     std::size_t index_mask = 1;
  56     index_mask <<= config.log2_stride;
  57     index_mask <<= config.log2_memory_location_count;
  58     index_mask--;
  59     push_bench(
  60         benches,
  61         [buf, fn, index_mask, log2_stride](T input, std::uint64_t iteration,
  62                                            std::uint32_t thread_num) {
  63             std::size_t index = iteration;
  64             index ^= static_cast<std::size_t>(thread_num) * 0x12345;
  65             index <<= log2_stride;
  66             index &= index_mask;
  67             std::atomic<T> *atomic = &(*buf)[index];
  68             input ^= static_cast<T>(iteration);
  69             return fn(input, atomic, iteration ^ thread_num);
  70         },
  71         T{}, name_parts...);
  72 }
  73
  74 template <typename T, std::memory_order order>
  75 static void rmw_benchmarks(std::vector<Benchmark> &benches,
  76                            const Config &config, Buf<T> buf)
  77 {
  78     push_atomic_bench(
  79         benches, config, buf,
  80         [](T input, std::atomic<T> *atomic, std::uint64_t) {
  81             return std::atomic_exchange_explicit(atomic, input, order);
  82         },
  83         "atomic_exchange_", int_type_name<T>, "_", memory_order_name<order>);
  84     push_atomic_bench(
  85         benches, config, buf,
  86         [](T input, std::atomic<T> *atomic, std::uint64_t) {
  87             return std::atomic_fetch_add_explicit(atomic, input, order);
  88         },
  89         "atomic_fetch_add_", int_type_name<T>, "_", memory_order_name<order>);
  90     push_atomic_bench(
  91         benches, config, buf,
  92         [](T input, std::atomic<T> *atomic, std::uint64_t) {
  93             return std::atomic_fetch_sub_explicit(atomic, input, order);
  94         },
  95         "atomic_fetch_sub_", int_type_name<T>, "_", memory_order_name<order>);
  96     push_atomic_bench(
  97         benches, config, buf,
  98         [](T input, std::atomic<T> *atomic, std::uint64_t) {
  99             return std::atomic_fetch_and_explicit(atomic, input, order);
 100         },
 101         "atomic_fetch_and_", int_type_name<T>, "_", memory_order_name<order>);
 102     push_atomic_bench(
 103         benches, config, buf,
 104         [](T input, std::atomic<T> *atomic, std::uint64_t) {
 105             return std::atomic_fetch_or_explicit(atomic, input, order);
 106         },
 107         "atomic_fetch_or_", int_type_name<T>, "_", memory_order_name<order>);
 108     push_atomic_bench(
 109         benches, config, buf,
 110         [](T input, std::atomic<T> *atomic, std::uint64_t) {
 111             return std::atomic_fetch_xor_explicit(atomic, input, order);
 112         },
 113         "atomic_fetch_xor_", int_type_name<T>, "_", memory_order_name<order>);
 114 }
 115
 116 template <typename T, std::memory_order order>
 117 static void load_benchmarks(std::vector<Benchmark> &benches,
 118                             const Config &config, Buf<T> buf)
 119 {
 120     push_atomic_bench(
 121         benches, config, buf,
 122         [](T, std::atomic<T> *atomic, std::uint64_t) {
 123             return std::atomic_load_explicit(atomic, order);
 124         },
 125         "atomic_load_", int_type_name<T>, "_", memory_order_name<order>);
 126 }
 127
 128 template <typename T, std::memory_order order>
 129 static void store_benchmarks(std::vector<Benchmark> &benches,
 130                              const Config &config, Buf<T> buf)
 131 {
 132     push_atomic_bench(
 133         benches, config, buf,
 134         [](T input, std::atomic<T> *atomic, std::uint64_t) {
 135             return std::atomic_store_explicit(atomic, input, order);
 136         },
 137         "atomic_store_", int_type_name<T>, "_", memory_order_name<order>);
 138 }
 139
 140 template <typename T, std::memory_order succ, std::memory_order fail>
 141 static void cmp_xchg_benchmarks(std::vector<Benchmark> &benches,
 142                                 const Config &config, Buf<T> buf)
 143 {
 144     push_atomic_bench(
 145         benches, config, buf,
 146         [](T input, std::atomic<T> *atomic, std::uint64_t state) {
 147             T expected = state;
 148             bool succeeded = std::atomic_compare_exchange_weak_explicit(
 149                 atomic, &expected, input, succ, fail);
 150             return std::pair(expected, succeeded);
 151         },
 152         "atomic_compare_exchange_weak_", int_type_name<T>, "_",
 153         memory_order_name<succ>, "_", memory_order_name<fail>);
 154     push_atomic_bench(
 155         benches, config, buf,
 156         [](T input, std::atomic<T> *atomic, std::uint64_t state) {
 157             T expected = state;
 158             bool succeeded = std::atomic_compare_exchange_strong_explicit(
 159                 atomic, &expected, input, succ, fail);
 160             return std::pair(expected, succeeded);
 161         },
 162         "atomic_compare_exchange_strong_", int_type_name<T>, "_",
 163         memory_order_name<succ>, "_", memory_order_name<fail>);
 164 }
 165
 166 template <typename T>
 167 static void benchmarks(std::vector<Benchmark> &benches, const Config &config)
 168 {
 169     std::size_t buf_size = 1;
 170     buf_size <<= config.log2_memory_location_count;
 171     buf_size <<= config.log2_stride;
 172     Buf<T> buf = std::make_shared<std::vector<std::atomic<T>>>(buf_size);
 173
 174     rmw_benchmarks<T, std::memory_order_relaxed>(benches, config, buf);
 175     rmw_benchmarks<T, std::memory_order_acquire>(benches, config, buf);
 176     rmw_benchmarks<T, std::memory_order_release>(benches, config, buf);
 177     rmw_benchmarks<T, std::memory_order_acq_rel>(benches, config, buf);
 178     rmw_benchmarks<T, std::memory_order_seq_cst>(benches, config, buf);
 179
 180     load_benchmarks<T, std::memory_order_relaxed>(benches, config, buf);
 181     load_benchmarks<T, std::memory_order_acquire>(benches, config, buf);
 182     load_benchmarks<T, std::memory_order_seq_cst>(benches, config, buf);
 183
 184     store_benchmarks<T, std::memory_order_relaxed>(benches, config, buf);
 185     store_benchmarks<T, std::memory_order_release>(benches, config, buf);
 186     store_benchmarks<T, std::memory_order_seq_cst>(benches, config, buf);
 187
 188     cmp_xchg_benchmarks<T, std::memory_order_relaxed,
 189                         std::memory_order_relaxed>(benches, config, buf);
 190
 191     cmp_xchg_benchmarks<T, std::memory_order_acquire,
 192                         std::memory_order_relaxed>(benches, config, buf);
 193     cmp_xchg_benchmarks<T, std::memory_order_acquire,
 194                         std::memory_order_acquire>(benches, config, buf);
 195
 196     cmp_xchg_benchmarks<T, std::memory_order_release,
 197                         std::memory_order_relaxed>(benches, config, buf);
 198
 199     cmp_xchg_benchmarks<T, std::memory_order_acq_rel,
 200                         std::memory_order_relaxed>(benches, config, buf);
 201     cmp_xchg_benchmarks<T, std::memory_order_acq_rel,
 202                         std::memory_order_acquire>(benches, config, buf);
 203
 204     cmp_xchg_benchmarks<T, std::memory_order_seq_cst,
 205                         std::memory_order_relaxed>(benches, config, buf);
 206     cmp_xchg_benchmarks<T, std::memory_order_seq_cst,
 207                         std::memory_order_acquire>(benches, config, buf);
 208     cmp_xchg_benchmarks<T, std::memory_order_seq_cst,
 209                         std::memory_order_seq_cst>(benches, config, buf);
 210 }
 211
 212 std::vector<Benchmark> c11_atomics_benchmarks(const Config &config)
 213 {
 214     std::vector<Benchmark> benches;
 215     benchmarks<std::uint8_t>(benches, config);
 216     benchmarks<std::uint16_t>(benches, config);
 217     benchmarks<std::uint32_t>(benches, config);
 218     benchmarks<std::uint64_t>(benches, config);
 219     benchmarks<std::int8_t>(benches, config);
 220     benchmarks<std::int16_t>(benches, config);
 221     benchmarks<std::int32_t>(benches, config);
 222     benchmarks<std::int64_t>(benches, config);
 223     return benches;
 224 }