include/simplev_cpp.h

   1 // SPDX-License-Identifier: LGPL-2.1-or-later
   2 // See Notices.txt for copyright information
   3 #pragma once
   4
   5 #include <bits/c++config.h>
   6 #include <sys/cdefs.h>
   7 #ifndef __cplusplus
   8 #error to use SimpleV Cpp with C, include "simplev_c.h"
   9 #endif
  10
  11 #include <cstddef>
  12 #include <cstdint>
  13 #include <type_traits>
  14
  15 // we need to use the register keyword as part of assigning particular variables to registers for
  16 // inline assembly
  17 #define DECLARE_ASM_REG(type, name, reg, value)                                       \
  18     _Pragma("GCC diagnostic push")                                                    \
  19         _Pragma("GCC diagnostic ignored \"-Wregister\"") register type name asm(reg); \
  20     _Pragma("GCC diagnostic pop") name = value;
  21
  22 namespace sv
  23 {
  24 template <typename ElementType, std::size_t SUB_VL, std::size_t MAX_VL, typename = void>
  25 struct VecTypeStruct;
  26
  27 template <typename ElementType, std::size_t SUB_VL, std::size_t MAX_VL>
  28 using VecType = typename VecTypeStruct<ElementType, SUB_VL, MAX_VL>::Type;
  29
  30
  31 #define MAKE_VEC_TYPE(size)                                                                 \
  32     template <typename ElementType, std::size_t SUB_VL, std::size_t MAX_VL>                 \
  33     struct VecTypeStruct<ElementType,                                                       \
  34                          SUB_VL,                                                            \
  35                          MAX_VL,                                                            \
  36                          std::enable_if_t<sizeof(ElementType) * SUB_VL * MAX_VL == (size)>> \
  37         final                                                                               \
  38     {                                                                                       \
  39         typedef ElementType Type __attribute__((vector_size(size)));                        \
  40     };
  41
  42 MAKE_VEC_TYPE(1)
  43 MAKE_VEC_TYPE(2)
  44 MAKE_VEC_TYPE(3)
  45 MAKE_VEC_TYPE(4)
  46 MAKE_VEC_TYPE(5)
  47 MAKE_VEC_TYPE(6)
  48 MAKE_VEC_TYPE(7)
  49 MAKE_VEC_TYPE(8)
  50
  51 #undef MAKE_VEC_TYPE
  52
  53 template <typename ElementType, std::size_t SUB_VL, std::size_t MAX_VL>
  54 struct Vec final
  55 {
  56     static_assert(MAX_VL > 0 && MAX_VL <= 64);
  57     static_assert(SUB_VL >= 1 && SUB_VL <= 4);
  58     using Type = VecType<ElementType, SUB_VL, MAX_VL>;
  59     Type value;
  60 };
  61
  62 struct Mask final
  63 {
  64     std::uint64_t value = ~0ULL;
  65 };
  66
  67 template <size_t MAX_VL = 64>
  68 struct VL final
  69 {
  70     static_assert(MAX_VL > 0 && MAX_VL <= 64);
  71     std::size_t value = MAX_VL;
  72 };
  73
  74 inline constexpr std::size_t PRIMARY_OPCODE_SHIFT = 32 - 6;
  75 /// unofficial value. see https://libre-soc.org/openpower/sv/setvl/
  76 /// FIXME: incorrect extended opcode value
  77 inline constexpr std::uint32_t SETVL_OPCODE = (19 << PRIMARY_OPCODE_SHIFT) | (0 << 1);
  78 inline constexpr std::size_t SETVL_IMMEDIATE_SHIFT = 32 - 7 - 16;
  79 inline constexpr std::size_t REG_FIELD_WIDTH = 5;
  80 inline constexpr std::size_t XL_FORM_RT_SHIFT = 32 - REG_FIELD_WIDTH - 6;
  81 inline constexpr std::size_t XL_FORM_RA_SHIFT = 32 - REG_FIELD_WIDTH - 11;
  82
  83 constexpr std::uint32_t encode_sv_prefix(std::uint32_t remapped_bits24)
  84 {
  85     std::uint32_t expanded26bits = (remapped_bits24 & 0x3FFFFFUL)
  86                                    | ((remapped_bits24 & 0x400000UL) << 1)
  87                                    | ((remapped_bits24 & 0x800000UL) << 2);
  88     expanded26bits |= 0x01400000UL; // set 2 constant-1 bits
  89     return expanded26bits | (1UL << PRIMARY_OPCODE_SHIFT);
  90 }
  91
  92 enum class MaskMode : std::uint32_t
  93 {
  94     Int = 0,
  95     CR = 1,
  96 };
  97
  98 enum class MaskField : std::uint32_t
  99 {
 100     Always = 0,
 101     OneShlR3 = 1, // 1 << R3
 102     R3 = 2,
 103     NotR3 = 3,
 104     R10 = 4,
 105     NotR10 = 5,
 106     R30 = 6,
 107     NotR30 = 7,
 108
 109     Lt = 0,
 110     NL = 1,
 111     Gt = 2,
 112     NG = 3,
 113     Eq = 4,
 114     NE = 5,
 115     SO = 6,
 116     NS = 7,
 117 };
 118
 119 enum class Mode : std::uint32_t
 120 {
 121     Normal = 0,
 122     // TODO: fill out
 123 };
 124
 125 enum class ElementWidth : std::uint32_t
 126 {
 127     I8 = 3,
 128     I16 = 2,
 129     I32 = 1,
 130     I64 = 0,
 131     Default = 0,
 132     F64 = 0,
 133     F32 = 1,
 134     F16 = 2,
 135     BF16 = 3,
 136 };
 137
 138 template <typename ElementType,
 139           std::size_t ELEMENT_SIZE = sizeof(ElementType),
 140           bool IS_INTEGRAL = std::is_integral_v<ElementType>>
 141 struct ElementProperties;
 142
 143 template <typename ElementType>
 144 struct ElementProperties<ElementType, 1, true> final
 145 {
 146     static inline constexpr ElementWidth element_width = ElementWidth::I8;
 147 };
 148
 149 template <typename ElementType>
 150 struct ElementProperties<ElementType, 2, true> final
 151 {
 152     static inline constexpr ElementWidth element_width = ElementWidth::I16;
 153 };
 154
 155 template <typename ElementType>
 156 struct ElementProperties<ElementType, 4, true> final
 157 {
 158     static inline constexpr ElementWidth element_width = ElementWidth::I32;
 159 };
 160
 161 template <typename ElementType>
 162 struct ElementProperties<ElementType, 8, true> final
 163 {
 164     static inline constexpr ElementWidth element_width = ElementWidth::I64;
 165 };
 166
 167 template <>
 168 struct ElementProperties<float, 4, false> final
 169 {
 170     static inline constexpr ElementWidth element_width = ElementWidth::F32;
 171 };
 172
 173 template <>
 174 struct ElementProperties<double, 8, false> final
 175 {
 176     static inline constexpr ElementWidth element_width = ElementWidth::F64;
 177 };
 178
 179 template <typename ElementType>
 180 inline constexpr ElementWidth element_width_for = ElementProperties<ElementType>::element_width;
 181
 182 template <std::size_t SUB_VL>
 183 constexpr std::uint32_t encode_sv_prefix(MaskMode mask_mode,
 184                                          MaskField mask_field,
 185                                          ElementWidth elwidth,
 186                                          ElementWidth elwidth_src,
 187                                          Mode mode,
 188                                          std::uint32_t remapped_bits24)
 189 {
 190     static_assert(SUB_VL >= 1 && SUB_VL <= 4);
 191     remapped_bits24 |= static_cast<std::uint32_t>(mask_mode) << (23 - 0);
 192     remapped_bits24 |= static_cast<std::uint32_t>(mask_field) << (23 - 3);
 193     remapped_bits24 |= static_cast<std::uint32_t>(elwidth) << (23 - 5);
 194     remapped_bits24 |= static_cast<std::uint32_t>(elwidth_src) << (23 - 7);
 195     remapped_bits24 |= static_cast<std::uint32_t>(SUB_VL - 1) << (23 - 9);
 196     remapped_bits24 |= static_cast<std::uint32_t>(mode);
 197     return remapped_bits24;
 198 }
 199
 200 enum class RegExtra2 : std::uint32_t
 201 {
 202     Scalar0 = 0,
 203     Scalar1 = 1,
 204     Vector0 = 2,
 205     Vector1 = 3,
 206 };
 207
 208 enum class RegExtra3 : std::uint32_t
 209 {
 210     Scalar0 = 0,
 211     Scalar1 = 1,
 212     Scalar2 = 2,
 213     Scalar3 = 3,
 214     Vector0 = 4,
 215     Vector1 = 5,
 216     Vector2 = 6,
 217     Vector3 = 7,
 218 };
 219
 220 template <std::size_t SUB_VL>
 221 constexpr std::uint32_t encode_sv_prefix_rm_1p_3s1d(MaskMode mask_mode,
 222                                                     MaskField mask_field,
 223                                                     ElementWidth elwidth,
 224                                                     ElementWidth elwidth_src,
 225                                                     Mode mode,
 226                                                     RegExtra2 rdest_extra2,
 227                                                     RegExtra2 rsrc1_extra2,
 228                                                     RegExtra2 rsrc2_extra2,
 229                                                     RegExtra2 rsrc3_extra2)
 230 {
 231     std::uint32_t remapped_bits24 = 0;
 232     remapped_bits24 |= static_cast<std::uint32_t>(rdest_extra2) << (23 - 11);
 233     remapped_bits24 |= static_cast<std::uint32_t>(rsrc1_extra2) << (23 - 13);
 234     remapped_bits24 |= static_cast<std::uint32_t>(rsrc2_extra2) << (23 - 15);
 235     remapped_bits24 |= static_cast<std::uint32_t>(rsrc3_extra2) << (23 - 17);
 236     return encode_sv_prefix<SUB_VL>(
 237         mask_mode, mask_field, elwidth, elwidth_src, mode, remapped_bits24);
 238 }
 239
 240 template <std::size_t SUB_VL>
 241 constexpr std::uint32_t encode_sv_prefix_rm_1p_2s1d(MaskMode mask_mode,
 242                                                     MaskField mask_field,
 243                                                     ElementWidth elwidth,
 244                                                     ElementWidth elwidth_src,
 245                                                     Mode mode,
 246                                                     RegExtra3 rdest_extra3,
 247                                                     RegExtra3 rsrc1_extra3,
 248                                                     RegExtra3 rsrc2_extra3)
 249 {
 250     std::uint32_t remapped_bits24 = 0;
 251     remapped_bits24 |= static_cast<std::uint32_t>(rdest_extra3) << (23 - 12);
 252     remapped_bits24 |= static_cast<std::uint32_t>(rsrc1_extra3) << (23 - 15);
 253     remapped_bits24 |= static_cast<std::uint32_t>(rsrc2_extra3) << (23 - 18);
 254     return encode_sv_prefix<SUB_VL>(
 255         mask_mode, mask_field, elwidth, elwidth_src, mode, remapped_bits24);
 256 }
 257
 258 #define SETVL_ASM(retval, vl)                                            \
 259     "# setvl " retval ", " vl                                            \
 260     ", MVL=%[max_vl]\n\t"                                                \
 261     ".long %[setvl_opcode] | (" retval " << %[xl_form_rt_shift]) | (" vl \
 262     " << %[xl_form_ra_shift]) | ((%[max_vl] - 1) << %[setvl_immediate_shift])"
 263
 264 #define SETVL_ASM_INPUT_ARGS()                                                                    \
 265     [setvl_opcode] "n"(sv::SETVL_OPCODE), [setvl_immediate_shift] "n"(sv::SETVL_IMMEDIATE_SHIFT), \
 266         [xl_form_rt_shift] "n"(sv::XL_FORM_RT_SHIFT), [xl_form_ra_shift] "n"(sv::XL_FORM_RA_SHIFT)
 267
 268 template <std::size_t MAX_VL>
 269 inline __attribute__((always_inline)) VL<MAX_VL> setvl(std::size_t vl)
 270 {
 271     VL<MAX_VL> retval;
 272     asm(SETVL_ASM("%[retval]", "%[vl]")
 273         : [retval] "=b"(retval.value)
 274         : [vl] "b"(vl), [max_vl] "n"(MAX_VL), SETVL_ASM_INPUT_ARGS());
 275     return retval;
 276 }
 277
 278 template <typename ElementType,
 279           std::size_t SUB_VL,
 280           std::size_t MAX_VL,
 281           typename = std::enable_if_t<std::is_integral_v<ElementType>>>
 282 inline __attribute__((always_inline)) Vec<ElementType, SUB_VL, MAX_VL> add(
 283     Vec<ElementType, SUB_VL, MAX_VL> ra,
 284     Vec<ElementType, SUB_VL, MAX_VL> rb,
 285     VL<MAX_VL> vl = VL<MAX_VL>(),
 286     Mask mask = Mask())
 287 {
 288     constexpr std::uint32_t prefix =
 289         encode_sv_prefix_rm_1p_2s1d<SUB_VL>(MaskMode::Int,
 290                                             MaskField::R10,
 291                                             element_width_for<ElementType>,
 292                                             element_width_for<ElementType>,
 293                                             Mode::Normal,
 294                                             RegExtra3::Vector0,
 295                                             RegExtra3::Vector0,
 296                                             RegExtra3::Vector0);
 297     Vec<ElementType, SUB_VL, MAX_VL> retval;
 298     DECLARE_ASM_REG(std::uint64_t, mask_r10, "r10", mask.value)
 299     asm(SETVL_ASM("0", "%[vl]") "\n\t"
 300         "# sv.add ew=%[el_width], subvl=%[sub_vl], m=%[mask], %[retval].v, %[ra].v, %[rb].v\n\t"
 301         ".long %[prefix]\n\t"
 302         "add %[retval], %[ra], %[rb]"
 303         : [retval] "=&b"(retval.value)
 304         : [vl] "b"(vl),
 305           [max_vl] "n"(MAX_VL),
 306           [sub_vl] "n"(SUB_VL),
 307           [mask] "b"(mask_r10),
 308           [ra] "b"(ra.value),
 309           [rb] "b"(rb.value),
 310           [el_width] "n"(8 * sizeof(ElementType)),
 311           [prefix] "n"(prefix),
 312           SETVL_ASM_INPUT_ARGS());
 313     return retval;
 314 }
 315
 316 template <typename ElementType,
 317           std::size_t SUB_VL,
 318           std::size_t MAX_VL,
 319           typename = std::enable_if_t<std::is_integral_v<ElementType>>>
 320 inline __attribute__((always_inline)) Vec<ElementType, SUB_VL, MAX_VL> sub(
 321     Vec<ElementType, SUB_VL, MAX_VL> rb, // intentionally reversed since we use sv.subf instruction
 322     Vec<ElementType, SUB_VL, MAX_VL> ra,
 323     VL<MAX_VL> vl = VL<MAX_VL>(),
 324     Mask mask = Mask())
 325 {
 326     constexpr std::uint32_t prefix =
 327         encode_sv_prefix_rm_1p_2s1d<SUB_VL>(MaskMode::Int,
 328                                             MaskField::R10,
 329                                             element_width_for<ElementType>,
 330                                             element_width_for<ElementType>,
 331                                             Mode::Normal,
 332                                             RegExtra3::Vector0,
 333                                             RegExtra3::Vector0,
 334                                             RegExtra3::Vector0);
 335     Vec<ElementType, SUB_VL, MAX_VL> retval;
 336     DECLARE_ASM_REG(std::uint64_t, mask_r10, "r10", mask.value)
 337     asm(SETVL_ASM("0", "%[vl]") "\n\t"
 338         "# sv.subf ew=%[el_width], subvl=%[sub_vl], m=%[mask], %[retval].v, %[ra].v, %[rb].v\n\t"
 339         ".long %[prefix]\n\t"
 340         "subf %[retval], %[ra], %[rb]"
 341         : [retval] "=&b"(retval.value)
 342         : [vl] "b"(vl),
 343           [max_vl] "n"(MAX_VL),
 344           [sub_vl] "n"(SUB_VL),
 345           [mask] "b"(mask_r10),
 346           [ra] "b"(ra.value),
 347           [rb] "b"(rb.value),
 348           [el_width] "n"(8 * sizeof(ElementType)),
 349           [prefix] "n"(prefix),
 350           SETVL_ASM_INPUT_ARGS());
 351     return retval;
 352 }
 353
 354 #undef SETVL_ASM
 355 #undef SETVL_ASM_INPUT_ARGS
 356 #undef DECLARE_ASM_REG
 357 } // namespace sv