src/main.cpp

   1 #include "all_benchmarks.h"
   2 #include "harness.h"
   3 #include "json.h"
   4 #include <charconv>
   5 #include <cstdint>
   6 #include <cstdlib>
   7 #include <functional>
   8 #include <initializer_list>
   9 #include <iostream>
  10 #include <limits>
  11 #include <map>
  12 #include <optional>
  13 #include <ostream>
  14 #include <sstream>
  15 #include <string_view>
  16 #include <system_error>
  17 #include <type_traits>
  18 #include <unordered_set>
  19 #include <vector>
  20
  21 using namespace std::literals;
  22
  23 enum class OptionValueKind
  24 {
  25     None,
  26     Required,
  27 };
  28
  29 class OptionsParser;
  30
  31 struct Option final
  32 {
  33     char short_name = '\0';
  34     std::string_view long_name = "", description = "";
  35     bool required = false;
  36     bool all_other_args_not_required = false;
  37     OptionValueKind value_kind = OptionValueKind::None;
  38     std::function<void(OptionsParser &parser,
  39                        std::optional<std::string_view> value)>
  40         parse_value;
  41     bool has_short_name() const
  42     {
  43         return short_name != '\0';
  44     }
  45     bool has_long_name() const
  46     {
  47         return !long_name.empty();
  48     }
  49     friend std::ostream &operator<<(std::ostream &os, const Option &option)
  50     {
  51         if (option.has_long_name())
  52         {
  53             os << "--" << option.long_name;
  54         }
  55         else if (option.has_short_name())
  56         {
  57             os << "-" << option.short_name;
  58         }
  59         else
  60         {
  61             os << "--<unnamed>";
  62         }
  63         return os;
  64     }
  65 };
  66
  67 class Options final
  68 {
  69     friend class OptionsParser;
  70
  71   private:
  72     std::vector<Option> options;
  73     std::map<char, std::size_t> short_options;
  74     std::map<std::string_view, std::size_t> long_options;
  75
  76   public:
  77     Options(std::initializer_list<Option> options_)
  78         : options(options_), short_options(), long_options()
  79     {
  80         for (std::size_t i = 0; i < options.size(); i++)
  81         {
  82             auto &option = options[i];
  83             if (option.has_short_name())
  84                 short_options[option.short_name] = i;
  85             if (option.has_long_name())
  86                 long_options[option.long_name] = i;
  87         }
  88     }
  89
  90   public:
  91     std::vector<std::string_view> parse(const char *const *argv) const;
  92 };
  93
  94 class OptionsParser final
  95 {
  96   private:
  97     const Options &options;
  98     const char *const *argv;
  99     std::string_view argv0;
 100     std::vector<bool> seen_options;
 101     bool check_required = true;
 102     std::optional<std::size_t> current_option_index;
 103
 104   private:
 105     const Option &current_option()
 106     {
 107         static const Option default_option{};
 108         if (current_option_index)
 109         {
 110             return options.options[*current_option_index];
 111         }
 112         else
 113         {
 114             return default_option;
 115         }
 116     }
 117
 118   public:
 119     OptionsParser(const Options &options, const char *const *argv)
 120         : options(options), argv(argv + 1),
 121           argv0(argv[0] ? argv[0] : "<none>"),
 122           seen_options(options.options.size(), false), current_option_index()
 123     {
 124         if (!argv[0])
 125         {
 126             static const char *null[] = {nullptr};
 127             this->argv = null;
 128             help_and_exit("missing program name");
 129         }
 130     }
 131
 132   private:
 133     void parse_option(std::size_t option_index, std::string_view prefix,
 134                       std::string_view name,
 135                       std::optional<std::string_view> value)
 136     {
 137         auto &option = options.options[option_index];
 138         switch (option.value_kind)
 139         {
 140         case OptionValueKind::None:
 141             if (value)
 142             {
 143                 help_and_exit("value not allowed for ", prefix, name);
 144             }
 145             break;
 146         case OptionValueKind::Required:
 147             if (!value)
 148             {
 149                 if (*argv)
 150                 {
 151                     value = *argv++;
 152                 }
 153                 else
 154                 {
 155                     help_and_exit("missing value for ", prefix, name);
 156                 }
 157             }
 158             break;
 159         }
 160         seen_options[option_index] = true;
 161         if (option.all_other_args_not_required)
 162             check_required = false;
 163         current_option_index = option_index;
 164         option.parse_value(*this, value);
 165         current_option_index = std::nullopt;
 166     }
 167
 168   public:
 169     std::vector<std::string_view> parse()
 170     {
 171         std::vector<std::string_view> retval;
 172         constexpr auto npos = std::string_view::npos;
 173         while (*argv)
 174         {
 175             std::string_view arg = *argv++;
 176             static constexpr std::string_view long_prefix = "--"sv;
 177             static constexpr std::string_view short_prefix = "-"sv;
 178             if (arg.rfind(long_prefix, 0) == 0) // it starts with `--`
 179             {
 180                 arg.remove_prefix(long_prefix.size());
 181                 auto eq = arg.find('=');
 182                 if (eq == 0)
 183                 {
 184                     eq = npos;
 185                 }
 186                 if (arg.empty()) // just `--`
 187                 {
 188                     while (*argv)
 189                     {
 190                         retval.push_back(*argv++);
 191                     }
 192                     break;
 193                 }
 194                 auto name = arg.substr(0, eq);
 195                 std::optional<std::string_view> value;
 196                 if (eq != npos)
 197                 {
 198                     value = arg.substr(eq + 1);
 199                 }
 200                 auto iter = options.long_options.find(name);
 201                 if (iter == options.long_options.end())
 202                 {
 203                     help_and_exit("unknown option: ", long_prefix, name);
 204                 }
 205                 auto option_index = iter->second;
 206                 parse_option(option_index, long_prefix, name, value);
 207                 continue;
 208             }
 209             else if (arg.rfind(short_prefix, 0) == 0) // it starts with `-`
 210             {
 211                 arg.remove_prefix(short_prefix.size());
 212                 if (arg.empty()) // just `-`
 213                 {
 214                     retval.push_back(short_prefix);
 215                     continue;
 216                 }
 217                 while (!arg.empty())
 218                 {
 219                     auto name = arg.substr(0, 1);
 220                     arg.remove_prefix(1);
 221                     auto iter = options.short_options.find(name[0]);
 222                     if (iter == options.short_options.end())
 223                     {
 224                         help_and_exit("unknown option: ", short_prefix, name);
 225                     }
 226                     auto option_index = iter->second;
 227                     std::optional<std::string_view> value;
 228                     switch (options.options[option_index].value_kind)
 229                     {
 230                     case OptionValueKind::None:
 231                         break;
 232                     case OptionValueKind::Required:
 233                         auto eq = arg.rfind('=', 0);
 234                         if (eq != npos)
 235                         {
 236                             value = arg.substr(eq + 1);
 237                         }
 238                         else if (!arg.empty())
 239                         {
 240                             value = arg;
 241                         }
 242                         arg = "";
 243                         break;
 244                     }
 245                     parse_option(option_index, short_prefix, name, value);
 246                 }
 247                 continue;
 248             }
 249             else
 250             {
 251                 retval.push_back(arg);
 252             }
 253         }
 254         if (check_required)
 255         {
 256             for (std::size_t i = 0; i < options.options.size(); i++)
 257             {
 258                 auto &option = options.options[i];
 259                 if (option.required && !seen_options[i])
 260                 {
 261                     help_and_exit("missing required option ", option);
 262                 }
 263             }
 264         }
 265         return retval;
 266     }
 267     template <typename... Options>
 268     [[noreturn]] void help_and_exit(const Options &...error_msg) const
 269     {
 270         auto &os = sizeof...(error_msg) == 0 ? std::cout : std::cerr;
 271         if (sizeof...(error_msg) != 0)
 272         {
 273             ((os << "Error: ") << ... << error_msg) << std::endl;
 274         }
 275         os << "Usage: " << argv0;
 276         for (auto &option : options.options)
 277         {
 278             os << " ";
 279             if (!option.required)
 280             {
 281                 os << "[";
 282             }
 283             auto sep = "";
 284             if (option.has_short_name())
 285             {
 286                 os << "-" << option.short_name;
 287                 sep = "|";
 288             }
 289             if (option.has_long_name())
 290             {
 291                 os << sep << "--" << option.long_name;
 292             }
 293             switch (option.value_kind)
 294             {
 295             case OptionValueKind::None:
 296                 break;
 297             // TODO: case OptionValueKind::Optional:
 298             case OptionValueKind::Required:
 299                 os << " <value>";
 300                 break;
 301             }
 302             if (!option.required)
 303             {
 304                 os << "]";
 305             }
 306         }
 307         os << std::endl << "Options:" << std::endl;
 308         for (auto &option : options.options)
 309         {
 310             auto sep = "";
 311             if (option.has_short_name())
 312             {
 313                 os << "-" << option.short_name;
 314                 sep = "|";
 315             }
 316             if (option.has_long_name())
 317             {
 318                 os << sep << "--" << option.long_name;
 319             }
 320             os << "   " << option.description << std::endl;
 321         }
 322         std::exit(sizeof...(error_msg) == 0 ? 0 : 1);
 323     }
 324     template <typename Int, typename = void> struct ParseIntLimits;
 325     template <typename Int>
 326     struct ParseIntLimits<Int, std::enable_if_t<std::is_integral_v<Int>>> final
 327     {
 328         Int min_value = std::numeric_limits<Int>::min();
 329         Int max_value = std::numeric_limits<Int>::max();
 330     };
 331     template <typename Int>
 332     std::enable_if_t<std::is_integral_v<Int>, void> parse_int(
 333         std::optional<std::string_view> value, Int &i_value,
 334         ParseIntLimits<Int> limits = {})
 335     {
 336         i_value = Int();
 337         if (!value)
 338         {
 339             help_and_exit("missing value for ", current_option());
 340         }
 341         auto str = *value;
 342         int base = 10;
 343         if (0 == str.rfind("0x", 0) || 0 == str.rfind("0X", 0))
 344         {
 345             base = 16;
 346             str.remove_prefix(2);
 347         }
 348         else if (0 == str.rfind("0o", 0) || 0 == str.rfind("0O", 0))
 349         {
 350             base = 8;
 351             str.remove_prefix(2);
 352         }
 353         else if (0 == str.rfind("0b", 0) || 0 == str.rfind("0B", 0))
 354         {
 355             base = 2;
 356             str.remove_prefix(2);
 357         }
 358         std::from_chars_result result = std::from_chars(
 359             str.data(), str.data() + str.size(), i_value, base);
 360         if (result.ptr != str.data() + str.size())
 361         {
 362             result.ec = std::errc::invalid_argument;
 363         }
 364         if (result.ec == std::errc())
 365         {
 366             if (i_value < limits.min_value || i_value > limits.max_value)
 367             {
 368                 result.ec = std::errc::result_out_of_range;
 369             }
 370         }
 371         if (result.ec == std::errc::result_out_of_range)
 372         {
 373             help_and_exit("value out of range: ", current_option(), "=",
 374                           *value);
 375         }
 376         else if (result.ec != std::errc())
 377         {
 378             help_and_exit("invalid value for: ", current_option());
 379         }
 380     }
 381     template <typename Int>
 382     std::enable_if_t<std::is_integral_v<Int>, void> parse_int(
 383         std::optional<std::string_view> value, std::optional<Int> &i_value,
 384         bool required = true, ParseIntLimits<Int> limits = {})
 385     {
 386         if (!required && !value)
 387         {
 388             i_value = std::nullopt;
 389             return;
 390         }
 391         i_value.emplace();
 392         this->parse_int(value, i_value.value(), limits);
 393     }
 394     template <typename Float>
 395     std::enable_if_t<std::is_floating_point_v<Float>, void> parse_float(
 396         std::optional<std::string_view> value, Float &f_value)
 397     {
 398         f_value = Float();
 399         if (!value)
 400         {
 401             help_and_exit("missing value for ", current_option());
 402         }
 403         auto str = *value;
 404         std::istringstream is{std::string(str)};
 405         if (!(is >> f_value))
 406         {
 407             help_and_exit("invalid value for: ", current_option());
 408         }
 409     }
 410     template <typename Float>
 411     std::enable_if_t<std::is_floating_point_v<Float>, void> parse_float(
 412         std::optional<std::string_view> value, std::optional<Float> &f_value,
 413         bool required = true)
 414     {
 415         if (!required && !value)
 416         {
 417             f_value = std::nullopt;
 418             return;
 419         }
 420         f_value.emplace();
 421         this->parse_float(value, f_value.value());
 422     }
 423 };
 424
 425 inline std::vector<std::string_view> Options::parse(
 426     const char *const *argv) const
 427 {
 428     return OptionsParser(*this, argv).parse();
 429 }
 430
 431 int main(int, char **argv)
 432 {
 433     Config config{};
 434     std::optional<std::unordered_set<std::string>> enabled_benchmarks;
 435     bool json_pretty = false;
 436     Options options{
 437         Option{
 438             .short_name = 'h',
 439             .long_name = "help",
 440             .description = "Display usage and exit.",
 441             .all_other_args_not_required = true,
 442             .parse_value = [](OptionsParser &parser,
 443                               auto) { parser.help_and_exit(); },
 444         },
 445         Option{.short_name = 'j',
 446                .long_name = "thread-count",
 447                .description = "Number of threads to run on",
 448                .value_kind = OptionValueKind::Required,
 449                .parse_value =
 450                    [&](OptionsParser &parser, auto value) {
 451                        parser.parse_int(value, config.thread_count);
 452                    }},
 453         Option{.short_name = 'n',
 454                .long_name = "iter-count",
 455                .description = "Number of iterations to run per thread",
 456                .value_kind = OptionValueKind::Required,
 457                .parse_value =
 458                    [&](OptionsParser &parser, auto value) {
 459                        parser.parse_int(value, config.iteration_count, true,
 460                                         {.min_value = 1});
 461                    }},
 462         Option{.long_name = "log2-mem-loc-count",
 463                .description =
 464                    "Log base 2 of the number of memory locations to access",
 465                .value_kind = OptionValueKind::Required,
 466                .parse_value =
 467                    [&](OptionsParser &parser, auto value) {
 468                        parser.parse_int(
 469                            value, config.log2_memory_location_count,
 470                            {.max_value =
 471                                 Config::max_sum_log2_mem_loc_count_and_stride -
 472                                 config.log2_stride});
 473                    }},
 474         Option{
 475             .long_name = "log2-stride",
 476             .description =
 477                 "Log base 2 of the stride used for accessing memory locations",
 478             .value_kind = OptionValueKind::Required,
 479             .parse_value =
 480                 [&](OptionsParser &parser, auto value) {
 481                     parser.parse_int(
 482                         value, config.log2_stride,
 483                         {.max_value =
 484                              Config::max_sum_log2_mem_loc_count_and_stride -
 485                              config.log2_memory_location_count});
 486                 }},
 487         Option{
 488             .short_name = 'b',
 489             .long_name = "bench",
 490             .description = "List of benchmarks that should be run",
 491             .value_kind = OptionValueKind::Required,
 492             .parse_value =
 493                 [&](OptionsParser &, std::optional<std::string_view> value) {
 494                     if (!enabled_benchmarks)
 495                     {
 496                         enabled_benchmarks.emplace();
 497                     }
 498                     enabled_benchmarks->emplace(value.value_or(""));
 499                 }},
 500         Option{.long_name = "json",
 501                .description = "Write the output in JSON format",
 502                .value_kind = OptionValueKind::None,
 503                .parse_value = [&](auto &, auto) { config.use_json = true; }},
 504         Option{.long_name = "json-pretty",
 505                .description = "Write the output in pretty JSON format",
 506                .value_kind = OptionValueKind::None,
 507                .parse_value =
 508                    [&](auto &, auto) {
 509                        config.use_json = true;
 510                        json_pretty = true;
 511                    }},
 512         Option{.short_name = 'd',
 513                .long_name = "target-duration",
 514                .description =
 515                    "target duration for a single benchmark in seconds",
 516                .value_kind = OptionValueKind::Required,
 517                .parse_value =
 518                    [&](OptionsParser &parser, auto value) {
 519                        parser.parse_float(value, config.target_duration);
 520                        if (config.target_duration &&
 521                            (!std::isfinite(*config.target_duration) ||
 522                             *config.target_duration < 0))
 523                        {
 524                            parser.help_and_exit(
 525                                "value out of range: target-duration=",
 526                                *config.target_duration);
 527                        }
 528                    }},
 529     };
 530     OptionsParser parser(options, argv);
 531     auto args = parser.parse();
 532     if (!args.empty())
 533     {
 534         parser.help_and_exit("unexpected argument");
 535     }
 536     auto benchmarks = LazyVec(all_benchmarks(config));
 537     if (enabled_benchmarks)
 538     {
 539         enabled_benchmarks->erase("");
 540         enabled_benchmarks->erase("help");
 541         enabled_benchmarks->erase("list");
 542         if (enabled_benchmarks->empty())
 543         {
 544             if (config.use_json)
 545             {
 546                 std::cout << JsonValue(benchmarks.map<JsonValue>(
 547                                            [](const Benchmark &benchmark) {
 548                                                return benchmark.name();
 549                                            }))
 550                                  .pretty(json_pretty)
 551                           << std::endl;
 552             }
 553             else
 554             {
 555                 std::cout << "Available Benchmarks:\n";
 556                 for (auto &benchmark : benchmarks)
 557                 {
 558                     std::cout << benchmark.name() << "\n";
 559                 }
 560                 std::cout << std::endl;
 561             }
 562             return 0;
 563         }
 564         std::unordered_set<std::string> unknown_benchmarks =
 565             *enabled_benchmarks;
 566         for (auto &benchmark : benchmarks)
 567         {
 568             unknown_benchmarks.erase(benchmark.name());
 569         }
 570         if (!unknown_benchmarks.empty())
 571         {
 572             parser.help_and_exit(
 573                 "unknown benchmark: ", *unknown_benchmarks.begin(),
 574                 "\nrun with `--bench=list` to see all supported benchmarks.");
 575         }
 576     }
 577     auto thread_cache = BenchHarnessBase::get_thread_cache();
 578     auto benchmark_results =
 579         benchmarks.filter_map<std::shared_ptr<BenchmarkResult>>(
 580             [&](const Benchmark &benchmark)
 581                 -> std::optional<std::shared_ptr<BenchmarkResult>> {
 582                 if (enabled_benchmarks &&
 583                     !enabled_benchmarks->count(benchmark.name()))
 584                 {
 585                     return std::nullopt;
 586                 }
 587                 std::cout.flush();
 588                 return benchmark.run(config);
 589             });
 590     if (config.use_json)
 591     {
 592         std::cout << JsonValue(benchmark_results.map<JsonValue>(
 593                                    [](const std::shared_ptr<BenchmarkResult>
 594                                           &benchmark_results) -> JsonValue {
 595                                        return *benchmark_results;
 596                                    }))
 597                          .pretty(json_pretty);
 598     }
 599     else
 600     {
 601         for (const std::shared_ptr<BenchmarkResult> &benchmark_results :
 602              benchmark_results)
 603         {
 604             benchmark_results->print();
 605         }
 606     }
 607     std::cout << std::endl;
 608     return 0;
 609 }