{"source": "/**\n * \\file\n * \\license\n * SPDX-FileType: SOURCE\n * SPDX-FileCopyrightText: 2026 Edward Kmett <ekmett@gmail.com>\n * SPDX-License-Identifier: BSD-2-Clause OR Apache-2.0\n * \\endlicense\n */\n\n// Standalone public prefix helpers and synthetic benchmark; no Everett includes.\n// Generic x86-64 compile: -O3 -std=c++20 -mpopcnt (do not use -march=native).\n#include <immintrin.h>\n#include <cpuid.h>\n#include <algorithm>\n#include <array>\n#include <bit>\n#include <chrono>\n#include <cstdint>\n#include <cstdio>\n#include <cstdlib>\n#include <cstring>\n#include <stdexcept>\n#include <vector>\n\n__attribute__((target(\"popcnt\"), noinline))\nunsigned prefix_scalar(std::uint64_t const * words, unsigned bits) {\n  unsigned result = 0;\n  for (unsigned word = 0; word < (bits >> 6); ++word)\n    result += unsigned(std::popcount(words[word]));\n  if (bits & 63)\n    result += unsigned(std::popcount(words[bits >> 6] & ((std::uint64_t{1} << (bits & 63)) - 1)));\n  return result;\n}\n\ntemplate <unsigned Vector> __attribute__((target(\"avx2\"), always_inline)) inline __m256i prefix512_masked_avx2(\n    std::uint64_t const * words, unsigned bits) noexcept {\n  auto positions = _mm256_setr_epi64x(4 * Vector, 4 * Vector + 1, 4 * Vector + 2, 4 * Vector + 3);\n  auto boundary = _mm256_set1_epi64x(bits >> 6);\n  auto tail = _mm256_set1_epi64x(static_cast<long long>((std::uint64_t{1} << (bits & 63)) - 1));\n  auto mask = _mm256_or_si256(_mm256_cmpgt_epi64(boundary, positions),\n    _mm256_and_si256(_mm256_cmpeq_epi64(positions, boundary), tail));\n  return _mm256_and_si256(_mm256_loadu_si256(reinterpret_cast<__m256i const *>(words + 4 * Vector)), mask);\n}\n\n// Exactly eight readable words, with no vector alignment requirement.\n__attribute__((target(\"avx2\"), noinline)) unsigned prefix512_avx2(std::uint64_t const * words, unsigned bits) noexcept {\n  auto lookup = _mm256_setr_epi8(0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,\n                                0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4);\n  auto nibble = _mm256_set1_epi8(15);\n  auto population = [&](auto data) __attribute__((target(\"avx2\"), always_inline)) {\n    auto lo = _mm256_shuffle_epi8(lookup, _mm256_and_si256(data, nibble));\n    auto hi = _mm256_shuffle_epi8(lookup, _mm256_and_si256(_mm256_srli_epi16(data, 4), nibble));\n    return _mm256_add_epi8(lo, hi);\n  };\n  // Each byte sum is at most sixteen, so both vectors can share one SAD.\n  auto counts = _mm256_add_epi8(population(prefix512_masked_avx2<0>(words, bits)),\n                                population(prefix512_masked_avx2<1>(words, bits)));\n  auto sums = _mm256_sad_epu8(counts, _mm256_setzero_si256());\n  auto pair = _mm_add_epi64(_mm256_castsi256_si128(sums), _mm256_extracti128_si256(sums, 1));\n  return unsigned(_mm_cvtsi128_si32(_mm_add_epi64(pair, _mm_srli_si128(pair, 8))));\n}\n\n__attribute__((target(\"avx512f\"), always_inline)) inline __m512i prefix512_masked_avx512(std::uint64_t const * words, unsigned bits) noexcept {\n  auto lane = bits >> 6;\n  auto full = __mmask8((1u << lane) - 1);\n  auto boundary = __mmask8(1u << lane); // lane=8 gives no boundary lane.\n  auto data = _mm512_maskz_loadu_epi64(full | boundary, words);\n  auto tail = _mm512_set1_epi64(static_cast<long long>((std::uint64_t{1} << (bits & 63)) - 1));\n  return _mm512_mask_and_epi64(data, boundary, data, tail);\n}\n\n__attribute__((target(\"avx512f,avx512vpopcntdq\"), noinline)) unsigned prefix512_avx512_vpopcnt(std::uint64_t const * words, unsigned bits) noexcept {\n  return unsigned(_mm512_reduce_add_epi64(_mm512_popcnt_epi64(prefix512_masked_avx512(words, bits))));\n}\n\n__attribute__((target(\"avx512f,avx512bw\"), noinline)) unsigned prefix512_avx512bw(std::uint64_t const * words, unsigned bits) noexcept {\n  auto data = prefix512_masked_avx512(words, bits);\n  auto lookup = _mm512_broadcast_i32x4(_mm_setr_epi8(0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4));\n  auto nibble = _mm512_set1_epi8(15);\n  auto lo = _mm512_shuffle_epi8(lookup, _mm512_and_si512(data, nibble));\n  auto hi = _mm512_shuffle_epi8(lookup, _mm512_and_si512(_mm512_srli_epi16(data, 4), nibble));\n  auto counts = _mm512_add_epi8(lo, hi);\n  return unsigned(_mm512_reduce_add_epi64(_mm512_sad_epu8(counts, _mm512_setzero_si512())));\n}\n\n\nusing prefix_function = unsigned (*)(std::uint64_t const *, unsigned);\nstruct variant { char const * name; prefix_function prefix; };\nstd::uint64_t random_word(std::uint64_t & state) {\n  state += 0x9e3779b97f4a7c15ull;\n  auto x = state;\n  x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9ull;\n  x = (x ^ (x >> 27)) * 0x94d049bb133111ebull;\n  return x ^ (x >> 31);\n}\n\nint main(int argc, char ** argv) try {\n  __builtin_cpu_init();\n  char brand[49]{};\n  for(unsigned i=0;i<3;++i){unsigned a,b,c,d;__get_cpuid(0x80000002u+i,&a,&b,&c,&d);unsigned x[]{a,b,c,d};std::memcpy(brand+16*i,x,16);}\n  std::printf(\"# cpu=%s\\n\",brand);\n  bool popcnt = __builtin_cpu_supports(\"popcnt\");\n  bool avx2 = __builtin_cpu_supports(\"avx2\");\n  bool avx512f = __builtin_cpu_supports(\"avx512f\");\n  bool avx512bw = avx512f && __builtin_cpu_supports(\"avx512bw\");\n  bool vpopcnt = avx512f && __builtin_cpu_supports(\"avx512vpopcntdq\");\n  std::printf(\"# popcnt=%d avx2=%d avx512bw=%d avx512vpopcntdq=%d\\n\", popcnt, avx2, avx512bw, vpopcnt);\n  if (!popcnt) throw std::runtime_error(\"POPCNT baseline is unavailable\");\n  auto trials = argc > 1 ? unsigned(std::strtoul(argv[1], nullptr, 10)) : 5u;\n  auto queries = argc > 2 ? std::strtoull(argv[2], nullptr, 10) : 1048576ull;\n  if (!trials || !queries) throw std::runtime_error(\"positive trials and queries required\");\n  std::vector<variant> variants{{\"scalar_popcnt\", prefix_scalar}};\n  if (avx2) variants.push_back({\"avx2\", prefix512_avx2});\n  if (avx512bw) variants.push_back({\"avx512bw\", prefix512_avx512bw});\n  if (vpopcnt) variants.push_back({\"avx512vpopcntdq\", prefix512_avx512_vpopcnt});\n  // Independent oracle: every prefix, including 0 and512, at eight uint64\n  // offsets, for zero/all-one/random data. No vector reduction builds it.\n  alignas(64) std::array<std::uint64_t, 15> input{};\n  std::uint64_t seed = 0x512f00d;\n  for (unsigned pattern = 0; pattern < 66; ++pattern) {\n    for (auto & word : input) word = pattern == 0 ? 0 : pattern == 1 ? ~std::uint64_t{0} : random_word(seed);\n    for (unsigned offset = 0; offset < 8; ++offset) {\n      unsigned expected = 0;\n      for (unsigned bits = 0; bits <= 512; ++bits) {\n        for (auto const & v : variants)\n          if (v.prefix(input.data()+offset, bits) != expected) throw std::runtime_error(\"prefix oracle mismatch\");\n        if (bits != 512) expected += unsigned((input[offset+(bits>>6)] >> (bits&63))&1);\n      }\n    }\n  }\n  constexpr unsigned blocks = 512;\n  std::vector<std::uint64_t> words(blocks*8), positions(65536);\n  std::vector<std::uint16_t> oracle(blocks*513);\n  for (auto & word : words) word = random_word(seed);\n  for (unsigned block = 0; block < blocks; ++block)\n    for (unsigned bit = 0; bit < 512; ++bit)\n      oracle[block*513+bit+1] = oracle[block*513+bit] + unsigned((words[block*8+(bit>>6)] >> (bit&63))&1);\n  for (auto & q : positions) q = random_word(seed);\n  std::printf(\"# prefix oracle passed; bitmap_bytes=%zu query_bytes=%zu\\n\",words.size()*8,positions.size()*8);\n  std::puts(\"pattern,variant,trial,queries,ns_per_prefix,checksum\");\n  for (bool dependent : {false,true}) {\n    auto position = [&](std::uint64_t i, std::uint64_t previous) {\n      auto q=positions[i&65535] ^ (dependent ? previous*0x9e3779b97f4a7c15ull : 0);\n      return std::pair{unsigned((q>>10)&(blocks-1)),unsigned(q%513)};\n    };\n    std::uint64_t expected_checksum=0, previous=0;\n    for (std::uint64_t i=0;i<queries;++i) {\n      auto [block,bits]=position(i,previous);\n      previous=oracle[block*513+bits]; expected_checksum+=previous;\n      for (auto const & v:variants)\n        if (v.prefix(words.data()+block*8,bits)!=previous) throw std::runtime_error(\"generated query oracle mismatch\");\n    }\n    auto run = [&](variant const & v, std::uint64_t count) {\n      std::uint64_t sum=0,last=0;\n      auto start=std::chrono::steady_clock::now();\n      for (std::uint64_t i=0;i<count;++i) {\n        auto [block,bits]=position(i,last);\n        last=v.prefix(words.data()+block*8,bits); sum+=last;\n      }\n      auto ns=std::chrono::duration<double,std::nano>(std::chrono::steady_clock::now()-start).count()/count;\n      return std::pair{ns,sum};\n    };\n    std::uint64_t warmed=0;\n    for (auto const & v:variants) warmed^=run(v,std::min<std::uint64_t>(queries,32768)).second;\n    for (unsigned trial=0;trial<trials;++trial)\n      for (unsigned slot=0;slot<variants.size();++slot) {\n        auto const & v=variants[(slot+trial)%variants.size()];\n        auto [ns,sum]=run(v,queries);\n        if(sum!=expected_checksum) throw std::runtime_error(\"timed checksum mismatch\");\n        std::printf(\"%s,%s,%u,%llu,%.6f,%llu\\n\",dependent?\"dependent\":\"independent\",v.name,trial,\n          static_cast<unsigned long long>(queries),ns,static_cast<unsigned long long>(sum));\n      }\n    std::printf(\"# warmed=%llu\\n\",static_cast<unsigned long long>(warmed));\n  }\n  return 0;\n} catch (std::exception const & e) { std::fprintf(stderr,\"%s\\n\",e.what()); return 1; }\n\n/**\n * \\file\n * \\author Edward Kmett <ekmett@gmail.com>\n * \\brief Standalone CPUID-gated Intel prefix benchmark with synthetic inputs.\n */\n", "compiler": "clang2010", "options": {"userArguments": "-O3 -std=c++20 -mpopcnt", "compilerOptions": {"executorRequest": true}, "filters": {"execute": true}, "executeParameters": {"args": ["5", "1048576"], "stdin": ""}}, "lang": "c++", "bypassCache": 2}