{"source": "/**\n * \\file\n * \\license\n * SPDX-FileType: SOURCE\n * SPDX-FileCopyrightText: 2026 Edward Kmett <ekmett@gmail.com>\n * SPDX-License-Identifier: BSD-2-Clause OR Apache-2.0\n * \\endlicense\n */\n\n// Standalone public rank arithmetic and synthetic benchmark; no Everett includes.\n// Generic x86-64 compile: -O3 -std=c++20 -mpopcnt (do not use -march=native).\n#include <immintrin.h>\n#include <cpuid.h>\n#include <algorithm>\n#include <array>\n#include <bit>\n#include <chrono>\n#include <cstdint>\n#include <cstdio>\n#include <cstdlib>\n#include <cstring>\n#include <stdexcept>\n#include <vector>\n#include <span>\n\n__attribute__((target(\"popcnt\"), always_inline))\ninline unsigned prefix_scalar(std::uint64_t const * words, unsigned bits) {\n  unsigned result = 0;\n  for (unsigned word = 0; word < (bits >> 6); ++word)\n    result += unsigned(std::popcount(words[word]));\n  if (bits & 63)\n    result += unsigned(std::popcount(words[bits >> 6] & ((std::uint64_t{1} << (bits & 63)) - 1)));\n  return result;\n}\n\ntemplate <unsigned Vector> __attribute__((target(\"avx2\"), always_inline)) inline __m256i prefix512_masked_avx2(\n    std::uint64_t const * words, unsigned bits) noexcept {\n  auto positions = _mm256_setr_epi64x(4 * Vector, 4 * Vector + 1, 4 * Vector + 2, 4 * Vector + 3);\n  auto boundary = _mm256_set1_epi64x(bits >> 6);\n  auto tail = _mm256_set1_epi64x(static_cast<long long>((std::uint64_t{1} << (bits & 63)) - 1));\n  auto mask = _mm256_or_si256(_mm256_cmpgt_epi64(boundary, positions),\n    _mm256_and_si256(_mm256_cmpeq_epi64(positions, boundary), tail));\n  return _mm256_and_si256(_mm256_loadu_si256(reinterpret_cast<__m256i const *>(words + 4 * Vector)), mask);\n}\n\n// Exactly eight readable words, with no vector alignment requirement.\n__attribute__((target(\"avx2\"), always_inline)) inline unsigned prefix512_avx2(std::uint64_t const * words, unsigned bits) noexcept {\n  auto lookup = _mm256_setr_epi8(0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,\n                                0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4);\n  auto nibble = _mm256_set1_epi8(15);\n  auto population = [&](auto data) __attribute__((target(\"avx2\"), always_inline)) {\n    auto lo = _mm256_shuffle_epi8(lookup, _mm256_and_si256(data, nibble));\n    auto hi = _mm256_shuffle_epi8(lookup, _mm256_and_si256(_mm256_srli_epi16(data, 4), nibble));\n    return _mm256_add_epi8(lo, hi);\n  };\n  // Each byte sum is at most sixteen, so both vectors can share one SAD.\n  auto counts = _mm256_add_epi8(population(prefix512_masked_avx2<0>(words, bits)),\n                                population(prefix512_masked_avx2<1>(words, bits)));\n  auto sums = _mm256_sad_epu8(counts, _mm256_setzero_si256());\n  auto pair = _mm_add_epi64(_mm256_castsi256_si128(sums), _mm256_extracti128_si256(sums, 1));\n  return unsigned(_mm_cvtsi128_si32(_mm_add_epi64(pair, _mm_srli_si128(pair, 8))));\n}\n\n\n__attribute__((target(\"avx512f\"), always_inline)) inline __m512i prefix512_masked_avx512(std::uint64_t const * words, unsigned bits) noexcept {\n  auto lane = bits >> 6;\n  auto full = __mmask8((1u << lane) - 1);\n  auto boundary = __mmask8(1u << lane); // lane=8 gives no boundary lane.\n  auto data = _mm512_maskz_loadu_epi64(full | boundary, words);\n  auto tail = _mm512_set1_epi64(static_cast<long long>((std::uint64_t{1} << (bits & 63)) - 1));\n  return _mm512_mask_and_epi64(data, boundary, data, tail);\n}\n\n__attribute__((target(\"avx512f,avx512vpopcntdq\"), always_inline)) inline unsigned prefix512_avx512_vpopcnt(std::uint64_t const * words, unsigned bits) noexcept {\n  return unsigned(_mm512_reduce_add_epi64(_mm512_popcnt_epi64(prefix512_masked_avx512(words, bits))));\n}\n\n__attribute__((target(\"avx512f,avx512bw\"), always_inline)) inline unsigned prefix512_avx512bw(std::uint64_t const * words, unsigned bits) noexcept {\n  auto data = prefix512_masked_avx512(words, bits);\n  auto lookup = _mm512_broadcast_i32x4(_mm_setr_epi8(0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4));\n  auto nibble = _mm512_set1_epi8(15);\n  auto lo = _mm512_shuffle_epi8(lookup, _mm512_and_si512(data, nibble));\n  auto hi = _mm512_shuffle_epi8(lookup, _mm512_and_si512(_mm512_srli_epi16(data, 4), nibble));\n  auto counts = _mm512_add_epi8(lo, hi);\n  return unsigned(_mm512_reduce_add_epi64(_mm512_sad_epu8(counts, _mm512_setzero_si512())));\n}\n\n\n// Same eight-byte directory and query arithmetic as rank_view. Construction of\n// these test directories uses the independent bit-at-a-time oracle below.\nstruct rank_block { std::uint32_t before, runs; };\nstatic_assert(sizeof(rank_block) == 8);\nstruct rank_view {\n  std::span<std::uint64_t const> words;\n  std::span<rank_block const> blocks;\n  std::span<std::uint64_t const> supers;\n  std::uint64_t bit_count;\n};\nconstexpr unsigned run_prefix(std::uint32_t packed, unsigned run) noexcept {\n  std::uint64_t selected = packed & ((std::uint64_t{1} << (11 * run)) - 1);\n  return unsigned(((selected * 0x400801ull) >> 22) & 2047u);\n}\n__attribute__((target(\"popcnt\"), noinline))\nstd::uint64_t rank_scalar(rank_view const & view, std::uint64_t position) {\n  if (position >= view.bit_count) throw std::out_of_range(\"rank position\");\n  auto block = view.blocks[position >> 11];\n  unsigned run = unsigned((position >> 9) & 3);\n  std::uint64_t result = view.supers[position >> 32] + block.before;\n  result += run_prefix(block.runs, run);\n  auto word = (position >> 9) << 3;\n  auto bits = unsigned(position & 511);\n  if (!bits) return result;\n  return result + prefix_scalar(view.words.data() + word, bits);\n}\n__attribute__((target(\"avx2,popcnt\"), noinline))\nstd::uint64_t rank_avx2(rank_view const & view, std::uint64_t position) {\n  if (position >= view.bit_count) throw std::out_of_range(\"rank position\");\n  auto block = view.blocks[position >> 11];\n  unsigned run = unsigned((position >> 9) & 3);\n  std::uint64_t result = view.supers[position >> 32] + block.before;\n  result += run_prefix(block.runs, run);\n  auto word = (position >> 9) << 3;\n  auto bits = unsigned(position & 511);\n  if (!bits) return result;\n  if (view.words.size() - word >= 8)\n    return result + prefix512_avx2(view.words.data() + word, bits);\n  return result + prefix_scalar(view.words.data() + word, bits);\n}\n__attribute__((target(\"avx512f,avx512bw,popcnt\"), noinline))\nstd::uint64_t rank_avx512bw(rank_view const & view, std::uint64_t position) {\n  if (position >= view.bit_count) throw std::out_of_range(\"rank position\");\n  auto block = view.blocks[position >> 11];\n  unsigned run = unsigned((position >> 9) & 3);\n  std::uint64_t result = view.supers[position >> 32] + block.before;\n  result += run_prefix(block.runs, run);\n  auto word = (position >> 9) << 3;\n  auto bits = unsigned(position & 511);\n  if (!bits) return result;\n  if (view.words.size() - word >= 8)\n    return result + prefix512_avx512bw(view.words.data() + word, bits);\n  return result + prefix_scalar(view.words.data() + word, bits);\n}\n__attribute__((target(\"avx512f,avx512vpopcntdq,popcnt\"), noinline))\nstd::uint64_t rank_avx512_vpopcnt(rank_view const & view, std::uint64_t position) {\n  if (position >= view.bit_count) throw std::out_of_range(\"rank position\");\n  auto block = view.blocks[position >> 11];\n  unsigned run = unsigned((position >> 9) & 3);\n  std::uint64_t result = view.supers[position >> 32] + block.before;\n  result += run_prefix(block.runs, run);\n  auto word = (position >> 9) << 3;\n  auto bits = unsigned(position & 511);\n  if (!bits) return result;\n  if (view.words.size() - word >= 8)\n    return result + prefix512_avx512_vpopcnt(view.words.data() + word, bits);\n  return result + prefix_scalar(view.words.data() + word, bits);\n}\nusing rank_function = std::uint64_t (*)(rank_view const &, std::uint64_t);\nstruct variant { char const * name; rank_function rank; };\nstd::uint64_t random_word(std::uint64_t & state) {\n  state += 0x9e3779b97f4a7c15ull;\n  auto x = state;\n  x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9ull;\n  x = (x ^ (x >> 27)) * 0x94d049bb133111ebull;\n  return x ^ (x >> 31);\n}\nstruct fixture {\n  std::vector<std::uint64_t> words, oracle, supers;\n  std::vector<rank_block> blocks;\n  std::uint64_t bits;\n  fixture(std::uint64_t count, std::uint64_t & seed) : words((count+63)>>6), oracle(count+1),\n      supers((count+0xffffffffull)>>32), blocks((count+2047)>>11), bits(count) {\n    for (auto & word : words) word = random_word(seed);\n    for (std::uint64_t i = 0; i < bits; ++i)\n      oracle[i+1] = oracle[i] + ((words[i>>6] >> (i&63)) & 1);\n    for (std::uint64_t i = 0; i < supers.size(); ++i) supers[i] = oracle[i<<32];\n    for (std::uint64_t i = 0; i < blocks.size(); ++i) {\n      auto begin = i<<11;\n      blocks[i].before = std::uint32_t(oracle[begin] - oracle[(begin>>32)<<32]);\n      blocks[i].runs = 0;\n      for (unsigned r = 0; r < 3; ++r) {\n        auto population = oracle[std::min(bits,begin+(r+1)*512)] - oracle[std::min(bits,begin+r*512)];\n        blocks[i].runs |= std::uint32_t(population) << (11*r);\n      }\n    }\n  }\n  rank_view view() const { return {words,blocks,supers,bits}; }\n};\nint main(int argc, char ** argv) try {\n  __builtin_cpu_init();\n  bool popcnt = __builtin_cpu_supports(\"popcnt\"), avx2 = __builtin_cpu_supports(\"avx2\");\n  bool avx512f = __builtin_cpu_supports(\"avx512f\");\n  bool avx512bw = avx512f && __builtin_cpu_supports(\"avx512bw\");\n  bool vpopcnt = avx512f && __builtin_cpu_supports(\"avx512vpopcntdq\");\n  char brand[49]{};\n  for (unsigned i=0; i<3; ++i) {\n    unsigned a=0,b=0,c=0,d=0;\n    __get_cpuid(0x80000002u+i,&a,&b,&c,&d);\n    unsigned x[]{a,b,c,d}; std::memcpy(brand+16*i,x,16);\n  }\n  std::printf(\"# cpu=%s\\n# popcnt=%d avx2=%d avx512bw=%d avx512vpopcntdq=%d\\n\",brand,popcnt,avx2,avx512bw,vpopcnt);\n  if (!popcnt) throw std::runtime_error(\"POPCNT baseline is unavailable\");\n  auto trials = argc > 1 ? unsigned(std::strtoul(argv[1],nullptr,10)) : 5u;\n  auto queries = argc > 2 ? std::strtoull(argv[2],nullptr,10) : 1048576ull;\n  if (!trials || !queries) throw std::runtime_error(\"positive trials and queries required\");\n  std::vector<variant> variants{{\"scalar_popcnt\",rank_scalar}};\n  if (avx2) variants.push_back({\"avx2\",rank_avx2});\n  if (avx512bw) variants.push_back({\"avx512bw\",rank_avx512bw});\n  if (vpopcnt) variants.push_back({\"avx512vpopcntdq\",rank_avx512_vpopcnt});\n  std::uint64_t seed = 0x512f00d;\n  // Every valid position in short/partial allocations, plus domain rejection.\n  for (std::uint64_t bits=0; bits<=1088; ++bits) {\n    fixture data(bits,seed); auto view=data.view();\n    for (auto const & v:variants) {\n      for (std::uint64_t i=0; i<bits; ++i)\n        if (v.rank(view,i)!=data.oracle[i]) throw std::runtime_error(\"short rank oracle mismatch\");\n      bool rejected=false;\n      try { (void)v.rank(view,bits); } catch(std::out_of_range const &) { rejected=true; }\n      if (!rejected) throw std::runtime_error(\"rank extent was accepted\");\n    }\n  }\n  fixture data(253577,seed); auto view=data.view();\n  for (std::uint64_t i=0; i<data.bits; ++i)\n    for (auto const & v:variants)\n      if (v.rank(view,i)!=data.oracle[i]) throw std::runtime_error(\"full rank oracle mismatch\");\n  std::vector<std::uint64_t> positions(65536);\n  for (auto & q:positions) q=random_word(seed);\n  std::printf(\"# rank oracle passed; bitmap_bytes=%zu directory_bytes=%zu query_bytes=%zu\\n\",\n    data.words.size()*8,data.blocks.size()*8+data.supers.size()*8,positions.size()*8);\n  std::puts(\"pattern,variant,trial,queries,ns_per_rank,checksum\");\n  for (bool dependent:{false,true}) {\n    auto position = [&](std::uint64_t i,std::uint64_t previous) {\n      auto q=positions[i&65535] ^ (dependent ? previous*0x9e3779b97f4a7c15ull : 0);\n      return std::uint64_t((static_cast<unsigned __int128>(q)*data.bits)>>64);\n    };\n    std::uint64_t expected_checksum=0,previous=0;\n    for (std::uint64_t i=0; i<queries; ++i) {\n      auto p=position(i,previous); previous=data.oracle[p]; expected_checksum+=previous;\n      for (auto const & v:variants)\n        if (v.rank(view,p)!=previous) throw std::runtime_error(\"generated rank oracle mismatch\");\n    }\n    auto run = [&](variant const & v,std::uint64_t count) {\n      std::uint64_t sum=0,last=0;\n      auto start=std::chrono::steady_clock::now();\n      for (std::uint64_t i=0; i<count; ++i) { last=v.rank(view,position(i,last)); sum+=last; }\n      auto ns=std::chrono::duration<double,std::nano>(std::chrono::steady_clock::now()-start).count()/count;\n      return std::pair{ns,sum};\n    };\n    std::uint64_t warmed=0;\n    for (auto const & v:variants) warmed^=run(v,std::min<std::uint64_t>(queries,32768)).second;\n    for (unsigned trial=0; trial<trials; ++trial)\n      for (unsigned slot=0; slot<variants.size(); ++slot) {\n        auto const & v=variants[(trial+slot)%variants.size()]; auto [ns,sum]=run(v,queries);\n        if (sum!=expected_checksum) throw std::runtime_error(\"timed rank checksum mismatch\");\n        std::printf(\"%s,%s,%u,%llu,%.6f,%llu\\n\",dependent?\"dependent\":\"independent\",v.name,trial,\n          static_cast<unsigned long long>(queries),ns,static_cast<unsigned long long>(sum));\n      }\n    std::printf(\"# warmed=%llu\\n\",static_cast<unsigned long long>(warmed));\n  }\n  return 0;\n} catch(std::exception const & e) { std::fprintf(stderr,\"%s\\n\",e.what()); return 1; }\n\n/**\n * \\file\n * \\author Edward Kmett <ekmett@gmail.com>\n * \\brief Standalone CPUID-gated complete bitmap rank comparison.\n */\n", "compiler": "g152", "options": {"userArguments": "-O3 -std=c++20 -mpopcnt", "compilerOptions": {"executorRequest": true}, "filters": {"execute": true}, "executeParameters": {"args": ["5", "1048576"], "stdin": ""}}, "lang": "c++", "bypassCache": 2}