native.x86.wait.ccm

// SPDX-FileCopyrightText: 2024 Edward Kmett <ekmett@gmail.com>
// SPDX-License-Identifier: BSD-2-Clause OR Apache-2.0
module;
#include "native/isa_import.h"
#include <cstdint>
#include <x86intrin.h>
#include "native/attributes.h"
#include "native/targets.h"
 
export module native.x86.wait;
export import native.x86.features;
 
export namespace native {



  template <typename T>
  concept waiter = requires (volatile void * p, std::uint32_t t) {
    bool(T::supported);
    T::monitor(p);
    T::mwait(t);
  };

  template <waiter W> native_flatten
  void wait_until(volatile auto * p, auto f)
    noexcept(noexcept(!f(p)) && noexcept(W::monitor(p)) && noexcept(W::mwait(0))) native_blocking {
    [[assume(W::supported)]];
    while (!f(p)) {
      W::monitor(p);
      if (!f(p))
        W::mwait(0);
    }
  }

  struct mwaitx {
    using timer_t = std::uint64_t;
    [[gnu::target("mwaitx")]] static void monitor(volatile void * p) noexcept;
    [[gnu::target("mwaitx")]] static void mwait(std::uint32_t timer = 0) noexcept native_blocking;
    static const bool supported;
  };
 

  struct umwait {
    using timer_t = std::uint64_t;
    [[gnu::target("waitpkg")]] static void monitor(volatile void * p) noexcept;
    [[gnu::target("waitpkg")]] static std::uint8_t mwait(std::uint64_t timer = 0) noexcept native_blocking;
    static const bool supported;
  };

  template<isa<x86> Arch = NATIVE_BASELINE, unsigned Control = 1>
    requires(Arch.has(x86_feature::waitpkg) && Control < 2)
  native_inline native_artificial native_target("waitpkg")
  std::uint8_t tpause(std::uint64_t deadline) noexcept native_blocking {
    return __builtin_ia32_tpause(Control, std::uint32_t(deadline >> 32),
      std::uint32_t(deadline));
  }

  struct spin {
    using timer_t = std::uint64_t;
    native_inline native_artificial static void monitor(volatile void *) noexcept {}
    native_inline native_artificial static void mwait(std::uint32_t = 0) noexcept native_blocking { _mm_pause(); }
    inline static constinit bool supported = true;
  };

  auto with_waiter(auto k)
    noexcept(noexcept(k.template operator()<mwaitx>()) &&
      noexcept(k.template operator()<umwait>()) && noexcept(k.template operator()<spin>())) {
    if (mwaitx::supported) return k.template operator()<mwaitx>();
    else if (umwait::supported) return k.template operator()<umwait>();
    else return k.template operator()<spin>();
  }

}
 
namespace native {
  // Out-of-line module-owned leaves avoid exposing the intrinsic headers'
  // TU-local inline functions through a BMI, and preserve baseline callers.
  [[gnu::target("mwaitx")]] void mwaitx::monitor(volatile void * p) noexcept {
    _mm_monitorx(const_cast<void *>(p), 0, 0);
  }
  [[gnu::target("mwaitx")]] void mwaitx::mwait(std::uint32_t timer) noexcept native_blocking {
    _mm_mwaitx(0, 0, timer);
  }
  [[gnu::target("waitpkg")]] void umwait::monitor(volatile void * p) noexcept {
    _umonitor(const_cast<void *>(p));
  }
  [[gnu::target("waitpkg")]] std::uint8_t umwait::mwait(std::uint64_t timer) noexcept native_blocking {
    return _umwait(1, timer);
  }
 
  namespace {
    // Share the common observer and admission rules, including leaf validity.
    // Waiting needs no vector state or unrelated SIMD instruction features.
    const auto wait_capabilities = observe_x86_capabilities();
  }
  const bool mwaitx::supported = classify_isa(wait_capabilities, x86_feature::mwaitx).admitted();
  const bool umwait::supported = classify_isa(wait_capabilities, x86_feature::waitpkg).admitted();
}