memory.hpp

Go to the documentation of this file.

/**
 * @file
 * @ingroup numapp_mem
 * @copyright ESO 2023 - European Southern Observatory
 *
 * @brief Contains memory function declarations.
 *
 * @defgroup numapp_mem_huge Huge Pages
 * @ingroup numapp_mem
 * @brief Utilities for allocating huge pages.
 *
 */
#ifndef NUMAPP_MEMORY_HPP_
#define NUMAPP_MEMORY_HPP_
#include <numapp/config.hpp>
 
#include <sys/mman.h>
 
#include <cstddef>
#include <memory_resource>
#include <optional>
#include <system_error>
 
#include <numapp/flags.hpp>
#include <numapp/mempolicy.hpp>
#include <numapp/nodemask.hpp>
 
namespace numapp {
 
/**
 * @name Hardware Queries
 *
 * Query host system/hardware.
 * @ingroup numapp_mem
 */
// @{
 
/**
 * Fast query of system page size.
 *
 * @returns number of bytes in a page.
 *
 * @manpages
 * @manpage{numa,3,@c numa_pagesize}
 * @ingroup numapp_mem
 */
[[nodiscard]] std::size_t GetPageSize() noexcept;
 
/**
 * Query number of configured NUMA nodes.
 *
 * @returns number of configured NUMA nodes (this also includes any disabled nodes).
 *
 * @manpages
 * @manpage{numa,3,@c numa_num_configured_nodes}
 * @ingroup numapp_mem
 */
[[nodiscard]] int GetNumNodes() noexcept;
 
/**
 * Get NUMA distance between two nodes.
 *
 * A node has distance 10 to itself.
 *
 * @param node1 NUMA node number.
 * @param node2 NUMA node number.
 *
 * @returns distance between @a node1 and @a node2.
 * @returns @c std::nullopt when distance cannot be determined.
 *
 * @manpages
 * @manpage{numa,3,@c numa_distance}
 * @ingroup numapp_mem
 */
[[nodiscard]] std::optional<int> GetNodeDistance(int node1, int node2) noexcept;
 
/**
 * Get NUMA node of the given CPU
 *
 * @param cpu CPU to get the NUMA node for.
 *
 * @returns NUMA node.
 * @returns @c std::nullopt if @a cpu is invalid.
 *
 * @manpages
 * @manpage{numa,3,@c numa_node_of_cpu}
 * @ingroup numapp_mem
 */
[[nodiscard]] std::optional<int> GetNodeOfCpu(int cpu) noexcept;
// @}
 
/**
 * @name Memory Locking
 *
 * These functions provide the means to prevent memory from being paged to the swap area.
 *
 * @note Care should be taken when using these as resident memory consumption will increase as the
 * risk of out-of-memory scenarios. In particular it should be noted that applications may allocate
 * memory that is never resident in normal use. Forcing this memory to be faulted-in may lead to
 * significant memory pressure. One way to avoid that is to use @c LockFlag::OnFault and
 * @c LockAllFlag::OnFault which will lock resident memory only. _For real-time applications_ this
 * should not be used however as page faults should be avoided in those scenarios.
 *
 * @par Permissions
 *
 * Locking may require additional permissions if it exceeds resource limits (\manpage{getrlimit,2}).
 * Permissions can be provided by `CAP_IPC_LOCK` (\manpage{capabilities,7}).
 *
 * See also page section @ref memory-api-locking.
 * @ingroup numapp_mem
 */
// @{
 
/**
 * Mutually exclusive flags that modifies behaviour of @ref MemLock().
 *
 * @ingroup numapp_mem
 */
enum class LockFlag : unsigned int {
    /**
     * Locks pages whether they are resident or not. After a successful call to MemLock access to
     * the full range will not trigger page faults.
     *
     * @note Behaves like @c mlock().
     */
    PreFault = 0u,
 
    /**
     * Locks currently resident pages as well as future resident pages _when_ they are first faulted
     * in.
     */
    OnFault = MLOCK_ONFAULT,
};
 
/**
 * Flags that are combined to modify behaviour of @ref MemLockAll().
 *
 * LockAllFlag have the following operators defined in namespace numapp:
 * - <tt>operator|(LockAllFlag, LockAllFlag)</tt>
 * - <tt>operator=|(LockAllFlag&, LockAllFlag)</tt>
 * - <tt>operator&(LockAllFlag, LockAllFlag)</tt>
 * - <tt>operator=&(LockAllFlag&, LockAllFlag)</tt>
 *
 * @ingroup numapp_mem
 */
enum class LockAllFlag : int {
    /**
     * Lock all pages which are currently mapped into the address space of the process.
     */
    Current = MCL_CURRENT,
 
    /**
     * Lock all future pages that are mapped into the address space of the process.
     */
    Future = MCL_FUTURE,
 
#if defined(MCL_ONFAULT) || defined(DOXYGEN)
    /**
     * OnFault must be used together with Current and/or Future and modifies the behaviour so pages
     * are not locked until they are faulted in.
     *
     * @since Linux 4.4
     */
    OnFault = MCL_ONFAULT,
#endif
};
 
NUMAPP_ENABLE_BITFLAG(LockAllFlag)
 
/**
 * Lock memory pages in the specified address range.
 *
 * To unlock memory use numapp::MemUnlock or numapp::MemUnlockAll.
 *
 * @note Locking memory can be used in conjunction with e.g. numapp::Allocate() to allocate memory
 * and pre-fault it to guarantee that no page-faults happen on subsequent access.
 *
 * @code
 * void* buffer = numapp::Allocate(size, policy);
 * if (std::error_code ec = numapp::MemLock(buffer, size, numapp::LockFlag::PreFault); ec) {
 *   // Handle error
 * }
 * @endcode
 *
 * @param addr Defines the starting address of the range.
 * @param len  Defines the length of the address range.
 * @param flag Behaviour modifier, see @ref LockFlag.
 * @return error code populated with errno error.
 *
 * @manpages
 * @manpage{mlock2,2}
 * @ingroup numapp_mem
 */
[[nodiscard]] std::error_code MemLock(void const* addr, std::size_t len, LockFlag flag) noexcept;
 
/**
 * Unlock memory pages in the specified address range.
 *
 * @param addr Defines the starting address of the range.
 * @param len  Defines the length of the address range.
 * @return error code populated with errno error.
 *
 * @manpages
 * @manpage{munlock,2}
 * @ingroup numapp_mem
 */
[[nodiscard]] std::error_code MemUnlock(void const* addr, std::size_t len) noexcept;
 
/**
 * Lock all memory pages as specified by provided flags.
 *
 * As an example this is how to lock current and future memory allocations:
 * @code
 *
 * using namespace numapp; // Also brings in operator| into scope so that flags can be OR-ed.
 *
 * if (auto ec = MemLockAll(LockAllFlag::Current | LockAllFlag::Future); ec) {
 *     // Failed to lock...
 * }
 *
 * @endcode
 *
 * To unlock memory use MemUnlock and MemUnlockAll.
 *
 * @warning If flag @c LockAllFlag::Future is used memory pages will be locked during
 * mapping using the calling thread default memory policy. For numapp::Allocate() this means that
 * pages may have to be moved after mapping (see also notes in numapp::Allocate()).
 *
 * @param flags Combination of @ref LockAllFlag values. See documentation of @ref LockAllFlag how it
 * modifies the behaviour.
 * @return error code populated with errno error.
 *
 * @manpages
 * @manpage{mlockall,2}
 * @ingroup numapp_mem
 */
[[nodiscard]] std::error_code MemLockAll(LockAllFlag flags) noexcept;
 
/**
 * Unlock all locked memory in this process.
 *
 * @return error code populated with errno error.
 *
 * @manpages
 * @manpage{munlockall,2}
 * @ingroup numapp_mem
 */
[[nodiscard]] std::error_code MemUnlockAll() noexcept;
// @}
 
/**
 * @name Allocate and free memory with specified memory policy
 *
 * @anchor numapp_alloc
 *
 * @par Allocate
 * numapp::Allocate() function is equivalent to performing the following:
 * - Create new memory mapping with `mmap` (anonymous read/write) memory
 * - Applying memory policy to allocated memory using provided @c policy and @c flag <sup>(1)</sup>
 *   using `mbind`.
 *
 * <sup>(1)</sup> Overloads without @c flag use no modifiers (i.e. won't move pages).
 *
 * @par Important
 * Memory is not pre-faulted by default. If memory is used by real-time threads consider using
 * numapp::MemLock() after allocation to guarantee that pages as faulted in.
 *
 * @par
 * However if process locks memory when it is mapped (as controlled by numapp::MemLockAll() with
 * @c numapp::LockAllFlag::Future) memory may have to be @a moved to apply the requested policy. It
 * is in this case the @c flags argument has an important role to control if pages should be moved.
 *
 * @par Free
 * numapp::Free()  frees memory previously allocated with numapp::Allocate().
 *
 * @param [in] size Number of bytes to allocate/free, will be rounded up to multiples of system page
 * size.
 * @param [in] policy Memory policy to apply.
 * @param [in] flags Controls behaviour if memory have to be moved because it was already paged-in
 * due to a numapp::MemLockAll policy.
 * @param [in] ptr Page-aligned pointer to memory previously allocated with numapp::Allocate().
 * @param [out] ec Error code for non-throwing overloads. If there is an out of memory situation the
 * error code @c std::errc::not_enough_memory is used.
 *
 * @throws std::system_error on failure (only applicable to throwing-overloads without
 * @c std::error_code return value).
 *
 * @returns numapp::Allocate() returns pointer to memory block that is at least @a size big or @c
 * nullptr on failure.
 *
 * Example allocating with bind policy and then lock and pre-fault the newly allocated memory:
 *
 * @include allocExample.cpp
 *
 * @manpages
 * @manpage{mmap,2}
 * @manpage{mbind,2}
 * @ingroup numapp_mem
 */
/// @{
 
// clang-format off
/**
 * @relatesalso MemPolicy
 * See @ref numapp_alloc "group" for details.
 * @ingroup numapp_mem */
[[nodiscard]] void* Allocate(std::size_t size,
                             MemPolicy const& policy,
                             std::error_code& ec) noexcept;
/**
 * @relatesalso MemPolicy
 * See @ref numapp_alloc "group" for details.
 * @ingroup numapp_mem */
[[nodiscard]] void* Allocate(std::size_t size,
                             MemPolicy const& policy,
                             MemPolicyFlag flags,
                             std::error_code& ec) noexcept;
/**
 * @relatesalso MemPolicy
 * See @ref numapp_alloc "group" for details.
 * @ingroup numapp_mem */
[[nodiscard]] void* Allocate(std::size_t size,
                             MemPolicy const& policy);
/**
 * @relatesalso MemPolicy
 * See @ref numapp_alloc "group" for details.
 * @ingroup numapp_mem */
[[nodiscard]] void* Allocate(std::size_t size,
                             MemPolicy const& policy,
                             MemPolicyFlag flags);
// clang-format on
/**
 * @relatesalso MemPolicy
 * See @ref numapp_alloc "group" for details.
 * @ingroup numapp_mem */
void Free(void* ptr, std::size_t size, std::error_code& ec) noexcept;
 
/**
 * @relatesalso MemPolicy
 * See @ref numapp_alloc "group" for details.
 * @ingroup numapp_mem */
void Free(void* ptr, std::size_t size);
/// @}
 
/**
 * Polymorphic memory resource allocating full system pages with specified NUMA policy. Memory is
 * page-aligned and allocated size is rounded up to nearest full page.
 *
 * Allocations are performed using numapp::Allocate() and deallocations use numapp::Free(), see @ref
 * numapp_alloc for details.
 *
 * @attention This is not a suitable general purpose memory resource as it is generally slow and
 * always allocates full pages and should be considered to be a block allocator serving a higher
 * level allocator that is aware of this, with blocks of memory.
 *
 * @ingroup numapp_mem
 * @headerfile <> <numapp/memory.hpp>
 */
class PageResource : public std::pmr::memory_resource {
public:
    /**
     * Create PageResource with specified memory policy and flags.
     *
     * @param policy Memory policy to use when allocating.
     * @param flags Memory policy flags to use when allocating.
     */
    PageResource(MemPolicy policy, MemPolicyFlag flags = MemPolicyFlag::None);
 
    /**
     * Destroy PageResource.
     */
    virtual ~PageResource() = default;
 
    /**
     * Page resource is not copy assignable.
     */
    PageResource& operator=(PageResource const&) = delete;
 
    /**
     * Get memory policy in use.
     */
    auto GetPolicy() const noexcept -> MemPolicy const&;
 
    /**
     * Get flags in use.
     */
    auto GetFlags() const noexcept -> MemPolicyFlag;
 
private:
    auto do_allocate(std::size_t bytes, std::size_t alignment) -> void* override;
    void do_deallocate(void* p, std::size_t bytes, std::size_t alignment) override;
    auto do_is_equal(std::pmr::memory_resource const& other) const noexcept -> bool override;
 
    MemPolicy m_policy;
    MemPolicyFlag m_flags;
};
 
/**
 * Lock memory allocated from upstream memory resource using specified @ref LockFlag.
 *
 * @note If locking fails memory is deallocated and `std::system_error` exception is thrown.
 *
 * @ingroup numapp_mem
 * @headerfile <> <numapp/memory.hpp>
 */
class LockResource : public std::pmr::memory_resource {
public:
    /**
     * Initialize with specified flag and upstream memory resource initialized from
     * `std::pmr::get_default_resource()`.
     *
     * @param flag Lock flag controlling memory locking behavior.
     */
    LockResource(LockFlag flag) noexcept;
 
    /**
     * Initialize with specified upstream memory resource using `LockFlag::PreFault`.
     *
     * @param upstream Upstream memory resource from where allocations are delegated; must point to
     * valid memory resource.
     */
    LockResource(std::pmr::memory_resource* upstream) noexcept;
 
    /**
     * Initialize with specified lock flag and upstream memory resource.
     *
     * @param flag Lock flag controlling memory locking behavior.
     * @param upstream Upstream memory resource from where allocations are delegated; must point to
     * valid memory resource.
     */
    LockResource(LockFlag flag, std::pmr::memory_resource* upstream) noexcept;
 
    /**
     * Destroy LockResource.
     */
    virtual ~LockResource() = default;
 
    /**
     * LockResource is not copy assignable.
     */
    LockResource& operator=(LockResource const&) = delete;
 
    /**
     * Get lock flag in use.
     *
     * @return Lock flags.
     */
    auto GetFlags() const noexcept -> LockFlag;
 
    /**
     * Get upstream memory resource.
     *
     * @return Upstream memory resoure.
     */
    auto GetUpstream() const noexcept -> std::pmr::memory_resource*;
 
private:
    auto do_allocate(std::size_t bytes, std::size_t alignment) -> void* override;
    void do_deallocate(void* p, std::size_t bytes, std::size_t alignment) override;
    auto do_is_equal(std::pmr::memory_resource const& other) const noexcept -> bool override;
 
    std::pmr::memory_resource* m_upstream;
    LockFlag m_flag;
};
 
/**
 * Preset huge page sizes.
 *
 * Commonly supported sizes are
 * - Huge2M and Huge1G on x86.
 * - Huge8k, Huge64k, Huge256k, Huge1M, Huge4M, Huge16M, Huge256M on ia64.
 * - Huge16M on ppc64.
 *
 * @ingroup numapp_mem_huge
 */
enum class HugePagePreset : std::size_t {
    /**
     * 8 KiB page size.
     */
    Huge8k = 8 * 1024,
 
    /**
     * 16 KiB page size.
     */
    Huge16k = 16 * 1024,
 
    /**
     * 64 kiB page size.
     */
    Huge64k = 64 * 1024,
 
    /**
     * 256 kiB page size.
     */
    Huge256k = 256 * 1024,
 
    /**
     * 1 MiB page size.
     */
    Huge1M = 1 * 1024 * 1024,
 
    /**
     * 2 MiB page size.
     */
    Huge2M = 2 * 1024 * 1024,
 
    /**
     * 4 MiB page size.
     */
    Huge4M = 4 * 1024 * 1024,
 
    /**
     * 16 MiB page size.
     */
    Huge16M = 16 * 1024 * 1024,
 
    /**
     * 256 MiB page size.
     */
    Huge256M = 256 * 1024 * 1024,
 
    /**
     * 1 GiB page size.
     */
    Huge1G = 1ul * 1024 * 1024 * 1024,
};
 
/**
 * Describes a huge page size and maintains the invariant that page size is an integral power of 2,
 * compatible with `mmap()` expectations.
 *
 * @ingroup numapp_mem_huge
 */
class HugePageSize {
public:
    /**
     * Construct from preset page size value.
     *
     * @param preset page size preset.
     */
    HugePageSize(HugePagePreset preset) noexcept;
 
    /**
     * Construct with given page size.
     *
     * Provided size @a bytes must an integer power of two and its base-2 logarithm must not be 0.
     *
     * @param bytes Size in bytes.
     *
     * @throws std::invalid_argument if @a bytes is not an integer power of two or its base-2
     * logarithm is 0.
     */
    explicit HugePageSize(std::size_t bytes);
 
    /**
     * Query size in bytes.
     *
     * @return Size in bytes, is always an integer power of two and its base-2 logarithm is > 0.
     */
    constexpr auto Size() const noexcept -> std::size_t;
 
    /**
     * Three-way comparison operator.
     */
    auto operator<=>(HugePageSize const&) const noexcept -> std::strong_ordering = default;
 
private:
    std::size_t m_size;
};
 
/**
 * Encodes page size into bit representation expected by `mmap()`.
 *
 * @return Page size encoded as `mmap()` flags.
 *
 * @manpages
 * @manpage{mmap,2}
 * @ingroup numapp_mem_huge
 */
auto EncodeMmapFlags(HugePageSize page_size) noexcept -> int;
 
/**
 * @name Allocate and free huge pages
 *
 * @anchor numapp_alloc_huge
 *
 * Allocation is similar to @ref numapp_alloc "Allocate and free" but with the additional argument
 * specifying the (huge) page size. Like non-huge allocations the pages are not pre-faulted by the
 * functions so to avoid page fault failurs it is recommended to do that before use.
 *
 * @include allocHugeExample.cpp
 *
 * @par Parameters
 * @param [in] size Number of bytes to allocate/free, will be rounded up to multiples of @a
 * page_size.
 * @param [in] page_size Page size.
 * @param [in] policy Memory policy to apply.
 * @param [in] flags Controls behaviour if memory have to be moved because it was already paged-in
 * due to a numapp::MemLockAll policy.
 * @param [in] ptr Page-aligned pointer to memory previously allocated with numapp::AllocateHuge().
 * @param [out] ec Error code for non-throwing overloads. If there is an out of memory situation the
 * error code @c std::errc::not_enough_memory is used.
 *
 * @throws std::system_error on failure (only applicable to throwing-overloads without
 * @c std::error_code return value).
 *
 * @returns numapp::AllocateHuge() returns pointer to memory block that is at least @a size big or
 * @c nullptr on failure.
 * @manpages
 * @manpage{mmap,2}
 * @manpage{mbind,2}
 */
// @{
/**
 * Non-throwing version of AllocateHuge()
 *
 * See @ref numapp_alloc_huge "Allocate and free huge pages" for details.
 *
 * @relatesalso MemPolicy
 * @ingroup numapp_mem_huge
 */
[[nodiscard]] void* AllocateHuge(std::size_t size,
                                 HugePageSize page_size,
                                 MemPolicy const& policy,
                                 MemPolicyFlag flags,
                                 std::error_code& ec) noexcept;
 
/**
 * Throwing version of AllocateHuge()
 *
 * See @ref numapp_alloc_huge "Allocate and free huge pages" for details.
 *
 * @relatesalso MemPolicy
 * @ingroup numapp_mem_huge
 */
[[nodiscard]] void* AllocateHuge(std::size_t size,
                                 HugePageSize page_size,
                                 MemPolicy const& policy,
                                 MemPolicyFlag flags);
 
/**
 * Free huge pages previously allocated with AllocateHuge.
 *
 * @note Behaviour is unspecified if @a ptr, @a size and @a page_size are different than from
 * previous call to AllocateHuge().
 *
 * @note page_size is needed because munmap requires that sz in munmap(ptr, sz) is a multiple of
 * page size, which is not the case for system pages.
 *
 * See @ref numapp_alloc_huge "Allocate and free huge pages" for details.
 *
 * @ingroup numapp_mem_huge
 */
void FreeHuge(void* ptr, std::size_t size, HugePageSize page_size, std::error_code& ec) noexcept;
 
/**
 * Throwing version of `FreeHuge()`.
 *
 * See @ref numapp_alloc_huge "Allocate and free huge pages" for details.
 *
 * @throws std::system_error on errors.
 */
void FreeHuge(void* ptr, std::size_t size, HugePageSize page_size);
// @}
 
/**
 * Polymorphic memory resource allocating huge pages with specified NUMA policy. Memory is
 * page-aligned and allocated size is rounded up to nearest full page.
 *
 * Allocations are performed using numapp::AllocateHuge() and deallocations use numapp::FreeHuge(),
 * see @ref numapp_alloc_huge for details.
 *
 * @attention This is not a suitable general purpose memory resource as it is generally slow and
 * always allocates full pages and should be considered to be a block allocator serving a higher
 * level allocator that is aware of this, with blocks of memory.
 *
 * @ingroup numapp_mem_huge
 * @headerfile <> <numapp/memory.hpp>
 */
class HugePageResource : public std::pmr::memory_resource {
public:
    /**
     * Create PageResource with specified memory policy and flags.
     *
     * @param page_size Page size to use when allocating.
     * @param policy Memory policy to use when allocating.
     * @param flags Memory policy flags to use when allocating.
     */
    HugePageResource(HugePageSize page_size,
                     MemPolicy policy,
                     MemPolicyFlag flags = MemPolicyFlag::None);
 
    /**
     * Destroy PageResource.
     */
    virtual ~HugePageResource() = default;
 
    /**
     * HugePage resource is not copy assignable.
     */
    HugePageResource& operator=(HugePageResource const&) = delete;
 
    /**
     * Get memory policy in use.
     */
    auto GetPolicy() const noexcept -> MemPolicy const&;
 
    /**
     * Get flags in use.
     */
    auto GetFlags() const noexcept -> MemPolicyFlag;
 
    /**
     * Get page size.
     */
    auto GetPageSize() const noexcept -> HugePageSize;
 
private:
    auto do_allocate(std::size_t bytes, std::size_t alignment) -> void* override;
    void do_deallocate(void* p, std::size_t bytes, std::size_t alignment) override;
    auto do_is_equal(std::pmr::memory_resource const& other) const noexcept -> bool override;
 
    HugePageSize m_page_size;
    MemPolicy m_policy;
    MemPolicyFlag m_flags;
};
 
constexpr auto HugePageSize::Size() const noexcept -> std::size_t {
    return m_size;
}
 
}  // namespace numapp
#endif  // #ifndef NUMAPP_MEMORY_HPP_