mempolicy.hpp

Go to the documentation of this file.

/**
 * @file
 * @ingroup numapp_mem
 * @copyright ESO 2024 - European Southern Observatory
 *
 * @brief Contains declarations for numapp::MemPolicy.
 *
 * @defgroup numapp_mem Memory APIs
 * @ingroup numapp
 * @brief NUMA++ memory policy APIs
 *
 * See @ref memory-api-policy for an overview.
 */
#ifndef NUMAPP_MEMPOLICY_HPP_
#define NUMAPP_MEMPOLICY_HPP_
#include <numapp/config.hpp>
 
#include <iosfwd>
#include <memory>
#include <numaif.h>
#include <optional>
#include <system_error>
 
#include <numapp/flags.hpp>
#include <numapp/nodemask.hpp>
 
namespace numapp {
class MemPolicy;
 
/**
 * Flag that modify the behaviour of applying a memory policy to a range of memory.
 * @ingroup numapp_mem
 * @headerfile <> <numapp/mempolicy.hpp>
 * @relatesalso MemPolicy
 */
enum class MemPolicyFlag : unsigned {
    /**
     * No-op flag.
     */
    None = 0u,
    /**
     * Determines if application of memory policy should be strict and fail if MemPolicy cannot be
     * used rather than e.g. allocate from other nodes.
     *
     * Cause Apply(void*, std::size_t, MemPolicy, MemPolicyFlag) to fail if existing pages don't
     * follow the policy. If combined with Move the operation will fail if all pages couldn't be
     * moved.
     *
     * This does not apply to MemPolicy::Mode::Default or MemPolicy::Mode::Interleaved.
     */
    Strict = MPOL_MF_STRICT,
    /**
     * Attempt to move pages so that they follow the policy. Can be combined with Strict to cause
     * operation to fail if pages couldn't be moved.
     */
    Move = MPOL_MF_MOVE,
    /**
     * Attempt to move pages so that they follow the policy regardless whether another process is
     * using the pages. This requires capability CAP_SYS_NICE.
     *
     * Can be combined with Strict to have the operation fail if all pages couldn't be moved.
     */
    MoveAll = MPOL_MF_MOVE_ALL,
};
 
NUMAPP_ENABLE_BITFLAG(MemPolicyFlag)
 
namespace thisThread {
 
/**
 * @name Apply Memory Policy to Current Thread
 *
 * Applies default memory policy to calling thread or to thread stack memory.
 * @ingroup numapp_mem
 */
/// @{
 
/**
 * Applies the default memory policy to the calling thread.
 *
 * @note The memory policy is only applied to new physical page allocations.
 *
 * @param policy memory policy to set.
 *
 * @manpages
 * @manpage{set_mempolicy,2}
 *
 * @relatesalso numapp::MemPolicy
 * @ingroup numapp_mem
 */
[[nodiscard]] std::error_code Apply(MemPolicy const& policy) noexcept;
 
/**
 * Convenience function that applies a memory policy to current thread stack memory.
 *
 * @note It relies on pthread API get the stack memory address and size. No pre-faulting is made.
 *
 * @param policy Memory policy to apply.
 * @param flags combination of MemPolicyFlag values that modify the behaviour. Default is to move
 * pages and fail if this cannot be done.
 *
 * @manpages
 * @manpage{mbind,2}
 * @relatesalso MemPolicy
 * @ingroup numapp_mem
 */
[[nodiscard]] std::error_code
ApplyStack(MemPolicy const& policy,
           MemPolicyFlag flags = MemPolicyFlag::Move | MemPolicyFlag::Strict) noexcept;
/// @}
 
}  // namespace thisThread
 
/**
 * @name Apply Memory Policy to Memory Range
 */
/// @{
 
/**
 * Applies memory policy to the memory pages that spans the range [@a address, @a adress +
 * @a length].
 *
 * @note The memory policy is by default only applied to new physical page allocations. This can be
 * modified using MemPolicyFlag::Move and MemPolicyFlag::MoveAll.
 *
 * Read related man-page of @c mbind() to understand the various behaviours if memory is mapped or
 * shared.
 *
 * @param address start of address range.
 * @param length length of range.
 * @param policy Policy to apply.
 * @param flags combination of MemPolicyFlag values that modify the behaviour.
 * @returns 0 if successful.
 * @returns non-zero if it failed.
 *
 * @manpages
 * @manpage{mbind,2}
 * @relatesalso MemPolicy
 */
[[nodiscard]] std::error_code Apply(void* address,
                                    std::size_t length,
                                    MemPolicy const& policy,
                                    MemPolicyFlag flags) NUMAPP_NOEXCEPT;
///@}
 
/**
 * Class representing a memory policy that can be modified and used to apply to the current thread
 * or a memory range.
 *
 * Uses numa APIs @c set_mempolicy and @c get_mempolicy.
 *
 * To read the current memory policy use MemPolicy::MakeFromActive. To apply a policy use
 * thisThread::Apply(MemPolicy const&).
 *
 * For CPU policy see @c numapp::CpuAffinity.
 *
 * @attention The memory policy is used for new physical page allocations. One way this happens is
 * by writing to unpaged memory which triggers a page-fault that then is handled according to the
 * policy.  This means that it's possible that memory returned by an allocator like
 * @c malloc/@c std::allocator/@c new after a sucessfull call to MemPolicy::Apply does not in
 * fact allocate a new physical page but may reuse recently freed memory or otherwise unused, but
 * physically allocated memory.
 *
 * @attention Bottom line is that MemPolicy does @a not apply directly to virtual memory allocators
 * but on the kernel level page allocation.
 *
 * @headerfile <> <numapp/mempolicy.hpp>
 * @related numapp::ScopedMemPolicy
 * @manpages
 * @manpage{get_mempolicy,2}
 * @manpage{set_mempolicy,2}
 * @ingroup numapp_mem
 */
class MemPolicy {
public:
    /**
     * Memory allocation modes.
     */
    enum class Mode : int {
        /**
         * The Default mode specifies that any nondefault process memory policy be removed, so that
         * the memory policy "falls back" to the system default policy. The system default policy is
         * "local allocation"-- i.e., allocate memory on the node of the CPU that triggered the
         * allocation. nodemask must be specified as NULL. If the "local node" contains no free
         * memory, the system will attempt to allocate memory from a "near by" node.
         *
         * Corresponds to native mode @c MPOL_DEFAULT.
         */
        Default = MPOL_DEFAULT,
        /**
         * The Bind mode defines a strict policy that restricts memory allocation to the nodes
         * specified in nodemask. If nodemask specifies more than one node, page allocations will
         * come from the node with the lowest numeric node ID first, until that node contains no
         * free memory. Allocations will then come from the node with the next highest node ID
         * specified in nodemask and so forth, until none of the specified nodes contain free
         * memory. Pages will not be allocated from any node not specified in the nodemask.
         *
         * Corresponds to native mode @c MPOL_BIND.
         */
        Bind = MPOL_BIND,
        /**
         * Interleave interleaves page allocations across the nodes specified in nodemask in
         * numeric node ID order. This optimizes for bandwidth instead of latency by spreading out
         * pages and memory accesses to those pages across multiple nodes. However, accesses to a
         * single page will still be limited to the memory bandwidth of a single node.
         *
         * Corresponds to native mode @c MPOL_INTERLEAVE.
         */
        Interleave = MPOL_INTERLEAVE,
        /**
         * Preferred sets the preferred node for allocation. The kernel will try to allocate pages
         * from this node first and fall back to "near by" nodes if the preferred node is low on
         * free memory. If nodemask specifies more than one node ID, the first node in the mask will
         * be selected as the preferred node. If the nodemask and maxnode arguments specify the
         * empty set, then the policy specifies "local allocation" (like the system default policy
         * discussed above).
         *
         * Corresponds to native mode @c MPOL_PREFERRED.
         */
        Preferred = MPOL_PREFERRED
    };
 
    /**
     * Create memory policy.
     *
     * @param mode Policy mode
     * @param node_mask
     */
    explicit MemPolicy(Mode mode, Nodemask&& node_mask) noexcept
        : m_mode(mode), m_nodes(std::move(node_mask)) {
    }
    explicit MemPolicy(Mode mode, Nodemask const& node_mask) : m_mode(mode), m_nodes(node_mask) {
    }
 
    MemPolicy(MemPolicy&&) noexcept = default;
    MemPolicy& operator=(MemPolicy&&) noexcept = default;
    MemPolicy(MemPolicy const& rhs) = default;
    MemPolicy& operator=(MemPolicy const& rhs) noexcept = default;
 
    /**
     * Creates a strict policy (using @c MPOL_BIND) to allocate all memory to the specified node.
     *
     * @param node Numa node.
     * @throws std::system_error if it fails.
     */
    [[nodiscard]] static MemPolicy MakeBindNode(int node);
 
    /**
     * Create MemPolicy from mode and `nodestring` while considering currently allowed nodes.
     *
     *
     * Nodestring use patterns such as:
     *
     * - <tt>1-5,7,10</tt>
     * - <tt>!4-5</tt>
     * - <tt>+0-3</tt>
     * - <tt>all</tt>
     *
     * @manpages
     * @manpage{numa,3}
     *
     * @param mode Memory policy mode such as @c MPOL_BIND
     * @param nodestring Nodes to include such as <tt>1-5,7,10</tt> or <tt>all</tt>.
     * @throws std::system_error on failure. This can also happen if the @a nodestring references a
     * disallowed node. Use @ref MakeFromNodeStringAll to ignore this.
     *
     * @sa MakeFromNodeStringAll
     */
    [[nodiscard]] static MemPolicy MakeFromNodeString(Mode mode, char const* nodestring) {
        return MemPolicy(mode, Nodemask::MakeFromNodestring(nodestring));
    }
 
    /**
     * Create MemPolicy from mode and `nodestring` considering all nodes, whether they are allowed
     * or not.
     *
     * Nodestring use patterns such as:
     *
     * - <tt>1-5,7,10</tt>
     * - <tt>!4-5</tt>
     * - <tt>+0-3</tt>
     * - <tt>all</tt>
     *
     * @manpages
     * @manpage{numa,3}
     *
     * @param mode Memory policy mode such as @c MPOL_BIND
     * @param nodestring Nodes to include such as <tt>1-5,7,10</tt> or <tt>all</tt>.
     * @throws std::system_error on failure. This can also happen if the @a nodestring references a
     * disallowed node. Use @ref MakeFromNodeStringAll to ignore this.
     *
     * @sa MakeFromNodeString
     */
    [[nodiscard]] static MemPolicy MakeFromNodeStringAll(Mode mode, char const* nodestring) {
        return MemPolicy(mode, Nodemask::MakeFromNodestringAll(nodestring));
    }
 
    /**
     * Make from active default policy of calling thread.
     *
     * @throws std::system_error if it fails.
     */
    [[nodiscard]] static MemPolicy MakeFromActive();
 
    /**
     * Make from policy at address.
     *
     * @param address memory address to make policy from.
     * @throws std::system_error if it fails.
     */
    [[nodiscard]] static MemPolicy MakeFromAddress(void* address);
 
    /**
     * @return true if this memory policy mode and nodes equals @a rhs.
     */
    [[nodiscard]] bool operator==(MemPolicy const& rhs) const noexcept {
        return m_mode == rhs.m_mode && m_nodes == rhs.m_nodes;
    }
    [[nodiscard]] bool operator!=(MemPolicy const& rhs) const noexcept {
        return !(*this == rhs);
    }
 
    /**
     * @return policy mode.
     */
    [[nodiscard]] Mode GetMode() const noexcept {
        return m_mode;
    }
 
    /**
     * Set memory policy mode.
     *
     * @param mode policy mode.
     */
    void SetMode(Mode mode) noexcept {
        m_mode = mode;
    }
 
    /**
     * @return Node mask.
     */
    Nodemask const& GetNodemask() const noexcept {
        return m_nodes;
    }
 
private:
    MemPolicy() : m_mode(Mode::Default), m_nodes() {
    }
 
    Mode m_mode;
    Nodemask m_nodes;
};
 
/**
 * Formats @a mode and inserts it to @a os.
 *
 * @param os output stream to insert into.
 * @param mode Memory policy mode to format.
 * @returns @a os
 *
 * @relates MemPolicy
 */
std::ostream& operator<<(std::ostream& os, MemPolicy::Mode const& mode);
 
/**
 * Formats @a mempolicy and inserts it to @a os.
 *
 * @param os output stream to insert into.
 * @param mempolicy Memory policy to format.
 * @returns @a os
 *
 * @relates MemPolicy
 */
std::ostream& operator<<(std::ostream& os, MemPolicy const& mempolicy);
 
/**
 * Applies memory policy to this thread at construction and reverts to previous at destruction.
 *
 * @sa numapp::MemPolicy
 *
 * @ingroup numapp_mem
 */
class [[maybe_unused]] ScopedMemPolicy {
public:
    /**
     * Constructs an object that applies the provided policy during construction
     * and restores it during destruction.
     *
     * @throws std::system_error containing the error code if it fails to apply new policy.
     * Attempt to restore old policy will be done.
     */
    explicit ScopedMemPolicy(MemPolicy const& new_policy)
        : m_old_policy(MemPolicy::MakeFromActive()) {
        auto ec = thisThread::Apply(new_policy);
        if (ec) {
            throw std::system_error(ec);
        }
    }
 
    /**
     * Restores old policy, if any.
     */
    ~ScopedMemPolicy() noexcept {
        (void)Restore();
    }
 
    /**
     * Initialized a scoped memory policy that does nothing
     * and holds no state about previous policy.
     */
    ScopedMemPolicy() noexcept = default;
 
    /**
     * Swap state with another @c ScopedMemPolicy.
     */
    void Swap(ScopedMemPolicy& rhs) noexcept {
        m_old_policy.swap(rhs.m_old_policy);
    }
 
    /**
     * Restores memory policy that was read during construction.
     *
     * @returns the error that caused it to fail to apply old policy.
     */
    [[nodiscard]] std::error_code Restore() noexcept;
 
private:
    std::optional<MemPolicy> m_old_policy;
};
 
}  // namespace numapp
#endif  // #ifndef NUMAPP_MEMPOLICY_HPP_