Files
linux-stable-mirror/arch/powerpc/platforms/pseries/hotplug-memory.c
T
Linus Torvalds a552c81ff4 Merge tag 'mm-stable-2026-06-18-09-26' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
Pull MM updates from Andrew Morton:

 - "selftests/mm: clean up build output and verbosity" (Li Wang)

   Remove some noise from the MM selftests build

 - "mm: Free contiguous order-0 pages efficiently" (Ryan Roberts)

   Speed up the freeing of a batch of 0-order pages by first scanning
   them for coalescing opportunities. This is applicable to vfree() and
   to the releasing of frozen pages

 - "mm/damon: introduce DAMOS failed region quota charge ratio"
   (SeongJae Park)

   Address a DAMOS usability issue: The DAMOS quota often exhausts
   prematurely because it charges for all memory attempted, causing slow
   and inconsistent performance when actions fail on unreclaimable
   memory.

   To fix this, a new feature lets users set a smaller, flexible quota
   charge ratio (via a numerator and denominator) for failed regions.
   Since failed actions cause less overhead, reducing their quota cost
   ensures more predictable and efficient DAMOS processing

 - "selftests/cgroup: improve zswap tests robustness and support large
   page sizes" (Li Wang)

   Fix various spurious failures and improves the overall robustness of
   the cgroup zswap selftests

 - "fix MAP_DROPPABLE not supported errno" (Anthony Yznaga)

   Fix an issue in the mlock selftests on arm32

 - "mm: huge_memory: clean up defrag sysfs with shared" (Breno Leitao)

   Some maintenance work in the huge_memory code

 - "treewide: fixup gfp_t printks" (Brendan Jackman)

   Use the special vprintf() gfp_t conversion in various places

 - "mm: Fix vmemmap optimization accounting and initialization" (Muchun
   Song)

   Fix several bugs in the vmemmap optimization, mainly around incorrect
   page accounting and memmap initialization in the DAX and memory
   hotplug paths. It also fixes pageblock migratetype initialization and
   struct page initialization for ZONE_DEVICE compound pages

 - "mm/damon: repost non-hotfix reviewed patches in damon/next tree"

   A sprinkle of unrelated minor bugfixes for DAMON

 - "mm: remove page_mapped()" (David Hildenbrand)

   Remove this function from the tree, replacing it with folio_mapped()

 - "mm/damon: let DAMON be paused and resumed" (SeongJae Park)

   Allow DAMON to be paused and resumed without losing its current state

 - "kasan: hw_tags: Disable tagging for stack and page-tables" (Muhammad
   Usama Anjum)

   Simplify and speed up kasan by removing its ineffective tagging of
   stacks and page tables

 - "mm/damon/reclaim,lru_sort: monitor all system rams by default"
   (SeongJae Park)

   Simplify deployment on diverse hardware like NUMA systems by updating
   DAMON_RECLAIM and DAMON_LRU_SORT to automatically monitor the
   physical address range covering all System RAM areas by default,
   replacing the overly restrictive behavior that only targeted the
   single largest memory block to save on negligible overhead

 - "mm/damon/sysfs: document filters/ directory as deprecated" (SeongJae
   Park)

   Update some DAMON docs

 - "mm: use spinlock guards for zone lock" (Dmitry Ilvokhin)

   Switch zone->lock handling over to using the guard() mechanisms

 - "mm/filemap: tighten mmap_miss hit accounting" (fujunjie)

   Fix a flaw where the mmap_miss counter over-credited page cache hits
   during fault-arounds and page-fault retries. This results in
   significant reduction of redundant synchronous mmap readahead I/O,
   drastically cutting down execution time and gigabytes read for sparse
   random or strided memory access workloads

 - "selftests/cgroup: Fix false positive failures in test_percpu_basic"
   (Li Wang)

   Fix a couple of false-positives in the cgroup kmem selftests

 - "mm/damon/reclaim: support monitoring intervals auto-tuning"
   (SeongJae Park)

   Add a new parameter to DAMON permitting DAMON_RECLAIM to
   automatically tune DAMON's sampling and aggregation intervals

 - "mm/damon/stat: add kdamond_pid parameter" (SeongJae Park)

   Change DAMON_STAT to provide the pid of its kdamond

 - "mm/kmemleak: dedupe verbose scan output" (Breno Leitao)

   Remove large amounts of duplicated backtraces from the verbose-mode
   kmemleak output

 - "mm: remove CONFIG_HAVE_BOOTMEM_INFO_NODE (Part 1)" (David
   Hildenbrand)

   Reduce our use of CONFIG_HAVE_BOOTMEM_INFO_NODE, with a view to
   removing it entirely in a later series

 - "mm/damon: validate min_region_size to be power of 2" (Liew Rui Yan)

   Prevent users from passing a non-power-of-2 value of `addr_unit', as
   this later results in undesirable behavior

 - "mm: document read_pages and simplify usage" (Frederick Mayle)

 - "tools/mm/page-types: Fix misc bugs" (Ye Liu)

   Fix three issues in tools/mm/page-types.c

 - "mm: misc cleanups from __GFP_UNMAPPED series" (Brendan Jackman)

   Implement several cleanups in the page allocator and related code

 - "mm, swap: swap table phase IV: unify allocation" (Kairui Song)

   Unify the allocation and charging of anon and shmem swap in folios,
   provides better synchronization, consolidates the metadata
   management, hence dropping the static array and map, and improves
   performance

 - "mm/damon: introduce data attributes monitoring" (SeongJae Park(

   Extend DAMON to monitor general data attributes other than accesses

 - "mm/vmalloc: free unused pages on vrealloc() shrink" (Shivam Kalra)

   Implement the TODO in vrealloc() to unmap and free unused pages when
   shrinking across a page boundary

 - "mm/damon: documentation and comment fixes" (niecheng)

 - "remove mmap_action success, error hooks" (Lorenzo Stoakes)

   Eliminate custom hooks from mmap_action by removing the problematic
   success_hook which allowed drivers to improperly access uninitialized
   VMAs. It replaces the error_hook with a simple error-code field and
   updates the memory char driver accordingly

 - "mm/damon: minor improvements for code readability and tests"
   (SeongJae Park)

 - "mm/damon: fix macro arguments and clarify quota goals doc" (Maksym
   Shcherba)

 - "userfaultfd: merge fs/userfaultfd.c into mm/userfaultfd.c" (Mike
   Rapoport)

 - "mm/mglru: improve reclaim loop and dirty folio" (Kairui Song and
   others)

   Clean up and slightly improves MGLRU's reclaim loop and dirty
   writeback handling. Large performance improvements are measured

 - "use vma locks for proc/pid/{smaps|numa_maps} reads" (Suren
   Baghdasaryan)

   Use per-vma locks when reading /proc/pid/smaps and numa_maps similar
   to reduce contention on central mmap_lock

 - "refactors thpsize_shmem_enabled_store() and thpsize_shmem_enabled_show()"
   (Ran Xiaokai)

   Some cleanup work in the THP code

 - "selftests/memfd: fix compilation warnings" (Konstantin Khorenko)

   Fix a few build glitches in the memfd selftest code.

 - "memcg: shrink obj_stock_pcp and cache multiple objcgs" (Shakeel
   Butt)

   Resolve a 68% performance regression caused by NUMA-node cache
   thrashing around struct obj_stock_pcp by shrinking its existing
   fields and expanding it into a multi-slot array that caches up to
   five obj_cgroup pointers per CPU, allowing per-node variants of the
   same memcg to coexist within a single 64-byte cache line.

 - "zram: writeback fixes" (Sergey Senozhatsky)

   address a couple of unrelated zram writeback issues

 - "mm: switch THP shrinker to list_lru" (Johannes Weiner)

   Resolve NUMA-awareness issues and streamlines callsite interaction by
   refactoring and extending the list_lru API to completely replace the
   complex, open-coded deferred split queue for Transparent Huge Pages

 - "mm: improve large folio readahead for exec memory" (Usama Arif)

   Improve large-folio readahead on systems like 64K-page arm64 by
   preventing the mmap_miss check from permanently disabling
   target-oriented VM_EXEC readahead, and by generalizing the
   force_thp_readahead gate to support mappings with any usefully large
   maximum folio order under the cache cap.

 - "userfaultfd/pagemap: pre-existing fixes" (Kiryl Shutsemau)

   Fix a bunch of minor issues in the userfaultfd/pagemap, all of which
   were flagged by Sashiko review of proposed new material

 - "mm/sparse-vmemmap: Provide generic vmemmap_set_pmd() and
   vmemmap_check_pmd()" (Muchun Song)

   Provide generic versions of these two functions so the four
   arch-specific implementations can be removed.

 - "mm/swap, PM: hibernate: fix swapoff race in uswsusp by pinning swap
   device" (Youngjun Park)

   Address a uswsusp-vs-swapoff race and reduces the swap device
   reference taking/releasing frequency.

 - "mm/hmm: A fix and a selftest" (Dev Jain)

* tag 'mm-stable-2026-06-18-09-26' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm: (321 commits)
  selftests/mm/hmm-tests: test pagemap reads of PMD device-private entries
  fs/proc/task_mmu: do not warn on seeing non-migration pmd entry
  lib/test_hmm: check alloc_page_vma() return value and handle OOM
  mm/compaction: cap compact_gap() at COMPACT_CLUSTER_MAX
  mm/swap: remove redundant swap device reference in alloc/free
  mm/swap, PM: hibernate: fix swapoff race in uswsusp by pinning swap device
  mm/filemap: use folio_next_index() for start
  vmalloc: fix NULL pointer dereference in is_vm_area_hugepages()
  sparc/mm: drop vmemmap_check_pmd helper and use generic code
  loongarch/mm: drop vmemmap_check_pmd helper and use generic code
  riscv/mm: drop vmemmap_pmd helpers and use generic code
  arm64/mm: drop vmemmap_pmd helpers and use generic code
  mm/sparse-vmemmap: provide generic vmemmap_set_pmd() and vmemmap_check_pmd()
  rust: page: mark Page::nid as inline
  userfaultfd: build __VMA_UFFD_FLAGS from config-gated masks
  userfaultfd: gate must_wait writability check on pte_present()
  mm/huge_memory: preserve pmd_swp_uffd_wp on device-private PMD downgrade
  fs/proc/task_mmu: fix hugetlb self-deadlock in pagemap_scan_pte_hole()
  fs/proc/task_mmu: use huge_page_size() in pagemap_scan_hugetlb_entry()
  fs/proc/task_mmu: fix make_uffd_wp_huge_pte() prot-update race
  ...
2026-06-19 10:14:34 -07:00

924 lines
20 KiB
C

// SPDX-License-Identifier: GPL-2.0-or-later
/*
* pseries Memory Hotplug infrastructure.
*
* Copyright (C) 2008 Badari Pulavarty, IBM Corporation
*/
#define pr_fmt(fmt) "pseries-hotplug-mem: " fmt
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/memblock.h>
#include <linux/memory.h>
#include <linux/memory_hotplug.h>
#include <linux/slab.h>
#include <asm/firmware.h>
#include <asm/machdep.h>
#include <asm/sparsemem.h>
#include <asm/fadump.h>
#include <asm/drmem.h>
#include "pseries.h"
static void dlpar_free_property(struct property *prop)
{
kfree(prop->name);
kfree(prop->value);
kfree(prop);
}
static struct property *dlpar_clone_property(struct property *prop,
u32 prop_size)
{
struct property *new_prop;
new_prop = kzalloc_obj(*new_prop);
if (!new_prop)
return NULL;
new_prop->name = kstrdup(prop->name, GFP_KERNEL);
new_prop->value = kzalloc(prop_size, GFP_KERNEL);
if (!new_prop->name || !new_prop->value) {
dlpar_free_property(new_prop);
return NULL;
}
memcpy(new_prop->value, prop->value, prop->length);
new_prop->length = prop_size;
of_property_set_flag(new_prop, OF_DYNAMIC);
return new_prop;
}
static bool find_aa_index(struct device_node *dr_node,
struct property *ala_prop,
const u32 *lmb_assoc, u32 *aa_index)
{
__be32 *assoc_arrays;
u32 new_prop_size;
struct property *new_prop;
int aa_arrays, aa_array_entries, aa_array_sz;
int i, index;
/*
* The ibm,associativity-lookup-arrays property is defined to be
* a 32-bit value specifying the number of associativity arrays
* followed by a 32-bitvalue specifying the number of entries per
* array, followed by the associativity arrays.
*/
assoc_arrays = ala_prop->value;
aa_arrays = be32_to_cpu(assoc_arrays[0]);
aa_array_entries = be32_to_cpu(assoc_arrays[1]);
aa_array_sz = aa_array_entries * sizeof(u32);
for (i = 0; i < aa_arrays; i++) {
index = (i * aa_array_entries) + 2;
if (memcmp(&assoc_arrays[index], &lmb_assoc[1], aa_array_sz))
continue;
*aa_index = i;
return true;
}
new_prop_size = ala_prop->length + aa_array_sz;
new_prop = dlpar_clone_property(ala_prop, new_prop_size);
if (!new_prop)
return false;
assoc_arrays = new_prop->value;
/* increment the number of entries in the lookup array */
assoc_arrays[0] = cpu_to_be32(aa_arrays + 1);
/* copy the new associativity into the lookup array */
index = aa_arrays * aa_array_entries + 2;
memcpy(&assoc_arrays[index], &lmb_assoc[1], aa_array_sz);
of_update_property(dr_node, new_prop);
/*
* The associativity lookup array index for this lmb is
* number of entries - 1 since we added its associativity
* to the end of the lookup array.
*/
*aa_index = be32_to_cpu(assoc_arrays[0]) - 1;
return true;
}
static int update_lmb_associativity_index(struct drmem_lmb *lmb)
{
struct device_node *parent, *lmb_node, *dr_node;
struct property *ala_prop;
const u32 *lmb_assoc;
u32 aa_index;
bool found;
parent = of_find_node_by_path("/");
if (!parent)
return -ENODEV;
lmb_node = dlpar_configure_connector(cpu_to_be32(lmb->drc_index),
parent);
of_node_put(parent);
if (!lmb_node)
return -EINVAL;
lmb_assoc = of_get_property(lmb_node, "ibm,associativity", NULL);
if (!lmb_assoc) {
dlpar_free_cc_nodes(lmb_node);
return -ENODEV;
}
update_numa_distance(lmb_node);
dr_node = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
if (!dr_node) {
dlpar_free_cc_nodes(lmb_node);
return -ENODEV;
}
ala_prop = of_find_property(dr_node, "ibm,associativity-lookup-arrays",
NULL);
if (!ala_prop) {
of_node_put(dr_node);
dlpar_free_cc_nodes(lmb_node);
return -ENODEV;
}
found = find_aa_index(dr_node, ala_prop, lmb_assoc, &aa_index);
of_node_put(dr_node);
dlpar_free_cc_nodes(lmb_node);
if (!found) {
pr_err("Could not find LMB associativity\n");
return -1;
}
lmb->aa_index = aa_index;
return 0;
}
static struct memory_block *lmb_to_memblock(struct drmem_lmb *lmb)
{
return memory_block_get(phys_to_block_id(lmb->base_addr));
}
static int get_lmb_range(u32 drc_index, int n_lmbs,
struct drmem_lmb **start_lmb,
struct drmem_lmb **end_lmb)
{
struct drmem_lmb *lmb, *start, *end;
struct drmem_lmb *limit;
start = NULL;
for_each_drmem_lmb(lmb) {
if (lmb->drc_index == drc_index) {
start = lmb;
break;
}
}
if (!start)
return -EINVAL;
end = &start[n_lmbs];
limit = &drmem_info->lmbs[drmem_info->n_lmbs];
if (end > limit)
return -EINVAL;
*start_lmb = start;
*end_lmb = end;
return 0;
}
static int dlpar_change_lmb_state(struct drmem_lmb *lmb, bool online)
{
struct memory_block *mem_block;
int rc;
mem_block = lmb_to_memblock(lmb);
if (!mem_block) {
pr_err("Failed memory block lookup for LMB 0x%x\n", lmb->drc_index);
return -EINVAL;
}
if (online && dev_offline(&mem_block->dev))
rc = device_online(&mem_block->dev);
else if (!online && !dev_offline(&mem_block->dev))
rc = device_offline(&mem_block->dev);
else
rc = 0;
memory_block_put(mem_block);
return rc;
}
static int dlpar_online_lmb(struct drmem_lmb *lmb)
{
return dlpar_change_lmb_state(lmb, true);
}
#ifdef CONFIG_MEMORY_HOTREMOVE
static int dlpar_offline_lmb(struct drmem_lmb *lmb)
{
return dlpar_change_lmb_state(lmb, false);
}
static int pseries_remove_memblock(unsigned long base, unsigned long memblock_size)
{
unsigned long start_pfn;
int sections_per_block;
int i;
start_pfn = base >> PAGE_SHIFT;
lock_device_hotplug();
if (!pfn_valid(start_pfn))
goto out;
sections_per_block = memory_block_size / MIN_MEMORY_BLOCK_SIZE;
for (i = 0; i < sections_per_block; i++) {
__remove_memory(base, MIN_MEMORY_BLOCK_SIZE);
base += MIN_MEMORY_BLOCK_SIZE;
}
out:
/* Update memory regions for memory remove */
memblock_remove(base, memblock_size);
unlock_device_hotplug();
return 0;
}
static int pseries_remove_mem_node(struct device_node *np)
{
int ret;
struct resource res;
/*
* Check to see if we are actually removing memory
*/
if (!of_node_is_type(np, "memory"))
return 0;
/*
* Find the base address and size of the memblock
*/
ret = of_address_to_resource(np, 0, &res);
if (ret)
return ret;
pseries_remove_memblock(res.start, resource_size(&res));
return 0;
}
static bool lmb_is_removable(struct drmem_lmb *lmb)
{
if ((lmb->flags & DRCONF_MEM_RESERVED) ||
!(lmb->flags & DRCONF_MEM_ASSIGNED))
return false;
#ifdef CONFIG_FA_DUMP
/*
* Don't hot-remove memory that falls in fadump boot memory area
* and memory that is reserved for capturing old kernel memory.
*/
if (is_fadump_memory_area(lmb->base_addr, memory_block_size_bytes()))
return false;
#endif
/* device_offline() will determine if we can actually remove this lmb */
return true;
}
static int dlpar_add_lmb(struct drmem_lmb *);
static int dlpar_remove_lmb(struct drmem_lmb *lmb)
{
struct memory_block *mem_block;
int rc;
if (!lmb_is_removable(lmb))
return -EINVAL;
mem_block = lmb_to_memblock(lmb);
if (mem_block == NULL)
return -EINVAL;
rc = dlpar_offline_lmb(lmb);
if (rc) {
memory_block_put(mem_block);
return rc;
}
__remove_memory(lmb->base_addr, memory_block_size);
memory_block_put(mem_block);
/* Update memory regions for memory remove */
memblock_remove(lmb->base_addr, memory_block_size);
invalidate_lmb_associativity_index(lmb);
lmb->flags &= ~DRCONF_MEM_ASSIGNED;
return 0;
}
static int dlpar_memory_remove_by_count(u32 lmbs_to_remove)
{
struct drmem_lmb *lmb;
int lmbs_reserved = 0;
int lmbs_available = 0;
int rc;
pr_info("Attempting to hot-remove %d LMB(s)\n", lmbs_to_remove);
if (lmbs_to_remove == 0)
return -EINVAL;
/* Validate that there are enough LMBs to satisfy the request */
for_each_drmem_lmb(lmb) {
if (lmb_is_removable(lmb))
lmbs_available++;
if (lmbs_available == lmbs_to_remove)
break;
}
if (lmbs_available < lmbs_to_remove) {
pr_info("Not enough LMBs available (%d of %d) to satisfy request\n",
lmbs_available, lmbs_to_remove);
return -EINVAL;
}
for_each_drmem_lmb(lmb) {
rc = dlpar_remove_lmb(lmb);
if (rc)
continue;
/* Mark this lmb so we can add it later if all of the
* requested LMBs cannot be removed.
*/
drmem_mark_lmb_reserved(lmb);
lmbs_reserved++;
if (lmbs_reserved == lmbs_to_remove)
break;
}
if (lmbs_reserved != lmbs_to_remove) {
pr_err("Memory hot-remove failed, adding LMB's back\n");
for_each_drmem_lmb(lmb) {
if (!drmem_lmb_reserved(lmb))
continue;
rc = dlpar_add_lmb(lmb);
if (rc)
pr_err("Failed to add LMB back, drc index %x\n",
lmb->drc_index);
drmem_remove_lmb_reservation(lmb);
lmbs_reserved--;
if (lmbs_reserved == 0)
break;
}
rc = -EINVAL;
} else {
for_each_drmem_lmb(lmb) {
if (!drmem_lmb_reserved(lmb))
continue;
dlpar_release_drc(lmb->drc_index);
pr_info("Memory at %llx was hot-removed\n",
lmb->base_addr);
drmem_remove_lmb_reservation(lmb);
lmbs_reserved--;
if (lmbs_reserved == 0)
break;
}
rc = 0;
}
return rc;
}
static int dlpar_memory_remove_by_index(u32 drc_index)
{
struct drmem_lmb *lmb;
int lmb_found;
int rc;
pr_debug("Attempting to hot-remove LMB, drc index %x\n", drc_index);
lmb_found = 0;
for_each_drmem_lmb(lmb) {
if (lmb->drc_index == drc_index) {
lmb_found = 1;
rc = dlpar_remove_lmb(lmb);
if (!rc)
dlpar_release_drc(lmb->drc_index);
break;
}
}
if (!lmb_found) {
pr_debug("Failed to look up LMB for drc index %x\n", drc_index);
rc = -EINVAL;
} else if (rc) {
pr_debug("Failed to hot-remove memory at %llx\n",
lmb->base_addr);
} else {
pr_debug("Memory at %llx was hot-removed\n", lmb->base_addr);
}
return rc;
}
static int dlpar_memory_remove_by_ic(u32 lmbs_to_remove, u32 drc_index)
{
struct drmem_lmb *lmb, *start_lmb, *end_lmb;
int rc;
pr_info("Attempting to hot-remove %u LMB(s) at %x\n",
lmbs_to_remove, drc_index);
if (lmbs_to_remove == 0)
return -EINVAL;
rc = get_lmb_range(drc_index, lmbs_to_remove, &start_lmb, &end_lmb);
if (rc)
return -EINVAL;
/*
* Validate that all LMBs in range are not reserved. Note that it
* is ok if they are !ASSIGNED since our goal here is to remove the
* LMB range, regardless of whether some LMBs were already removed
* by any other reason.
*
* This is a contrast to what is done in remove_by_count() where we
* check for both RESERVED and !ASSIGNED (via lmb_is_removable()),
* because we want to remove a fixed amount of LMBs in that function.
*/
for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) {
if (lmb->flags & DRCONF_MEM_RESERVED) {
pr_err("Memory at %llx (drc index %x) is reserved\n",
lmb->base_addr, lmb->drc_index);
return -EINVAL;
}
}
for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) {
/*
* dlpar_remove_lmb() will error out if the LMB is already
* !ASSIGNED, but this case is a no-op for us.
*/
if (!(lmb->flags & DRCONF_MEM_ASSIGNED))
continue;
rc = dlpar_remove_lmb(lmb);
if (rc)
break;
drmem_mark_lmb_reserved(lmb);
}
if (rc) {
pr_err("Memory indexed-count-remove failed, adding any removed LMBs\n");
for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) {
if (!drmem_lmb_reserved(lmb))
continue;
/*
* Setting the isolation state of an UNISOLATED/CONFIGURED
* device to UNISOLATE is a no-op, but the hypervisor can
* use it as a hint that the LMB removal failed.
*/
dlpar_unisolate_drc(lmb->drc_index);
rc = dlpar_add_lmb(lmb);
if (rc)
pr_err("Failed to add LMB, drc index %x\n",
lmb->drc_index);
drmem_remove_lmb_reservation(lmb);
}
rc = -EINVAL;
} else {
for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) {
if (!drmem_lmb_reserved(lmb))
continue;
dlpar_release_drc(lmb->drc_index);
pr_info("Memory at %llx (drc index %x) was hot-removed\n",
lmb->base_addr, lmb->drc_index);
drmem_remove_lmb_reservation(lmb);
}
}
return rc;
}
#else
static inline int pseries_remove_memblock(unsigned long base,
unsigned long memblock_size)
{
return -EOPNOTSUPP;
}
static inline int pseries_remove_mem_node(struct device_node *np)
{
return 0;
}
static int dlpar_remove_lmb(struct drmem_lmb *lmb)
{
return -EOPNOTSUPP;
}
static int dlpar_memory_remove_by_count(u32 lmbs_to_remove)
{
return -EOPNOTSUPP;
}
static int dlpar_memory_remove_by_index(u32 drc_index)
{
return -EOPNOTSUPP;
}
static int dlpar_memory_remove_by_ic(u32 lmbs_to_remove, u32 drc_index)
{
return -EOPNOTSUPP;
}
#endif /* CONFIG_MEMORY_HOTREMOVE */
static int dlpar_add_lmb(struct drmem_lmb *lmb)
{
unsigned long block_sz;
int nid, rc;
if (lmb->flags & DRCONF_MEM_ASSIGNED)
return -EINVAL;
rc = update_lmb_associativity_index(lmb);
if (rc) {
dlpar_release_drc(lmb->drc_index);
pr_err("Failed to configure LMB 0x%x\n", lmb->drc_index);
return rc;
}
block_sz = memory_block_size_bytes();
/* Find the node id for this LMB. Fake one if necessary. */
nid = of_drconf_to_nid_single(lmb);
if (nid < 0 || !node_possible(nid))
nid = first_online_node;
/* Add the memory */
rc = __add_memory(nid, lmb->base_addr, block_sz, MHP_MEMMAP_ON_MEMORY);
if (rc) {
pr_err("Failed to add LMB 0x%x to node %u", lmb->drc_index, nid);
invalidate_lmb_associativity_index(lmb);
return rc;
}
rc = dlpar_online_lmb(lmb);
if (rc) {
pr_err("Failed to online LMB 0x%x on node %u\n", lmb->drc_index, nid);
__remove_memory(lmb->base_addr, block_sz);
invalidate_lmb_associativity_index(lmb);
} else {
lmb->flags |= DRCONF_MEM_ASSIGNED;
}
return rc;
}
static int dlpar_memory_add_by_count(u32 lmbs_to_add)
{
struct drmem_lmb *lmb;
int lmbs_available = 0;
int lmbs_reserved = 0;
int rc;
pr_info("Attempting to hot-add %d LMB(s)\n", lmbs_to_add);
if (lmbs_to_add == 0)
return -EINVAL;
/* Validate that there are enough LMBs to satisfy the request */
for_each_drmem_lmb(lmb) {
if (lmb->flags & DRCONF_MEM_RESERVED)
continue;
if (!(lmb->flags & DRCONF_MEM_ASSIGNED))
lmbs_available++;
if (lmbs_available == lmbs_to_add)
break;
}
if (lmbs_available < lmbs_to_add)
return -EINVAL;
for_each_drmem_lmb(lmb) {
if (lmb->flags & DRCONF_MEM_ASSIGNED)
continue;
rc = dlpar_acquire_drc(lmb->drc_index);
if (rc)
continue;
rc = dlpar_add_lmb(lmb);
if (rc) {
dlpar_release_drc(lmb->drc_index);
continue;
}
/* Mark this lmb so we can remove it later if all of the
* requested LMBs cannot be added.
*/
drmem_mark_lmb_reserved(lmb);
lmbs_reserved++;
if (lmbs_reserved == lmbs_to_add)
break;
}
if (lmbs_reserved != lmbs_to_add) {
pr_err("Memory hot-add failed, removing any added LMBs\n");
for_each_drmem_lmb(lmb) {
if (!drmem_lmb_reserved(lmb))
continue;
rc = dlpar_remove_lmb(lmb);
if (rc)
pr_err("Failed to remove LMB, drc index %x\n",
lmb->drc_index);
else
dlpar_release_drc(lmb->drc_index);
drmem_remove_lmb_reservation(lmb);
lmbs_reserved--;
if (lmbs_reserved == 0)
break;
}
rc = -EINVAL;
} else {
for_each_drmem_lmb(lmb) {
if (!drmem_lmb_reserved(lmb))
continue;
pr_debug("Memory at %llx (drc index %x) was hot-added\n",
lmb->base_addr, lmb->drc_index);
drmem_remove_lmb_reservation(lmb);
lmbs_reserved--;
if (lmbs_reserved == 0)
break;
}
rc = 0;
}
return rc;
}
static int dlpar_memory_add_by_index(u32 drc_index)
{
struct drmem_lmb *lmb;
int rc, lmb_found;
pr_info("Attempting to hot-add LMB, drc index %x\n", drc_index);
lmb_found = 0;
for_each_drmem_lmb(lmb) {
if (lmb->drc_index == drc_index) {
lmb_found = 1;
rc = dlpar_acquire_drc(lmb->drc_index);
if (!rc) {
rc = dlpar_add_lmb(lmb);
if (rc)
dlpar_release_drc(lmb->drc_index);
}
break;
}
}
if (!lmb_found)
rc = -EINVAL;
if (rc)
pr_info("Failed to hot-add memory, drc index %x\n", drc_index);
else
pr_info("Memory at %llx (drc index %x) was hot-added\n",
lmb->base_addr, drc_index);
return rc;
}
static int dlpar_memory_add_by_ic(u32 lmbs_to_add, u32 drc_index)
{
struct drmem_lmb *lmb, *start_lmb, *end_lmb;
int rc;
pr_info("Attempting to hot-add %u LMB(s) at index %x\n",
lmbs_to_add, drc_index);
if (lmbs_to_add == 0)
return -EINVAL;
rc = get_lmb_range(drc_index, lmbs_to_add, &start_lmb, &end_lmb);
if (rc)
return -EINVAL;
/* Validate that the LMBs in this range are not reserved */
for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) {
/* Fail immediately if the whole range can't be hot-added */
if (lmb->flags & DRCONF_MEM_RESERVED) {
pr_err("Memory at %llx (drc index %x) is reserved\n",
lmb->base_addr, lmb->drc_index);
return -EINVAL;
}
}
for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) {
if (lmb->flags & DRCONF_MEM_ASSIGNED)
continue;
rc = dlpar_acquire_drc(lmb->drc_index);
if (rc)
break;
rc = dlpar_add_lmb(lmb);
if (rc) {
dlpar_release_drc(lmb->drc_index);
break;
}
drmem_mark_lmb_reserved(lmb);
}
if (rc) {
pr_err("Memory indexed-count-add failed, removing any added LMBs\n");
for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) {
if (!drmem_lmb_reserved(lmb))
continue;
rc = dlpar_remove_lmb(lmb);
if (rc)
pr_err("Failed to remove LMB, drc index %x\n",
lmb->drc_index);
else
dlpar_release_drc(lmb->drc_index);
drmem_remove_lmb_reservation(lmb);
}
rc = -EINVAL;
} else {
for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) {
if (!drmem_lmb_reserved(lmb))
continue;
pr_info("Memory at %llx (drc index %x) was hot-added\n",
lmb->base_addr, lmb->drc_index);
drmem_remove_lmb_reservation(lmb);
}
}
return rc;
}
int dlpar_memory(struct pseries_hp_errorlog *hp_elog)
{
u32 count, drc_index;
int rc;
lock_device_hotplug();
switch (hp_elog->action) {
case PSERIES_HP_ELOG_ACTION_ADD:
switch (hp_elog->id_type) {
case PSERIES_HP_ELOG_ID_DRC_COUNT:
count = be32_to_cpu(hp_elog->_drc_u.drc_count);
rc = dlpar_memory_add_by_count(count);
break;
case PSERIES_HP_ELOG_ID_DRC_INDEX:
drc_index = be32_to_cpu(hp_elog->_drc_u.drc_index);
rc = dlpar_memory_add_by_index(drc_index);
break;
case PSERIES_HP_ELOG_ID_DRC_IC:
count = be32_to_cpu(hp_elog->_drc_u.ic.count);
drc_index = be32_to_cpu(hp_elog->_drc_u.ic.index);
rc = dlpar_memory_add_by_ic(count, drc_index);
break;
default:
rc = -EINVAL;
break;
}
break;
case PSERIES_HP_ELOG_ACTION_REMOVE:
switch (hp_elog->id_type) {
case PSERIES_HP_ELOG_ID_DRC_COUNT:
count = be32_to_cpu(hp_elog->_drc_u.drc_count);
rc = dlpar_memory_remove_by_count(count);
break;
case PSERIES_HP_ELOG_ID_DRC_INDEX:
drc_index = be32_to_cpu(hp_elog->_drc_u.drc_index);
rc = dlpar_memory_remove_by_index(drc_index);
break;
case PSERIES_HP_ELOG_ID_DRC_IC:
count = be32_to_cpu(hp_elog->_drc_u.ic.count);
drc_index = be32_to_cpu(hp_elog->_drc_u.ic.index);
rc = dlpar_memory_remove_by_ic(count, drc_index);
break;
default:
rc = -EINVAL;
break;
}
break;
default:
pr_err("Invalid action (%d) specified\n", hp_elog->action);
rc = -EINVAL;
break;
}
if (!rc)
rc = drmem_update_dt();
unlock_device_hotplug();
return rc;
}
static int pseries_add_mem_node(struct device_node *np)
{
int ret;
struct resource res;
/*
* Check to see if we are actually adding memory
*/
if (!of_node_is_type(np, "memory"))
return 0;
/*
* Find the base and size of the memblock
*/
ret = of_address_to_resource(np, 0, &res);
if (ret)
return ret;
/*
* Update memory region to represent the memory add
*/
ret = memblock_add(res.start, resource_size(&res));
return (ret < 0) ? -EINVAL : 0;
}
static int pseries_memory_notifier(struct notifier_block *nb,
unsigned long action, void *data)
{
struct of_reconfig_data *rd = data;
int err = 0;
switch (action) {
case OF_RECONFIG_ATTACH_NODE:
err = pseries_add_mem_node(rd->dn);
break;
case OF_RECONFIG_DETACH_NODE:
err = pseries_remove_mem_node(rd->dn);
break;
case OF_RECONFIG_UPDATE_PROPERTY:
if (!strcmp(rd->dn->name,
"ibm,dynamic-reconfiguration-memory"))
drmem_update_lmbs(rd->prop);
}
return notifier_from_errno(err);
}
static struct notifier_block pseries_mem_nb = {
.notifier_call = pseries_memory_notifier,
};
static int __init pseries_memory_hotplug_init(void)
{
if (firmware_has_feature(FW_FEATURE_LPAR))
of_reconfig_notifier_register(&pseries_mem_nb);
return 0;
}
machine_device_initcall(pseries, pseries_memory_hotplug_init);