// SPDX-License-Identifier: GPL-2.0-only

/*

* linux/mm/memory_hotplug.c

*

* Copyright (C)

*/

#include <linux/stddef.h>

#include <linux/mm.h>

#include <linux/sched/signal.h>

#include <linux/swap.h>

#include <linux/interrupt.h>

#include <linux/pagemap.h>

#include <linux/compiler.h>

#include <linux/export.h>

#include <linux/pagevec.h>

#include <linux/writeback.h>

#include <linux/slab.h>

#include <linux/sysctl.h>

#include <linux/cpu.h>

#include <linux/memory.h>

#include <linux/memremap.h>

#include <linux/memory_hotplug.h>

#include <linux/highmem.h>

#include <linux/vmalloc.h>

#include <linux/ioport.h>

#include <linux/delay.h>

#include <linux/migrate.h>

#include <linux/page-isolation.h>

#include <linux/pfn.h>

#include <linux/suspend.h>

#include <linux/mm_inline.h>

#include <linux/firmware-map.h>

#include <linux/stop_machine.h>

#include <linux/hugetlb.h>

#include <linux/memblock.h>

#include <linux/compaction.h>

#include <linux/rmap.h>

#include <asm/tlbflush.h>

#include "internal.h"

#include "shuffle.h"

/*

* online_page_callback contains pointer to current page onlining function.

* Initially it is generic_online_page(). If it is required it could be

* changed by calling set_online_page_callback() for callback registration

* and restore_online_page_callback() for generic callback restore.

*/

static void generic_online_page(struct page *page, unsigned int order);

static online_page_callback_t online_page_callback = generic_online_page;

static DEFINE_MUTEX(online_page_callback_lock);

DEFINE_STATIC_PERCPU_RWSEM(mem_hotplug_lock);

void get_online_mems(void)

{

percpu_down_read(&mem_hotplug_lock);

}

void put_online_mems(void)

{

percpu_up_read(&mem_hotplug_lock);

}

bool movable_node_enabled = false;

#ifndef CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE

bool memhp_auto_online;

#else

bool memhp_auto_online = true;

#endif

EXPORT_SYMBOL_GPL(memhp_auto_online);

static int __init setup_memhp_default_state(char *str)

{

if (!strcmp(str, "online"))

memhp_auto_online = true;

else if (!strcmp(str, "offline"))

memhp_auto_online = false;

return 1;

}

__setup("memhp_default_state=", setup_memhp_default_state);

void mem_hotplug_begin(void)

{

cpus_read_lock();

percpu_down_write(&mem_hotplug_lock);

}

void mem_hotplug_done(void)

{

percpu_up_write(&mem_hotplug_lock);

cpus_read_unlock();

}

u64 max_mem_size = U64_MAX;

/* add this memory to iomem resource */

static struct resource *register_memory_resource(u64 start, u64 size)

{

struct resource *res;

unsigned long flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;

char *resource_name = "System RAM";

if (start + size > max_mem_size)

return ERR_PTR(-E2BIG);

/*

* Request ownership of the new memory range. This might be

* a child of an existing resource that was present but

* not marked as busy.

*/

res = __request_region(&iomem_resource, start, size,

resource_name, flags);

if (!res) {

pr_debug("Unable to reserve System RAM region: %016llx->%016llx\n",

start, start + size);

return ERR_PTR(-EEXIST);

}

return res;

}

static void release_memory_resource(struct resource *res)

{

if (!res)

return;

release_resource(res);

kfree(res);

}

#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE

void get_page_bootmem(unsigned long info, struct page *page,

unsigned long type)

{

page->freelist = (void *)type;

SetPagePrivate(page);

set_page_private(page, info);

page_ref_inc(page);

}

void put_page_bootmem(struct page *page)

{

unsigned long type;

type = (unsigned long) page->freelist;

BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||

type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);

if (page_ref_dec_return(page) == 1) {

page->freelist = NULL;

ClearPagePrivate(page);

set_page_private(page, 0);

INIT_LIST_HEAD(&page->lru);

free_reserved_page(page);

}

}

#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE

#ifndef CONFIG_SPARSEMEM_VMEMMAP

static void register_page_bootmem_info_section(unsigned long start_pfn)

{

unsigned long mapsize, section_nr, i;

struct mem_section *ms;

struct page *page, *memmap;

struct mem_section_usage *usage;

section_nr = pfn_to_section_nr(start_pfn);

ms = __nr_to_section(section_nr);

/* Get section's memmap address */

memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);

/*

* Get page for the memmap's phys address

* XXX: need more consideration for sparse_vmemmap...

*/

page = virt_to_page(memmap);

mapsize = sizeof(struct page) * PAGES_PER_SECTION;

mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;

/* remember memmap's page */

for (i = 0; i < mapsize; i++, page++)

get_page_bootmem(section_nr, page, SECTION_INFO);

usage = ms->usage;

page = virt_to_page(usage);

mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;

for (i = 0; i < mapsize; i++, page++)

get_page_bootmem(section_nr, page, MIX_SECTION_INFO);

}

#else /* CONFIG_SPARSEMEM_VMEMMAP */

static void register_page_bootmem_info_section(unsigned long start_pfn)

{

unsigned long mapsize, section_nr, i;

struct mem_section *ms;

struct page *page, *memmap;

struct mem_section_usage *usage;

section_nr = pfn_to_section_nr(start_pfn);

ms = __nr_to_section(section_nr);

memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);

register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);

usage = ms->usage;

page = virt_to_page(usage);

mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;

for (i = 0; i < mapsize; i++, page++)

get_page_bootmem(section_nr, page, MIX_SECTION_INFO);

}

#endif /* !CONFIG_SPARSEMEM_VMEMMAP */

void __init register_page_bootmem_info_node(struct pglist_data *pgdat)

{

unsigned long i, pfn, end_pfn, nr_pages;

int node = pgdat->node_id;

struct page *page;

nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;

page = virt_to_page(pgdat);

for (i = 0; i < nr_pages; i++, page++)

get_page_bootmem(node, page, NODE_INFO);

pfn = pgdat->node_start_pfn;

end_pfn = pgdat_end_pfn(pgdat);

/* register section info */

for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {

/*

* Some platforms can assign the same pfn to multiple nodes - on

* node0 as well as nodeN. To avoid registering a pfn against

* multiple nodes we check that this pfn does not already

* reside in some other nodes.

*/

if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node))

register_page_bootmem_info_section(pfn);

}

}

#endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */

static int check_pfn_span(unsigned long pfn, unsigned long nr_pages,

const char *reason)

{

/*

* Disallow all operations smaller than a sub-section and only

* allow operations smaller than a section for

* SPARSEMEM_VMEMMAP. Note that check_hotplug_memory_range()

* enforces a larger memory_block_size_bytes() granularity for

* memory that will be marked online, so this check should only

* fire for direct arch_{add,remove}_memory() users outside of

* add_memory_resource().

*/

unsigned long min_align;

if (IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP))

min_align = PAGES_PER_SUBSECTION;

else

min_align = PAGES_PER_SECTION;

if (!IS_ALIGNED(pfn, min_align)

|| !IS_ALIGNED(nr_pages, min_align)) {

WARN(1, "Misaligned __%s_pages start: %#lx end: #%lx\n",

reason, pfn, pfn + nr_pages - 1);

return -EINVAL;

}

return 0;

}

/*

* Reasonably generic function for adding memory. It is

* expected that archs that support memory hotplug will

* call this function after deciding the zone to which to

* add the new pages.

*/

int __ref __add_pages(int nid, unsigned long pfn, unsigned long nr_pages,

struct mhp_restrictions *restrictions)

{

int err;

unsigned long nr, start_sec, end_sec;

struct vmem_altmap *altmap = restrictions->altmap;

if (altmap) {

/*

* Validate altmap is within bounds of the total request

*/

if (altmap->base_pfn != pfn

|| vmem_altmap_offset(altmap) > nr_pages) {

pr_warn_once("memory add fail, invalid altmap\n");

return -EINVAL;

}

altmap->alloc = 0;

}

err = check_pfn_span(pfn, nr_pages, "add");

if (err)

return err;

start_sec = pfn_to_section_nr(pfn);

end_sec = pfn_to_section_nr(pfn + nr_pages - 1);

for (nr = start_sec; nr <= end_sec; nr++) {

unsigned long pfns;

pfns = min(nr_pages, PAGES_PER_SECTION

- (pfn & ~PAGE_SECTION_MASK));

err = sparse_add_section(nid, pfn, pfns, altmap);

if (err)

break;

pfn += pfns;

nr_pages -= pfns;

cond_resched();

}

vmemmap_populate_print_last();

return err;

}

/* find the smallest valid pfn in the range [start_pfn, end_pfn) */

static unsigned long find_smallest_section_pfn(int nid, struct zone *zone,

unsigned long start_pfn,

unsigned long end_pfn)

{

for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SUBSECTION) {

if (unlikely(!pfn_valid(start_pfn)))

continue;

if (unlikely(pfn_to_nid(start_pfn) != nid))

continue;

if (zone && zone != page_zone(pfn_to_page(start_pfn)))

continue;

return start_pfn;

}

return 0;

}

/* find the biggest valid pfn in the range [start_pfn, end_pfn). */

static unsigned long find_biggest_section_pfn(int nid, struct zone *zone,

unsigned long start_pfn,

unsigned long end_pfn)

{

unsigned long pfn;

/* pfn is the end pfn of a memory section. */

pfn = end_pfn - 1;

for (; pfn >= start_pfn; pfn -= PAGES_PER_SUBSECTION) {

if (unlikely(!pfn_valid(pfn)))

continue;

if (unlikely(pfn_to_nid(pfn) != nid))

continue;

if (zone && zone != page_zone(pfn_to_page(pfn)))

continue;

return pfn;

}

return 0;

}

static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,

unsigned long end_pfn)

{

unsigned long zone_start_pfn = zone->zone_start_pfn;

unsigned long z = zone_end_pfn(zone); /* zone_end_pfn namespace clash */

unsigned long zone_end_pfn = z;

unsigned long pfn;

int nid = zone_to_nid(zone);

zone_span_writelock(zone);

if (zone_start_pfn == start_pfn) {

/*

* If the section is smallest section in the zone, it need

* shrink zone->zone_start_pfn and zone->zone_spanned_pages.

* In this case, we find second smallest valid mem_section

* for shrinking zone.

*/

pfn = find_smallest_section_pfn(nid, zone, end_pfn,

zone_end_pfn);

if (pfn) {

zone->zone_start_pfn = pfn;

zone->spanned_pages = zone_end_pfn - pfn;

}

} else if (zone_end_pfn == end_pfn) {

/*

* If the section is biggest section in the zone, it need

* shrink zone->spanned_pages.

* In this case, we find second biggest valid mem_section for

* shrinking zone.

*/

pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn,

start_pfn);

if (pfn)

zone->spanned_pages = pfn - zone_start_pfn + 1;

}

/*

* The section is not biggest or smallest mem_section in the zone, it

* only creates a hole in the zone. So in this case, we need not

* change the zone. But perhaps, the zone has only hole data. Thus

* it check the zone has only hole or not.

*/

pfn = zone_start_pfn;

for (; pfn < zone_end_pfn; pfn += PAGES_PER_SUBSECTION) {

if (unlikely(!pfn_valid(pfn)))

continue;

if (page_zone(pfn_to_page(pfn)) != zone)

continue;

/* Skip range to be removed */

if (pfn >= start_pfn && pfn < end_pfn)

continue;

/* If we find valid section, we have nothing to do */

zone_span_writeunlock(zone);

return;

}

/* The zone has no valid section */

zone->zone_start_pfn = 0;

zone->spanned_pages = 0;

zone_span_writeunlock(zone);

}

static void update_pgdat_span(struct pglist_data *pgdat)

{

unsigned long node_start_pfn = 0, node_end_pfn = 0;

struct zone *zone;

for (zone = pgdat->node_zones;

zone < pgdat->node_zones + MAX_NR_ZONES; zone++) {

unsigned long zone_end_pfn = zone->zone_start_pfn +

zone->spanned_pages;

/* No need to lock the zones, they can't change. */

if (zone_end_pfn > node_end_pfn)

node_end_pfn = zone_end_pfn;

if (zone->zone_start_pfn < node_start_pfn)

node_start_pfn = zone->zone_start_pfn;

}

pgdat->node_start_pfn = node_start_pfn;

pgdat->node_spanned_pages = node_end_pfn - node_start_pfn;

}

static void __remove_zone(struct zone *zone, unsigned long start_pfn,

unsigned long nr_pages)

{

struct pglist_data *pgdat = zone->zone_pgdat;

unsigned long flags;

pgdat_resize_lock(zone->zone_pgdat, &flags);

shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);

update_pgdat_span(pgdat);

pgdat_resize_unlock(zone->zone_pgdat, &flags);

}

static void __remove_section(struct zone *zone, unsigned long pfn,

unsigned long nr_pages, unsigned long map_offset,

struct vmem_altmap *altmap)

{

struct mem_section *ms = __nr_to_section(pfn_to_section_nr(pfn));

if (WARN_ON_ONCE(!valid_section(ms)))

return;

__remove_zone(zone, pfn, nr_pages);

sparse_remove_section(ms, pfn, nr_pages, map_offset, altmap);

}

/**

* __remove_pages() - remove sections of pages from a zone

* @zone: zone from which pages need to be removed

* @pfn: starting pageframe (must be aligned to start of a section)

* @nr_pages: number of pages to remove (must be multiple of section size)

* @altmap: alternative device page map or %NULL if default memmap is used

*

* Generic helper function to remove section mappings and sysfs entries

* for the section of the memory we are removing. Caller needs to make

* sure that pages are marked reserved and zones are adjust properly by

* calling offline_pages().

*/

void __remove_pages(struct zone *zone, unsigned long pfn,

unsigned long nr_pages, struct vmem_altmap *altmap)

{

unsigned long map_offset = 0;

unsigned long nr, start_sec, end_sec;

map_offset = vmem_altmap_offset(altmap);

clear_zone_contiguous(zone);

if (check_pfn_span(pfn, nr_pages, "remove"))

return;

start_sec = pfn_to_section_nr(pfn);

end_sec = pfn_to_section_nr(pfn + nr_pages - 1);

for (nr = start_sec; nr <= end_sec; nr++) {

unsigned long pfns;

cond_resched();

pfns = min(nr_pages, PAGES_PER_SECTION

- (pfn & ~PAGE_SECTION_MASK));

__remove_section(zone, pfn, pfns, map_offset, altmap);

pfn += pfns;

nr_pages -= pfns;

map_offset = 0;

}

set_zone_contiguous(zone);

}

int set_online_page_callback(online_page_callback_t callback)

{

int rc = -EINVAL;

get_online_mems();

mutex_lock(&online_page_callback_lock);

if (online_page_callback == generic_online_page) {

online_page_callback = callback;

rc = 0;

}

mutex_unlock(&online_page_callback_lock);

put_online_mems();

return rc;

}

EXPORT_SYMBOL_GPL(set_online_page_callback);

int restore_online_page_callback(online_page_callback_t callback)

{

int rc = -EINVAL;

get_online_mems();

mutex_lock(&online_page_callback_lock);

if (online_page_callback == callback) {

online_page_callback = generic_online_page;

rc = 0;

}

mutex_unlock(&online_page_callback_lock);

put_online_mems();

return rc;

}

EXPORT_SYMBOL_GPL(restore_online_page_callback);

void __online_page_set_limits(struct page *page)

{

}

EXPORT_SYMBOL_GPL(__online_page_set_limits);

void __online_page_increment_counters(struct page *page)

{

adjust_managed_page_count(page, 1);

}

EXPORT_SYMBOL_GPL(__online_page_increment_counters);

void __online_page_free(struct page *page)

{

__free_reserved_page(page);

}

EXPORT_SYMBOL_GPL(__online_page_free);

static void generic_online_page(struct page *page, unsigned int order)

{

kernel_map_pages(page, 1 << order, 1);

__free_pages_core(page, order);

totalram_pages_add(1UL << order);

#ifdef CONFIG_HIGHMEM

if (PageHighMem(page))

totalhigh_pages_add(1UL << order);

#endif

}

static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages,

void *arg)

{

const unsigned long end_pfn = start_pfn + nr_pages;

unsigned long pfn;

int order;

/*

* Online the pages. The callback might decide to keep some pages

* PG_reserved (to add them to the buddy later), but we still account

* them as being online/belonging to this zone ("present").

*/

for (pfn = start_pfn; pfn < end_pfn; pfn += 1ul << order) {

order = min(MAX_ORDER - 1, get_order(PFN_PHYS(end_pfn - pfn)));

/* __free_pages_core() wants pfns to be aligned to the order */

if (WARN_ON_ONCE(!IS_ALIGNED(pfn, 1ul << order)))

order = 0;

(*online_page_callback)(pfn_to_page(pfn), order);

}

/* mark all involved sections as online */

online_mem_sections(start_pfn, end_pfn);

*(unsigned long *)arg += nr_pages;

return 0;

}

/* check which state of node_states will be changed when online memory */

static void node_states_check_changes_online(unsigned long nr_pages,

struct zone *zone, struct memory_notify *arg)

{

int nid = zone_to_nid(zone);

arg->status_change_nid = NUMA_NO_NODE;

arg->status_change_nid_normal = NUMA_NO_NODE;

arg->status_change_nid_high = NUMA_NO_NODE;

if (!node_state(nid, N_MEMORY))

arg->status_change_nid = nid;

if (zone_idx(zone) <= ZONE_NORMAL && !node_state(nid, N_NORMAL_MEMORY))

arg->status_change_nid_normal = nid;

#ifdef CONFIG_HIGHMEM

if (zone_idx(zone) <= ZONE_HIGHMEM && !node_state(nid, N_HIGH_MEMORY))

arg->status_change_nid_high = nid;

#endif

}

static void node_states_set_node(int node, struct memory_notify *arg)

{

if (arg->status_change_nid_normal >= 0)

node_set_state(node, N_NORMAL_MEMORY);

if (arg->status_change_nid_high >= 0)

node_set_state(node, N_HIGH_MEMORY);

if (arg->status_change_nid >= 0)

node_set_state(node, N_MEMORY);

}

static void __meminit resize_zone_range(struct zone *zone, unsigned long start_pfn,

unsigned long nr_pages)

{

unsigned long old_end_pfn = zone_end_pfn(zone);

if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn)

zone->zone_start_pfn = start_pfn;

zone->spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - zone->zone_start_pfn;

}

static void __meminit resize_pgdat_range(struct pglist_data *pgdat, unsigned long start_pfn,

unsigned long nr_pages)

{

unsigned long old_end_pfn = pgdat_end_pfn(pgdat);

if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn)

pgdat->node_start_pfn = start_pfn;

pgdat->node_spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - pgdat->node_start_pfn;

}

/*

* Associate the pfn range with the given zone, initializing the memmaps

* and resizing the pgdat/zone data to span the added pages. After this

* call, all affected pages are PG_reserved.

*/

void __ref move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn,

unsigned long nr_pages, struct vmem_altmap *altmap)

{

struct pglist_data *pgdat = zone->zone_pgdat;

int nid = pgdat->node_id;

unsigned long flags;

clear_zone_contiguous(zone);

/* TODO Huh pgdat is irqsave while zone is not. It used to be like that before */

pgdat_resize_lock(pgdat, &flags);

zone_span_writelock(zone);

if (zone_is_empty(zone))

init_currently_empty_zone(zone, start_pfn, nr_pages);

resize_zone_range(zone, start_pfn, nr_pages);

zone_span_writeunlock(zone);

resize_pgdat_range(pgdat, start_pfn, nr_pages);

pgdat_resize_unlock(pgdat, &flags);

/*

* TODO now we have a visible range of pages which are not associated

* with their zone properly. Not nice but set_pfnblock_flags_mask

* expects the zone spans the pfn range. All the pages in the range

* are reserved so nobody should be touching them so we should be safe

*/

memmap_init_zone(nr_pages, nid, zone_idx(zone), start_pfn,

MEMMAP_HOTPLUG, altmap);

set_zone_contiguous(zone);

}

/*

* Returns a default kernel memory zone for the given pfn range.

* If no kernel zone covers this pfn range it will automatically go

* to the ZONE_NORMAL.

*/

static struct zone *default_kernel_zone_for_pfn(int nid, unsigned long start_pfn,

unsigned long nr_pages)

{

struct pglist_data *pgdat = NODE_DATA(nid);

int zid;

for (zid = 0; zid <= ZONE_NORMAL; zid++) {

struct zone *zone = &pgdat->node_zones[zid];

if (zone_intersects(zone, start_pfn, nr_pages))

return zone;

}

return &pgdat->node_zones[ZONE_NORMAL];

}

static inline struct zone *default_zone_for_pfn(int nid, unsigned long start_pfn,

unsigned long nr_pages)

{

struct zone *kernel_zone = default_kernel_zone_for_pfn(nid, start_pfn,

nr_pages);

struct zone *movable_zone = &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];

bool in_kernel = zone_intersects(kernel_zone, start_pfn, nr_pages);

bool in_movable = zone_intersects(movable_zone, start_pfn, nr_pages);

/*

* We inherit the existing zone in a simple case where zones do not

* overlap in the given range

*/

if (in_kernel ^ in_movable)

return (in_kernel) ? kernel_zone : movable_zone;

/*

* If the range doesn't belong to any zone or two zones overlap in the

* given range then we use movable zone only if movable_node is

* enabled because we always online to a kernel zone by default.

*/

return movable_node_enabled ? movable_zone : kernel_zone;

}

struct zone * zone_for_pfn_range(int online_type, int nid, unsigned start_pfn,

unsigned long nr_pages)

{

if (online_type == MMOP_ONLINE_KERNEL)

return default_kernel_zone_for_pfn(nid, start_pfn, nr_pages);

if (online_type == MMOP_ONLINE_MOVABLE)

return &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];

return default_zone_for_pfn(nid, start_pfn, nr_pages);

}

int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type)

{

unsigned long flags;

unsigned long onlined_pages = 0;

struct zone *zone;

int need_zonelists_rebuild = 0;

int nid;

int ret;

struct memory_notify arg;

struct memory_block *mem;

mem_hotplug_begin();

/*

* We can't use pfn_to_nid() because nid might be stored in struct page

* which is not yet initialized. Instead, we find nid from memory block.

*/

mem = find_memory_block(__pfn_to_section(pfn));

nid = mem->nid;

put_device(&mem->dev);

/* associate pfn range with the zone */

zone = zone_for_pfn_range(online_type, nid, pfn, nr_pages);

move_pfn_range_to_zone(zone, pfn, nr_pages, NULL);

arg.start_pfn = pfn;

arg.nr_pages = nr_pages;

node_states_check_changes_online(nr_pages, zone, &arg);

ret = memory_notify(MEM_GOING_ONLINE, &arg);

ret = notifier_to_errno(ret);

if (ret)

goto failed_addition;

/*

* If this zone is not populated, then it is not in zonelist.

* This means the page allocator ignores this zone.

* So, zonelist must be updated after online.

*/

if (!populated_zone(zone)) {

need_zonelists_rebuild = 1;

setup_zone_pageset(zone);

}

ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages,

online_pages_range);

if (ret) {

/* not a single memory resource was applicable */

if (need_zonelists_rebuild)

zone_pcp_reset(zone);

goto failed_addition;

}

zone->present_pages += onlined_pages;

pgdat_resize_lock(zone->zone_pgdat, &flags);

zone->zone_pgdat->node_present_pages += onlined_pages;

pgdat_resize_unlock(zone->zone_pgdat, &flags);

shuffle_zone(zone);

node_states_set_node(nid, &arg);

if (need_zonelists_rebuild)

build_all_zonelists(NULL);

else

zone_pcp_update(zone);

init_per_zone_wmark_min();

kswapd_run(nid);

kcompactd_run(nid);

vm_total_pages = nr_free_pagecache_pages();

writeback_set_ratelimit();

memory_notify(MEM_ONLINE, &arg);

mem_hotplug_done();

return 0;

failed_addition:

pr_debug("online_pages [mem %#010llx-%#010llx] failed\n",

(unsigned long long) pfn << PAGE_SHIFT,

(((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1);

memory_notify(MEM_CANCEL_ONLINE, &arg);

mem_hotplug_done();

return ret;

}

#endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */

static void reset_node_present_pages(pg_data_t *pgdat)

{

struct zone *z;

for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)

z->present_pages = 0;

pgdat->node_present_pages = 0;

}

/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */

static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start)

{

struct pglist_data *pgdat;

unsigned long start_pfn = PFN_DOWN(start);

pgdat = NODE_DATA(nid);

if (!pgdat) {

pgdat = arch_alloc_nodedata(nid);

if (!pgdat)

return NULL;

pgdat->per_cpu_nodestats =

alloc_percpu(struct per_cpu_nodestat);

arch_refresh_nodedata(nid, pgdat);

} else {

int cpu;

/*

* Reset the nr_zones, order and classzone_idx before reuse.

* Note that kswapd will init kswapd_classzone_idx properly

* when it starts in the near future.

*/

pgdat->nr_zones = 0;

pgdat->kswapd_order = 0;

pgdat->kswapd_classzone_idx = 0;

for_each_online_cpu(cpu) {

struct per_cpu_nodestat *p;

p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu);

memset(p, 0, sizeof(*p));

}

}

/* we can use NODE_DATA(nid) from here */

pgdat->node_id = nid;

pgdat->node_start_pfn = start_pfn;

/* init node's zones as empty zones, we don't have any present pages.*/

free_area_init_core_hotplug(nid);

/*

* The node we allocated has no zone fallback lists. For avoiding

* to access not-initialized zonelist, build here.

*/

build_all_zonelists(pgdat);

/*

* When memory is hot-added, all the memory is in offline state. So

* clear all zones' present_pages because they will be updated in

* online_pages() and offline_pages().

*/

reset_node_managed_pages(pgdat);

reset_node_present_pages(pgdat);

return pgdat;

}

static void rollback_node_hotadd(int nid)

{

pg_data_t *pgdat = NODE_DATA(nid);

arch_refresh_nodedata(nid, NULL);

free_percpu(pgdat->per_cpu_nodestats);

arch_free_nodedata(pgdat);

}

/**

* try_online_node - online a node if offlined

* @nid: the node ID

* @start: start addr of the node

* @set_node_online: Whether we want to online the node

* called by cpu_up() to online a node without onlined memory.

*

* Returns:

* 1 -> a new node has been allocated

* 0 -> the node is already online

* -ENOMEM -> the node could not be allocated

*/

static int __try_online_node(int nid, u64 start, bool set_node_online)

{

pg_data_t *pgdat;

int ret = 1;

if (node_online(nid))

return 0;

pgdat = hotadd_new_pgdat(nid, start);

if (!pgdat) {

pr_err("Cannot online node %d due to NULL pgdat\n", nid);

ret = -ENOMEM;

goto out;

}

if (set_node_online) {

node_set_online(nid);

ret = register_one_node(nid);

BUG_ON(ret);

}

out:

return ret;

}

/*

* Users of this function always want to online/register the node

*/

int try_online_node(int nid)

{

int ret;

mem_hotplug_begin();

ret = __try_online_node(nid, 0, true);

mem_hotplug_done();

return ret;

}

static int check_hotplug_memory_range(u64 start, u64 size)

{

/* memory range must be block size aligned */

if (!size || !IS_ALIGNED(start, memory_block_size_bytes()) ||

!IS_ALIGNED(size, memory_block_size_bytes())) {

pr_err("Block size [%#lx] unaligned hotplug range: start %#llx, size %#llx",

memory_block_size_bytes(), start, size);

return -EINVAL;

}

return 0;

}

static int online_memory_block(struct memory_block *mem, void *arg)

{

return device_online(&mem->dev);

}