Blame view
drivers/dax/kmem.c
5.69 KB
c221c0b03 device-dax: "Hotp... |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
// SPDX-License-Identifier: GPL-2.0 /* Copyright(c) 2016-2019 Intel Corporation. All rights reserved. */ #include <linux/memremap.h> #include <linux/pagemap.h> #include <linux/memory.h> #include <linux/module.h> #include <linux/device.h> #include <linux/pfn_t.h> #include <linux/slab.h> #include <linux/dax.h> #include <linux/fs.h> #include <linux/mm.h> #include <linux/mman.h> #include "dax-private.h" #include "bus.h" |
8a725e469 device-dax: add m... |
16 17 18 19 |
/* Memory resource name used for add_memory_driver_managed(). */ static const char *kmem_name; /* Set if any memory will remain added when the driver will be unloaded. */ static bool any_hotremove_failed; |
60e93dc09 device-dax: add d... |
20 |
static int dax_kmem_range(struct dev_dax *dev_dax, int i, struct range *r) |
59bc8d10d device-dax/kmem: ... |
21 |
{ |
60e93dc09 device-dax: add d... |
22 23 |
struct dev_dax_range *dax_range = &dev_dax->ranges[i]; struct range *range = &dax_range->range; |
59bc8d10d device-dax/kmem: ... |
24 25 |
/* memory-block align the hotplug range */ |
60e93dc09 device-dax: add d... |
26 27 28 29 30 31 32 33 |
r->start = ALIGN(range->start, memory_block_size_bytes()); r->end = ALIGN_DOWN(range->end + 1, memory_block_size_bytes()) - 1; if (r->start >= r->end) { r->start = range->start; r->end = range->end; return -ENOSPC; } return 0; |
59bc8d10d device-dax/kmem: ... |
34 |
} |
a455aa72f device-dax/kmem: ... |
35 36 37 38 |
struct dax_kmem_data { const char *res_name; struct resource *res[]; }; |
f11cf813d device-dax: intro... |
39 |
static int dev_dax_kmem_probe(struct dev_dax *dev_dax) |
c221c0b03 device-dax: "Hotp... |
40 |
{ |
f11cf813d device-dax: intro... |
41 |
struct device *dev = &dev_dax->dev; |
a455aa72f device-dax/kmem: ... |
42 43 |
struct dax_kmem_data *data; int rc = -ENOMEM; |
60e93dc09 device-dax: add d... |
44 |
int i, mapped = 0; |
c221c0b03 device-dax: "Hotp... |
45 |
int numa_node; |
c221c0b03 device-dax: "Hotp... |
46 47 48 49 50 51 52 53 54 |
/* * Ensure good NUMA information for the persistent memory. * Without this check, there is a risk that slow memory * could be mixed in a node with faster memory, causing * unavoidable performance issues. */ numa_node = dev_dax->target_node; if (numa_node < 0) { |
f5516ec5e device-dax: make ... |
55 56 57 |
dev_warn(dev, "rejecting DAX region with invalid node: %d ", numa_node); |
c221c0b03 device-dax: "Hotp... |
58 59 |
return -EINVAL; } |
a455aa72f device-dax/kmem: ... |
60 61 |
data = kzalloc(sizeof(*data) + sizeof(struct resource *) * dev_dax->nr_range, GFP_KERNEL); if (!data) |
60858c00e device-dax: don't... |
62 |
return -ENOMEM; |
a455aa72f device-dax/kmem: ... |
63 64 65 |
data->res_name = kstrdup(dev_name(dev), GFP_KERNEL); if (!data->res_name) goto err_res_name; |
60e93dc09 device-dax: add d... |
66 67 68 |
for (i = 0; i < dev_dax->nr_range; i++) { struct resource *res; struct range range; |
c221c0b03 device-dax: "Hotp... |
69 |
|
60e93dc09 device-dax: add d... |
70 71 72 73 74 75 76 |
rc = dax_kmem_range(dev_dax, i, &range); if (rc) { dev_info(dev, "mapping%d: %#llx-%#llx too small after alignment ", i, range.start, range.end); continue; } |
c221c0b03 device-dax: "Hotp... |
77 |
|
60e93dc09 device-dax: add d... |
78 |
/* Region is permanently reserved if hotremove fails. */ |
a455aa72f device-dax/kmem: ... |
79 |
res = request_mem_region(range.start, range_len(&range), data->res_name); |
60e93dc09 device-dax: add d... |
80 81 82 83 84 85 86 87 88 89 |
if (!res) { dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve region ", i, range.start, range.end); /* * Once some memory has been onlined we can't * assume that it can be un-onlined safely. */ if (mapped) continue; |
a455aa72f device-dax/kmem: ... |
90 91 |
rc = -EBUSY; goto err_request_mem; |
60e93dc09 device-dax: add d... |
92 |
} |
a455aa72f device-dax/kmem: ... |
93 |
data->res[i] = res; |
60e93dc09 device-dax: add d... |
94 95 96 97 98 99 100 101 102 103 104 105 106 107 |
/* * Set flags appropriate for System RAM. Leave ..._BUSY clear * so that add_memory() can add a child resource. Do not * inherit flags from the parent since it may set new flags * unknown to us that will break add_memory() below. */ res->flags = IORESOURCE_SYSTEM_RAM; /* * Ensure that future kexec'd kernels will not treat * this as RAM automatically. */ rc = add_memory_driver_managed(numa_node, range.start, |
b61171997 mm/memory_hotplug... |
108 |
range_len(&range), kmem_name, MHP_NONE); |
60e93dc09 device-dax: add d... |
109 110 111 112 113 |
if (rc) { dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed ", i, range.start, range.end); |
a455aa72f device-dax/kmem: ... |
114 115 116 |
release_resource(res); kfree(res); data->res[i] = NULL; |
60e93dc09 device-dax: add d... |
117 118 |
if (mapped) continue; |
a455aa72f device-dax/kmem: ... |
119 |
goto err_request_mem; |
60e93dc09 device-dax: add d... |
120 121 |
} mapped++; |
31e4ca92a device-dax: fix m... |
122 |
} |
7e6b431aa device-dax/kmem: ... |
123 |
|
a455aa72f device-dax/kmem: ... |
124 |
dev_set_drvdata(dev, data); |
c221c0b03 device-dax: "Hotp... |
125 126 |
return 0; |
a455aa72f device-dax/kmem: ... |
127 128 129 130 131 132 |
err_request_mem: kfree(data->res_name); err_res_name: kfree(data); return rc; |
c221c0b03 device-dax: "Hotp... |
133 |
} |
9f960da72 device-dax: "Hotr... |
134 |
#ifdef CONFIG_MEMORY_HOTREMOVE |
f11cf813d device-dax: intro... |
135 |
static int dev_dax_kmem_remove(struct dev_dax *dev_dax) |
9f960da72 device-dax: "Hotr... |
136 |
{ |
60e93dc09 device-dax: add d... |
137 |
int i, success = 0; |
f11cf813d device-dax: intro... |
138 |
struct device *dev = &dev_dax->dev; |
a455aa72f device-dax/kmem: ... |
139 |
struct dax_kmem_data *data = dev_get_drvdata(dev); |
9f960da72 device-dax: "Hotr... |
140 141 142 143 144 145 146 |
/* * We have one shot for removing memory, if some memory blocks were not * offline prior to calling this function remove_memory() will fail, and * there is no way to hotremove this memory until reboot because device * unbind will succeed even if we return failure. */ |
60e93dc09 device-dax: add d... |
147 148 149 150 151 152 153 154 155 156 157 |
for (i = 0; i < dev_dax->nr_range; i++) { struct range range; int rc; rc = dax_kmem_range(dev_dax, i, &range); if (rc) continue; rc = remove_memory(dev_dax->target_node, range.start, range_len(&range)); if (rc == 0) { |
a455aa72f device-dax/kmem: ... |
158 159 160 |
release_resource(data->res[i]); kfree(data->res[i]); data->res[i] = NULL; |
60e93dc09 device-dax: add d... |
161 162 163 |
success++; continue; } |
8a725e469 device-dax: add m... |
164 |
any_hotremove_failed = true; |
60e93dc09 device-dax: add d... |
165 166 167 168 |
dev_err(dev, "mapping%d: %#llx-%#llx cannot be hotremoved until the next reboot ", i, range.start, range.end); |
9f960da72 device-dax: "Hotr... |
169 |
} |
60e93dc09 device-dax: add d... |
170 |
if (success >= dev_dax->nr_range) { |
a455aa72f device-dax/kmem: ... |
171 172 |
kfree(data->res_name); kfree(data); |
60e93dc09 device-dax: add d... |
173 174 |
dev_set_drvdata(dev, NULL); } |
9f960da72 device-dax: "Hotr... |
175 176 177 178 |
return 0; } #else |
f11cf813d device-dax: intro... |
179 |
static int dev_dax_kmem_remove(struct dev_dax *dev_dax) |
c221c0b03 device-dax: "Hotp... |
180 181 |
{ /* |
9f960da72 device-dax: "Hotr... |
182 183 184 185 |
* Without hotremove purposely leak the request_mem_region() for the * device-dax range and return '0' to ->remove() attempts. The removal * of the device from the driver always succeeds, but the region is * permanently pinned as reserved by the unreleased |
c221c0b03 device-dax: "Hotp... |
186 187 |
* request_mem_region(). */ |
8a725e469 device-dax: add m... |
188 |
any_hotremove_failed = true; |
c221c0b03 device-dax: "Hotp... |
189 190 |
return 0; } |
9f960da72 device-dax: "Hotr... |
191 |
#endif /* CONFIG_MEMORY_HOTREMOVE */ |
c221c0b03 device-dax: "Hotp... |
192 193 |
static struct dax_device_driver device_dax_kmem_driver = { |
f11cf813d device-dax: intro... |
194 195 |
.probe = dev_dax_kmem_probe, .remove = dev_dax_kmem_remove, |
c221c0b03 device-dax: "Hotp... |
196 197 198 199 |
}; static int __init dax_kmem_init(void) { |
8a725e469 device-dax: add m... |
200 201 202 203 204 205 206 207 208 209 210 |
int rc; /* Resource name is permanently allocated if any hotremove fails. */ kmem_name = kstrdup_const("System RAM (kmem)", GFP_KERNEL); if (!kmem_name) return -ENOMEM; rc = dax_driver_register(&device_dax_kmem_driver); if (rc) kfree_const(kmem_name); return rc; |
c221c0b03 device-dax: "Hotp... |
211 212 213 214 215 |
} static void __exit dax_kmem_exit(void) { dax_driver_unregister(&device_dax_kmem_driver); |
8a725e469 device-dax: add m... |
216 217 |
if (!any_hotremove_failed) kfree_const(kmem_name); |
c221c0b03 device-dax: "Hotp... |
218 219 220 221 222 223 224 |
} MODULE_AUTHOR("Intel Corporation"); MODULE_LICENSE("GPL v2"); module_init(dax_kmem_init); module_exit(dax_kmem_exit); MODULE_ALIAS_DAX_DEVICE(0); |