Blame view
drivers/lightnvm/core.c
30.7 KB
a98c5b196 treewide: Replace... |
1 |
// SPDX-License-Identifier: GPL-2.0-only |
cd9e9808d lightnvm: Support... |
2 3 4 |
/* * Copyright (C) 2015 IT University of Copenhagen. All rights reserved. * Initial release: Matias Bjorling <m@bjorling.me> |
cd9e9808d lightnvm: Support... |
5 |
*/ |
43db059ea lightnvm: introdu... |
6 |
#define pr_fmt(fmt) "nvm: " fmt |
cd9e9808d lightnvm: Support... |
7 8 9 10 |
#include <linux/list.h> #include <linux/types.h> #include <linux/sem.h> #include <linux/bitmap.h> |
900148296 lightnvm: prevent... |
11 |
#include <linux/module.h> |
389b2a1c0 lightnvm: make co... |
12 |
#include <linux/moduleparam.h> |
cd9e9808d lightnvm: Support... |
13 14 |
#include <linux/miscdevice.h> #include <linux/lightnvm.h> |
91276162d lightnvm: refacto... |
15 |
#include <linux/sched/sysctl.h> |
cd9e9808d lightnvm: Support... |
16 |
|
6063fe399 lightnvm: rename ... |
17 |
static LIST_HEAD(nvm_tgt_types); |
5cd907853 lightnvm: remove ... |
18 |
static DECLARE_RWSEM(nvm_tgtt_lock); |
cd9e9808d lightnvm: Support... |
19 20 |
static LIST_HEAD(nvm_devices); static DECLARE_RWSEM(nvm_lock); |
ade69e243 lightnvm: merge g... |
21 22 23 |
/* Map between virtual and physical channel and lun */ struct nvm_ch_map { int ch_off; |
a40afad90 lightnvm: normali... |
24 |
int num_lun; |
ade69e243 lightnvm: merge g... |
25 26 27 28 29 |
int *lun_offs; }; struct nvm_dev_map { struct nvm_ch_map *chnls; |
a40afad90 lightnvm: normali... |
30 |
int num_ch; |
ade69e243 lightnvm: merge g... |
31 |
}; |
e69397ea0 lightnvm: track i... |
32 |
static void nvm_free(struct kref *ref); |
ade69e243 lightnvm: merge g... |
33 34 35 36 37 38 39 40 41 42 |
static struct nvm_target *nvm_find_target(struct nvm_dev *dev, const char *name) { struct nvm_target *tgt; list_for_each_entry(tgt, &dev->targets, list) if (!strcmp(name, tgt->disk->disk_name)) return tgt; return NULL; } |
bd77b23b4 lightnvm: guarant... |
43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 |
static bool nvm_target_exists(const char *name) { struct nvm_dev *dev; struct nvm_target *tgt; bool ret = false; down_write(&nvm_lock); list_for_each_entry(dev, &nvm_devices, devices) { mutex_lock(&dev->mlock); list_for_each_entry(tgt, &dev->targets, list) { if (!strcmp(name, tgt->disk->disk_name)) { ret = true; mutex_unlock(&dev->mlock); goto out; } } mutex_unlock(&dev->mlock); } out: up_write(&nvm_lock); return ret; } |
ade69e243 lightnvm: merge g... |
66 67 68 69 70 71 |
static int nvm_reserve_luns(struct nvm_dev *dev, int lun_begin, int lun_end) { int i; for (i = lun_begin; i <= lun_end; i++) { if (test_and_set_bit(i, dev->lun_map)) { |
43db059ea lightnvm: introdu... |
72 73 |
pr_err("lun %d already allocated ", i); |
ade69e243 lightnvm: merge g... |
74 75 76 77 78 79 |
goto err; } } return 0; err: |
507f7d68f lightnvm: fix bad... |
80 |
while (--i >= lun_begin) |
ade69e243 lightnvm: merge g... |
81 82 83 84 85 86 87 88 89 90 91 92 93 |
clear_bit(i, dev->lun_map); return -EBUSY; } static void nvm_release_luns_err(struct nvm_dev *dev, int lun_begin, int lun_end) { int i; for (i = lun_begin; i <= lun_end; i++) WARN_ON(!test_and_clear_bit(i, dev->lun_map)); } |
edee1bdd6 lightnvm: double-... |
94 |
static void nvm_remove_tgt_dev(struct nvm_tgt_dev *tgt_dev, int clear) |
ade69e243 lightnvm: merge g... |
95 96 97 98 |
{ struct nvm_dev *dev = tgt_dev->parent; struct nvm_dev_map *dev_map = tgt_dev->map; int i, j; |
a40afad90 lightnvm: normali... |
99 |
for (i = 0; i < dev_map->num_ch; i++) { |
ade69e243 lightnvm: merge g... |
100 101 102 |
struct nvm_ch_map *ch_map = &dev_map->chnls[i]; int *lun_offs = ch_map->lun_offs; int ch = i + ch_map->ch_off; |
edee1bdd6 lightnvm: double-... |
103 |
if (clear) { |
a40afad90 lightnvm: normali... |
104 |
for (j = 0; j < ch_map->num_lun; j++) { |
edee1bdd6 lightnvm: double-... |
105 |
int lun = j + lun_offs[j]; |
a40afad90 lightnvm: normali... |
106 |
int lunid = (ch * dev->geo.num_lun) + lun; |
ade69e243 lightnvm: merge g... |
107 |
|
edee1bdd6 lightnvm: double-... |
108 109 110 |
WARN_ON(!test_and_clear_bit(lunid, dev->lun_map)); } |
ade69e243 lightnvm: merge g... |
111 112 113 114 115 116 117 118 119 120 121 122 123 |
} kfree(ch_map->lun_offs); } kfree(dev_map->chnls); kfree(dev_map); kfree(tgt_dev->luns); kfree(tgt_dev); } static struct nvm_tgt_dev *nvm_create_tgt_dev(struct nvm_dev *dev, |
e53927393 lightnvm: set tar... |
124 125 |
u16 lun_begin, u16 lun_end, u16 op) |
ade69e243 lightnvm: merge g... |
126 127 128 129 130 |
{ struct nvm_tgt_dev *tgt_dev = NULL; struct nvm_dev_map *dev_rmap = dev->rmap; struct nvm_dev_map *dev_map; struct ppa_addr *luns; |
a40afad90 lightnvm: normali... |
131 132 133 134 135 136 |
int num_lun = lun_end - lun_begin + 1; int luns_left = num_lun; int num_ch = num_lun / dev->geo.num_lun; int num_ch_mod = num_lun % dev->geo.num_lun; int bch = lun_begin / dev->geo.num_lun; int blun = lun_begin % dev->geo.num_lun; |
ade69e243 lightnvm: merge g... |
137 138 |
int lunid = 0; int lun_balanced = 1; |
a40afad90 lightnvm: normali... |
139 |
int sec_per_lun, prev_num_lun; |
ade69e243 lightnvm: merge g... |
140 |
int i, j; |
a40afad90 lightnvm: normali... |
141 |
num_ch = (num_ch_mod == 0) ? num_ch : num_ch + 1; |
ade69e243 lightnvm: merge g... |
142 143 144 145 |
dev_map = kmalloc(sizeof(struct nvm_dev_map), GFP_KERNEL); if (!dev_map) goto err_dev; |
a40afad90 lightnvm: normali... |
146 |
dev_map->chnls = kcalloc(num_ch, sizeof(struct nvm_ch_map), GFP_KERNEL); |
ade69e243 lightnvm: merge g... |
147 148 |
if (!dev_map->chnls) goto err_chnls; |
a40afad90 lightnvm: normali... |
149 |
luns = kcalloc(num_lun, sizeof(struct ppa_addr), GFP_KERNEL); |
ade69e243 lightnvm: merge g... |
150 151 |
if (!luns) goto err_luns; |
a40afad90 lightnvm: normali... |
152 153 154 |
prev_num_lun = (luns_left > dev->geo.num_lun) ? dev->geo.num_lun : luns_left; for (i = 0; i < num_ch; i++) { |
ade69e243 lightnvm: merge g... |
155 156 157 158 |
struct nvm_ch_map *ch_rmap = &dev_rmap->chnls[i + bch]; int *lun_roffs = ch_rmap->lun_offs; struct nvm_ch_map *ch_map = &dev_map->chnls[i]; int *lun_offs; |
a40afad90 lightnvm: normali... |
159 160 |
int luns_in_chnl = (luns_left > dev->geo.num_lun) ? dev->geo.num_lun : luns_left; |
ade69e243 lightnvm: merge g... |
161 |
|
a40afad90 lightnvm: normali... |
162 |
if (lun_balanced && prev_num_lun != luns_in_chnl) |
ade69e243 lightnvm: merge g... |
163 164 165 |
lun_balanced = 0; ch_map->ch_off = ch_rmap->ch_off = bch; |
a40afad90 lightnvm: normali... |
166 |
ch_map->num_lun = luns_in_chnl; |
ade69e243 lightnvm: merge g... |
167 168 169 170 171 172 173 |
lun_offs = kcalloc(luns_in_chnl, sizeof(int), GFP_KERNEL); if (!lun_offs) goto err_ch; for (j = 0; j < luns_in_chnl; j++) { luns[lunid].ppa = 0; |
694715137 lightnvm: add sup... |
174 175 |
luns[lunid].a.ch = i; luns[lunid++].a.lun = j; |
ade69e243 lightnvm: merge g... |
176 177 178 179 180 181 182 183 184 185 186 |
lun_offs[j] = blun; lun_roffs[j + blun] = blun; } ch_map->lun_offs = lun_offs; /* when starting a new channel, lun offset is reset */ blun = 0; luns_left -= luns_in_chnl; } |
a40afad90 lightnvm: normali... |
187 |
dev_map->num_ch = num_ch; |
ade69e243 lightnvm: merge g... |
188 189 190 191 |
tgt_dev = kmalloc(sizeof(struct nvm_tgt_dev), GFP_KERNEL); if (!tgt_dev) goto err_ch; |
e46f4e482 lightnvm: simplif... |
192 |
/* Inherit device geometry from parent */ |
ade69e243 lightnvm: merge g... |
193 |
memcpy(&tgt_dev->geo, &dev->geo, sizeof(struct nvm_geo)); |
e46f4e482 lightnvm: simplif... |
194 |
|
ade69e243 lightnvm: merge g... |
195 |
/* Target device only owns a portion of the physical device */ |
a40afad90 lightnvm: normali... |
196 197 198 199 |
tgt_dev->geo.num_ch = num_ch; tgt_dev->geo.num_lun = (lun_balanced) ? prev_num_lun : -1; tgt_dev->geo.all_luns = num_lun; tgt_dev->geo.all_chunks = num_lun * dev->geo.num_chk; |
e46f4e482 lightnvm: simplif... |
200 |
|
e53927393 lightnvm: set tar... |
201 |
tgt_dev->geo.op = op; |
e46f4e482 lightnvm: simplif... |
202 |
|
a40afad90 lightnvm: normali... |
203 204 |
sec_per_lun = dev->geo.clba * dev->geo.num_chk; tgt_dev->geo.total_secs = num_lun * sec_per_lun; |
e46f4e482 lightnvm: simplif... |
205 |
|
ade69e243 lightnvm: merge g... |
206 207 208 |
tgt_dev->q = dev->q; tgt_dev->map = dev_map; tgt_dev->luns = luns; |
ade69e243 lightnvm: merge g... |
209 210 211 212 |
tgt_dev->parent = dev; return tgt_dev; err_ch: |
507f7d68f lightnvm: fix bad... |
213 |
while (--i >= 0) |
ade69e243 lightnvm: merge g... |
214 215 216 217 218 219 220 221 222 |
kfree(dev_map->chnls[i].lun_offs); kfree(luns); err_luns: kfree(dev_map->chnls); err_chnls: kfree(dev_map); err_dev: return tgt_dev; } |
e29c80e6d lightnvm: refacto... |
223 |
static struct nvm_tgt_type *__nvm_find_target_type(const char *name) |
eb6f168f9 lightnvm: remove ... |
224 |
{ |
e29c80e6d lightnvm: refacto... |
225 |
struct nvm_tgt_type *tt; |
eb6f168f9 lightnvm: remove ... |
226 |
|
e29c80e6d lightnvm: refacto... |
227 228 229 |
list_for_each_entry(tt, &nvm_tgt_types, list) if (!strcmp(name, tt->name)) return tt; |
eb6f168f9 lightnvm: remove ... |
230 |
|
e29c80e6d lightnvm: refacto... |
231 232 233 234 235 236 237 238 239 240 |
return NULL; } static struct nvm_tgt_type *nvm_find_target_type(const char *name) { struct nvm_tgt_type *tt; down_write(&nvm_tgtt_lock); tt = __nvm_find_target_type(name); up_write(&nvm_tgtt_lock); |
eb6f168f9 lightnvm: remove ... |
241 |
|
eb6f168f9 lightnvm: remove ... |
242 243 |
return tt; } |
e53927393 lightnvm: set tar... |
244 245 246 247 |
static int nvm_config_check_luns(struct nvm_geo *geo, int lun_begin, int lun_end) { if (lun_begin > lun_end || lun_end >= geo->all_luns) { |
43db059ea lightnvm: introdu... |
248 249 |
pr_err("lun out of bound (%u:%u > %u) ", |
e53927393 lightnvm: set tar... |
250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 |
lun_begin, lun_end, geo->all_luns - 1); return -EINVAL; } return 0; } static int __nvm_config_simple(struct nvm_dev *dev, struct nvm_ioctl_create_simple *s) { struct nvm_geo *geo = &dev->geo; if (s->lun_begin == -1 && s->lun_end == -1) { s->lun_begin = 0; s->lun_end = geo->all_luns - 1; } return nvm_config_check_luns(geo, s->lun_begin, s->lun_end); } static int __nvm_config_extended(struct nvm_dev *dev, struct nvm_ioctl_create_extended *e) { |
e53927393 lightnvm: set tar... |
273 274 275 276 277 278 |
if (e->lun_begin == 0xFFFF && e->lun_end == 0xFFFF) { e->lun_begin = 0; e->lun_end = dev->geo.all_luns - 1; } /* op not set falls into target's default */ |
9d7aa4a48 lightnvm: Avoid v... |
279 |
if (e->op == 0xFFFF) { |
e53927393 lightnvm: set tar... |
280 |
e->op = NVM_TARGET_DEFAULT_OP; |
9d7aa4a48 lightnvm: Avoid v... |
281 |
} else if (e->op < NVM_TARGET_MIN_OP || e->op > NVM_TARGET_MAX_OP) { |
43db059ea lightnvm: introdu... |
282 283 |
pr_err("invalid over provisioning value "); |
e53927393 lightnvm: set tar... |
284 285 |
return -EINVAL; } |
e46f4e482 lightnvm: simplif... |
286 |
return nvm_config_check_luns(&dev->geo, e->lun_begin, e->lun_end); |
e53927393 lightnvm: set tar... |
287 |
} |
ade69e243 lightnvm: merge g... |
288 289 |
static int nvm_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create) { |
e53927393 lightnvm: set tar... |
290 |
struct nvm_ioctl_create_extended e; |
ade69e243 lightnvm: merge g... |
291 292 293 294 295 296 |
struct request_queue *tqueue; struct gendisk *tdisk; struct nvm_tgt_type *tt; struct nvm_target *t; struct nvm_tgt_dev *tgt_dev; void *targetdata; |
a14669ebc lightnvm: Inherit... |
297 |
unsigned int mdts; |
8d77bb827 lightnvm: propaga... |
298 |
int ret; |
ade69e243 lightnvm: merge g... |
299 |
|
e53927393 lightnvm: set tar... |
300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 |
switch (create->conf.type) { case NVM_CONFIG_TYPE_SIMPLE: ret = __nvm_config_simple(dev, &create->conf.s); if (ret) return ret; e.lun_begin = create->conf.s.lun_begin; e.lun_end = create->conf.s.lun_end; e.op = NVM_TARGET_DEFAULT_OP; break; case NVM_CONFIG_TYPE_EXTENDED: ret = __nvm_config_extended(dev, &create->conf.e); if (ret) return ret; e = create->conf.e; break; default: |
43db059ea lightnvm: introdu... |
318 319 |
pr_err("config type not valid "); |
e53927393 lightnvm: set tar... |
320 321 |
return -EINVAL; } |
e29c80e6d lightnvm: refacto... |
322 |
tt = nvm_find_target_type(create->tgttype); |
ade69e243 lightnvm: merge g... |
323 |
if (!tt) { |
43db059ea lightnvm: introdu... |
324 325 |
pr_err("target type %s not found ", create->tgttype); |
ade69e243 lightnvm: merge g... |
326 327 |
return -EINVAL; } |
656e33ca3 lightnvm: move de... |
328 |
if ((tt->flags & NVM_TGT_F_HOST_L2P) != (dev->geo.dom & NVM_RSP_L2P)) { |
43db059ea lightnvm: introdu... |
329 330 |
pr_err("device is incompatible with target L2P type. "); |
656e33ca3 lightnvm: move de... |
331 332 |
return -EINVAL; } |
bd77b23b4 lightnvm: guarant... |
333 |
if (nvm_target_exists(create->tgtname)) { |
43db059ea lightnvm: introdu... |
334 335 |
pr_err("target name already exists (%s) ", |
bd77b23b4 lightnvm: guarant... |
336 |
create->tgtname); |
ade69e243 lightnvm: merge g... |
337 338 |
return -EINVAL; } |
ade69e243 lightnvm: merge g... |
339 |
|
e53927393 lightnvm: set tar... |
340 |
ret = nvm_reserve_luns(dev, e.lun_begin, e.lun_end); |
12e9a6d62 lightnvm: if LUNs... |
341 342 |
if (ret) return ret; |
ade69e243 lightnvm: merge g... |
343 344 |
t = kmalloc(sizeof(struct nvm_target), GFP_KERNEL); |
8d77bb827 lightnvm: propaga... |
345 346 |
if (!t) { ret = -ENOMEM; |
ade69e243 lightnvm: merge g... |
347 |
goto err_reserve; |
8d77bb827 lightnvm: propaga... |
348 |
} |
ade69e243 lightnvm: merge g... |
349 |
|
e53927393 lightnvm: set tar... |
350 |
tgt_dev = nvm_create_tgt_dev(dev, e.lun_begin, e.lun_end, e.op); |
ade69e243 lightnvm: merge g... |
351 |
if (!tgt_dev) { |
43db059ea lightnvm: introdu... |
352 353 |
pr_err("could not create target device "); |
8d77bb827 lightnvm: propaga... |
354 |
ret = -ENOMEM; |
ade69e243 lightnvm: merge g... |
355 356 |
goto err_t; } |
7d1ef2f40 lightnvm: fix cle... |
357 |
tdisk = alloc_disk(0); |
8d77bb827 lightnvm: propaga... |
358 359 |
if (!tdisk) { ret = -ENOMEM; |
7d1ef2f40 lightnvm: fix cle... |
360 |
goto err_dev; |
8d77bb827 lightnvm: propaga... |
361 |
} |
7d1ef2f40 lightnvm: fix cle... |
362 |
|
c62b37d96 block: move ->mak... |
363 |
tqueue = blk_alloc_queue(dev->q->node); |
8d77bb827 lightnvm: propaga... |
364 365 |
if (!tqueue) { ret = -ENOMEM; |
7d1ef2f40 lightnvm: fix cle... |
366 |
goto err_disk; |
8d77bb827 lightnvm: propaga... |
367 |
} |
ade69e243 lightnvm: merge g... |
368 |
|
6eb082452 lightnvm: convert... |
369 |
strlcpy(tdisk->disk_name, create->tgtname, sizeof(tdisk->disk_name)); |
ade69e243 lightnvm: merge g... |
370 371 372 |
tdisk->flags = GENHD_FL_EXT_DEVT; tdisk->major = 0; tdisk->first_minor = 0; |
c62b37d96 block: move ->mak... |
373 |
tdisk->fops = tt->bops; |
ade69e243 lightnvm: merge g... |
374 |
tdisk->queue = tqueue; |
4af3f75d7 lightnvm: allow t... |
375 |
targetdata = tt->init(tgt_dev, tdisk, create->flags); |
8d77bb827 lightnvm: propaga... |
376 377 |
if (IS_ERR(targetdata)) { ret = PTR_ERR(targetdata); |
ade69e243 lightnvm: merge g... |
378 |
goto err_init; |
8d77bb827 lightnvm: propaga... |
379 |
} |
ade69e243 lightnvm: merge g... |
380 381 382 |
tdisk->private_data = targetdata; tqueue->queuedata = targetdata; |
a14669ebc lightnvm: Inherit... |
383 384 385 386 387 388 |
mdts = (dev->geo.csecs >> 9) * NVM_MAX_VLBA; if (dev->geo.mdts) { mdts = min_t(u32, dev->geo.mdts, (dev->geo.csecs >> 9) * NVM_MAX_VLBA); } blk_queue_max_hw_sectors(tqueue, mdts); |
ade69e243 lightnvm: merge g... |
389 390 391 |
set_capacity(tdisk, tt->capacity(targetdata)); add_disk(tdisk); |
8d77bb827 lightnvm: propaga... |
392 393 |
if (tt->sysfs_init && tt->sysfs_init(tdisk)) { ret = -ENOMEM; |
9a69b0ed6 lightnvm: allow t... |
394 |
goto err_sysfs; |
8d77bb827 lightnvm: propaga... |
395 |
} |
9a69b0ed6 lightnvm: allow t... |
396 |
|
ade69e243 lightnvm: merge g... |
397 398 399 400 401 402 403 |
t->type = tt; t->disk = tdisk; t->dev = tgt_dev; mutex_lock(&dev->mlock); list_add_tail(&t->list, &dev->targets); mutex_unlock(&dev->mlock); |
900148296 lightnvm: prevent... |
404 |
__module_get(tt->owner); |
ade69e243 lightnvm: merge g... |
405 |
return 0; |
9a69b0ed6 lightnvm: allow t... |
406 407 |
err_sysfs: if (tt->exit) |
a7c9e9109 lightnvm: pass fl... |
408 |
tt->exit(targetdata, true); |
ade69e243 lightnvm: merge g... |
409 |
err_init: |
ade69e243 lightnvm: merge g... |
410 |
blk_cleanup_queue(tqueue); |
75ba4ada8 ligtnvm: fix doub... |
411 |
tdisk->queue = NULL; |
7d1ef2f40 lightnvm: fix cle... |
412 413 |
err_disk: put_disk(tdisk); |
ade69e243 lightnvm: merge g... |
414 |
err_dev: |
edee1bdd6 lightnvm: double-... |
415 |
nvm_remove_tgt_dev(tgt_dev, 0); |
ade69e243 lightnvm: merge g... |
416 417 418 |
err_t: kfree(t); err_reserve: |
e53927393 lightnvm: set tar... |
419 |
nvm_release_luns_err(dev, e.lun_begin, e.lun_end); |
8d77bb827 lightnvm: propaga... |
420 |
return ret; |
ade69e243 lightnvm: merge g... |
421 |
} |
a7c9e9109 lightnvm: pass fl... |
422 |
static void __nvm_remove_target(struct nvm_target *t, bool graceful) |
ade69e243 lightnvm: merge g... |
423 424 425 426 427 428 429 |
{ struct nvm_tgt_type *tt = t->type; struct gendisk *tdisk = t->disk; struct request_queue *q = tdisk->queue; del_gendisk(tdisk); blk_cleanup_queue(q); |
9a69b0ed6 lightnvm: allow t... |
430 431 |
if (tt->sysfs_exit) tt->sysfs_exit(tdisk); |
ade69e243 lightnvm: merge g... |
432 |
if (tt->exit) |
a7c9e9109 lightnvm: pass fl... |
433 |
tt->exit(tdisk->private_data, graceful); |
ade69e243 lightnvm: merge g... |
434 |
|
edee1bdd6 lightnvm: double-... |
435 |
nvm_remove_tgt_dev(t->dev, 1); |
ade69e243 lightnvm: merge g... |
436 |
put_disk(tdisk); |
900148296 lightnvm: prevent... |
437 |
module_put(t->type->owner); |
ade69e243 lightnvm: merge g... |
438 439 440 441 442 443 444 |
list_del(&t->list); kfree(t); } /** * nvm_remove_tgt - Removes a target from the media manager |
ade69e243 lightnvm: merge g... |
445 446 447 448 449 450 451 |
* @remove: ioctl structure with target name to remove. * * Returns: * 0: on success * 1: on not found * <0: on error */ |
843f2edbd lightnvm: do not ... |
452 |
static int nvm_remove_tgt(struct nvm_ioctl_remove *remove) |
ade69e243 lightnvm: merge g... |
453 |
{ |
2f5af4ab7 lightnvm: fix uni... |
454 |
struct nvm_target *t = NULL; |
843f2edbd lightnvm: do not ... |
455 |
struct nvm_dev *dev; |
ade69e243 lightnvm: merge g... |
456 |
|
843f2edbd lightnvm: do not ... |
457 458 459 460 461 462 463 464 |
down_read(&nvm_lock); list_for_each_entry(dev, &nvm_devices, devices) { mutex_lock(&dev->mlock); t = nvm_find_target(dev, remove->tgtname); if (t) { mutex_unlock(&dev->mlock); break; } |
ade69e243 lightnvm: merge g... |
465 |
mutex_unlock(&dev->mlock); |
ade69e243 lightnvm: merge g... |
466 |
} |
843f2edbd lightnvm: do not ... |
467 |
up_read(&nvm_lock); |
362cd2b1f lightnvm: print e... |
468 469 470 471 |
if (!t) { pr_err("failed to remove target %s ", remove->tgtname); |
843f2edbd lightnvm: do not ... |
472 |
return 1; |
362cd2b1f lightnvm: print e... |
473 |
} |
843f2edbd lightnvm: do not ... |
474 |
|
a7c9e9109 lightnvm: pass fl... |
475 |
__nvm_remove_target(t, true); |
e69397ea0 lightnvm: track i... |
476 |
kref_put(&dev->ref, nvm_free); |
ade69e243 lightnvm: merge g... |
477 478 479 480 481 482 483 484 485 486 487 488 |
return 0; } static int nvm_register_map(struct nvm_dev *dev) { struct nvm_dev_map *rmap; int i, j; rmap = kmalloc(sizeof(struct nvm_dev_map), GFP_KERNEL); if (!rmap) goto err_rmap; |
a40afad90 lightnvm: normali... |
489 |
rmap->chnls = kcalloc(dev->geo.num_ch, sizeof(struct nvm_ch_map), |
ade69e243 lightnvm: merge g... |
490 491 492 |
GFP_KERNEL); if (!rmap->chnls) goto err_chnls; |
a40afad90 lightnvm: normali... |
493 |
for (i = 0; i < dev->geo.num_ch; i++) { |
ade69e243 lightnvm: merge g... |
494 495 |
struct nvm_ch_map *ch_rmap; int *lun_roffs; |
a40afad90 lightnvm: normali... |
496 |
int luns_in_chnl = dev->geo.num_lun; |
ade69e243 lightnvm: merge g... |
497 498 499 500 |
ch_rmap = &rmap->chnls[i]; ch_rmap->ch_off = -1; |
a40afad90 lightnvm: normali... |
501 |
ch_rmap->num_lun = luns_in_chnl; |
ade69e243 lightnvm: merge g... |
502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 |
lun_roffs = kcalloc(luns_in_chnl, sizeof(int), GFP_KERNEL); if (!lun_roffs) goto err_ch; for (j = 0; j < luns_in_chnl; j++) lun_roffs[j] = -1; ch_rmap->lun_offs = lun_roffs; } dev->rmap = rmap; return 0; err_ch: while (--i >= 0) kfree(rmap->chnls[i].lun_offs); err_chnls: kfree(rmap); err_rmap: return -ENOMEM; } |
7a3de2b33 lightnvm: free re... |
524 525 526 527 |
static void nvm_unregister_map(struct nvm_dev *dev) { struct nvm_dev_map *rmap = dev->rmap; int i; |
a40afad90 lightnvm: normali... |
528 |
for (i = 0; i < dev->geo.num_ch; i++) |
7a3de2b33 lightnvm: free re... |
529 530 531 532 533 |
kfree(rmap->chnls[i].lun_offs); kfree(rmap->chnls); kfree(rmap); } |
61a561d8d lightnvm: make nv... |
534 |
static void nvm_map_to_dev(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *p) |
ade69e243 lightnvm: merge g... |
535 536 |
{ struct nvm_dev_map *dev_map = tgt_dev->map; |
694715137 lightnvm: add sup... |
537 538 |
struct nvm_ch_map *ch_map = &dev_map->chnls[p->a.ch]; int lun_off = ch_map->lun_offs[p->a.lun]; |
ade69e243 lightnvm: merge g... |
539 |
|
694715137 lightnvm: add sup... |
540 541 |
p->a.ch += ch_map->ch_off; p->a.lun += lun_off; |
ade69e243 lightnvm: merge g... |
542 |
} |
61a561d8d lightnvm: make nv... |
543 |
static void nvm_map_to_tgt(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *p) |
ade69e243 lightnvm: merge g... |
544 545 546 |
{ struct nvm_dev *dev = tgt_dev->parent; struct nvm_dev_map *dev_rmap = dev->rmap; |
694715137 lightnvm: add sup... |
547 548 |
struct nvm_ch_map *ch_rmap = &dev_rmap->chnls[p->a.ch]; int lun_roff = ch_rmap->lun_offs[p->a.lun]; |
ade69e243 lightnvm: merge g... |
549 |
|
694715137 lightnvm: add sup... |
550 551 |
p->a.ch -= ch_rmap->ch_off; p->a.lun -= lun_roff; |
ade69e243 lightnvm: merge g... |
552 |
} |
dab8ee9e8 lightnvm: cleanup... |
553 554 |
static void nvm_ppa_tgt_to_dev(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *ppa_list, int nr_ppas) |
ade69e243 lightnvm: merge g... |
555 556 |
{ int i; |
ade69e243 lightnvm: merge g... |
557 |
|
dab8ee9e8 lightnvm: cleanup... |
558 559 |
for (i = 0; i < nr_ppas; i++) { nvm_map_to_dev(tgt_dev, &ppa_list[i]); |
7100d50a7 lightnvm: make ad... |
560 |
ppa_list[i] = generic_to_dev_addr(tgt_dev->parent, ppa_list[i]); |
ade69e243 lightnvm: merge g... |
561 |
} |
dab8ee9e8 lightnvm: cleanup... |
562 |
} |
ade69e243 lightnvm: merge g... |
563 |
|
dab8ee9e8 lightnvm: cleanup... |
564 565 566 567 568 569 |
static void nvm_ppa_dev_to_tgt(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *ppa_list, int nr_ppas) { int i; for (i = 0; i < nr_ppas; i++) { |
7100d50a7 lightnvm: make ad... |
570 |
ppa_list[i] = dev_to_generic_addr(tgt_dev->parent, ppa_list[i]); |
dab8ee9e8 lightnvm: cleanup... |
571 |
nvm_map_to_tgt(tgt_dev, &ppa_list[i]); |
ade69e243 lightnvm: merge g... |
572 |
} |
ade69e243 lightnvm: merge g... |
573 |
} |
dab8ee9e8 lightnvm: cleanup... |
574 |
static void nvm_rq_tgt_to_dev(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd) |
ade69e243 lightnvm: merge g... |
575 |
{ |
d68a93440 lightnvm: introdu... |
576 |
struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); |
ade69e243 lightnvm: merge g... |
577 |
|
d68a93440 lightnvm: introdu... |
578 |
nvm_ppa_tgt_to_dev(tgt_dev, ppa_list, rqd->nr_ppas); |
dab8ee9e8 lightnvm: cleanup... |
579 580 581 582 |
} static void nvm_rq_dev_to_tgt(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd) { |
d68a93440 lightnvm: introdu... |
583 |
struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); |
ade69e243 lightnvm: merge g... |
584 |
|
d68a93440 lightnvm: introdu... |
585 |
nvm_ppa_dev_to_tgt(tgt_dev, ppa_list, rqd->nr_ppas); |
ade69e243 lightnvm: merge g... |
586 |
} |
6063fe399 lightnvm: rename ... |
587 |
int nvm_register_tgt_type(struct nvm_tgt_type *tt) |
cd9e9808d lightnvm: Support... |
588 589 |
{ int ret = 0; |
5cd907853 lightnvm: remove ... |
590 |
down_write(&nvm_tgtt_lock); |
e29c80e6d lightnvm: refacto... |
591 |
if (__nvm_find_target_type(tt->name)) |
cd9e9808d lightnvm: Support... |
592 593 |
ret = -EEXIST; else |
6063fe399 lightnvm: rename ... |
594 |
list_add(&tt->list, &nvm_tgt_types); |
5cd907853 lightnvm: remove ... |
595 |
up_write(&nvm_tgtt_lock); |
cd9e9808d lightnvm: Support... |
596 597 598 |
return ret; } |
6063fe399 lightnvm: rename ... |
599 |
EXPORT_SYMBOL(nvm_register_tgt_type); |
cd9e9808d lightnvm: Support... |
600 |
|
6063fe399 lightnvm: rename ... |
601 |
void nvm_unregister_tgt_type(struct nvm_tgt_type *tt) |
cd9e9808d lightnvm: Support... |
602 603 604 |
{ if (!tt) return; |
88d31ea26 lightnvm: protect... |
605 |
down_write(&nvm_tgtt_lock); |
cd9e9808d lightnvm: Support... |
606 |
list_del(&tt->list); |
88d31ea26 lightnvm: protect... |
607 |
up_write(&nvm_tgtt_lock); |
cd9e9808d lightnvm: Support... |
608 |
} |
6063fe399 lightnvm: rename ... |
609 |
EXPORT_SYMBOL(nvm_unregister_tgt_type); |
cd9e9808d lightnvm: Support... |
610 611 612 613 |
void *nvm_dev_dma_alloc(struct nvm_dev *dev, gfp_t mem_flags, dma_addr_t *dma_handler) { |
75b856493 lightnvm: rename ... |
614 |
return dev->ops->dev_dma_alloc(dev, dev->dma_pool, mem_flags, |
cd9e9808d lightnvm: Support... |
615 616 617 |
dma_handler); } EXPORT_SYMBOL(nvm_dev_dma_alloc); |
da2d7cb82 lightnvm: use tar... |
618 |
void nvm_dev_dma_free(struct nvm_dev *dev, void *addr, dma_addr_t dma_handler) |
cd9e9808d lightnvm: Support... |
619 |
{ |
75b856493 lightnvm: rename ... |
620 |
dev->ops->dev_dma_free(dev->dma_pool, addr, dma_handler); |
cd9e9808d lightnvm: Support... |
621 622 |
} EXPORT_SYMBOL(nvm_dev_dma_free); |
cd9e9808d lightnvm: Support... |
623 624 625 626 627 628 629 630 631 632 |
static struct nvm_dev *nvm_find_nvm_dev(const char *name) { struct nvm_dev *dev; list_for_each_entry(dev, &nvm_devices, devices) if (!strcmp(name, dev->name)) return dev; return NULL; } |
eb6f168f9 lightnvm: remove ... |
633 634 635 636 637 638 639 |
static int nvm_set_rqd_ppalist(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd, const struct ppa_addr *ppas, int nr_ppas) { struct nvm_dev *dev = tgt_dev->parent; struct nvm_geo *geo = &tgt_dev->geo; int i, plane_cnt, pl_idx; struct ppa_addr ppa; |
a40afad90 lightnvm: normali... |
640 |
if (geo->pln_mode == NVM_PLANE_SINGLE && nr_ppas == 1) { |
eb6f168f9 lightnvm: remove ... |
641 642 643 644 645 646 647 648 649 |
rqd->nr_ppas = nr_ppas; rqd->ppa_addr = ppas[0]; return 0; } rqd->nr_ppas = nr_ppas; rqd->ppa_list = nvm_dev_dma_alloc(dev, GFP_KERNEL, &rqd->dma_ppa_list); if (!rqd->ppa_list) { |
43db059ea lightnvm: introdu... |
650 651 |
pr_err("failed to allocate dma memory "); |
eb6f168f9 lightnvm: remove ... |
652 653 |
return -ENOMEM; } |
a40afad90 lightnvm: normali... |
654 |
plane_cnt = geo->pln_mode; |
eb6f168f9 lightnvm: remove ... |
655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 |
rqd->nr_ppas *= plane_cnt; for (i = 0; i < nr_ppas; i++) { for (pl_idx = 0; pl_idx < plane_cnt; pl_idx++) { ppa = ppas[i]; ppa.g.pl = pl_idx; rqd->ppa_list[(pl_idx * nr_ppas) + i] = ppa; } } return 0; } static void nvm_free_rqd_ppalist(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd) { if (!rqd->ppa_list) return; nvm_dev_dma_free(tgt_dev->parent, rqd->ppa_list, rqd->dma_ppa_list); } |
d7b680167 lightnvm: combine... |
676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 |
static int nvm_set_flags(struct nvm_geo *geo, struct nvm_rq *rqd) { int flags = 0; if (geo->version == NVM_OCSSD_SPEC_20) return 0; if (rqd->is_seq) flags |= geo->pln_mode >> 1; if (rqd->opcode == NVM_OP_PREAD) flags |= (NVM_IO_SCRAMBLE_ENABLE | NVM_IO_SUSPEND); else if (rqd->opcode == NVM_OP_PWRITE) flags |= NVM_IO_SCRAMBLE_ENABLE; return flags; } |
48e5da725 lightnvm: move me... |
693 |
int nvm_submit_io(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd, void *buf) |
cd9e9808d lightnvm: Support... |
694 |
{ |
8e53624d4 lightnvm: elimina... |
695 |
struct nvm_dev *dev = tgt_dev->parent; |
3e505afb4 lightnvm: re-conv... |
696 |
int ret; |
8e53624d4 lightnvm: elimina... |
697 |
|
ade69e243 lightnvm: merge g... |
698 699 |
if (!dev->ops->submit_io) return -ENODEV; |
dab8ee9e8 lightnvm: cleanup... |
700 |
nvm_rq_tgt_to_dev(tgt_dev, rqd); |
ade69e243 lightnvm: merge g... |
701 702 |
rqd->dev = tgt_dev; |
d7b680167 lightnvm: combine... |
703 |
rqd->flags = nvm_set_flags(&tgt_dev->geo, rqd); |
3e505afb4 lightnvm: re-conv... |
704 705 |
/* In case of error, fail with right address format */ |
48e5da725 lightnvm: move me... |
706 |
ret = dev->ops->submit_io(dev, rqd, buf); |
3e505afb4 lightnvm: re-conv... |
707 708 709 |
if (ret) nvm_rq_dev_to_tgt(tgt_dev, rqd); return ret; |
cd9e9808d lightnvm: Support... |
710 711 |
} EXPORT_SYMBOL(nvm_submit_io); |
98d87f70f lightnvm: remove ... |
712 713 714 715 716 717 |
static void nvm_sync_end_io(struct nvm_rq *rqd) { struct completion *waiting = rqd->private; complete(waiting); } |
48e5da725 lightnvm: move me... |
718 719 |
static int nvm_submit_io_wait(struct nvm_dev *dev, struct nvm_rq *rqd, void *buf) |
98d87f70f lightnvm: remove ... |
720 721 722 723 724 725 |
{ DECLARE_COMPLETION_ONSTACK(wait); int ret = 0; rqd->end_io = nvm_sync_end_io; rqd->private = &wait; |
48e5da725 lightnvm: move me... |
726 |
ret = dev->ops->submit_io(dev, rqd, buf); |
98d87f70f lightnvm: remove ... |
727 728 729 730 731 732 733 |
if (ret) return ret; wait_for_completion_io(&wait); return 0; } |
48e5da725 lightnvm: move me... |
734 735 |
int nvm_submit_io_sync(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd, void *buf) |
cd9e9808d lightnvm: Support... |
736 |
{ |
1a94b2d48 lightnvm: impleme... |
737 738 |
struct nvm_dev *dev = tgt_dev->parent; int ret; |
98d87f70f lightnvm: remove ... |
739 |
if (!dev->ops->submit_io) |
1a94b2d48 lightnvm: impleme... |
740 741 742 |
return -ENODEV; nvm_rq_tgt_to_dev(tgt_dev, rqd); |
10995c3dc lightnvm: collaps... |
743 |
|
1a94b2d48 lightnvm: impleme... |
744 |
rqd->dev = tgt_dev; |
d7b680167 lightnvm: combine... |
745 |
rqd->flags = nvm_set_flags(&tgt_dev->geo, rqd); |
1a94b2d48 lightnvm: impleme... |
746 |
|
48e5da725 lightnvm: move me... |
747 |
ret = nvm_submit_io_wait(dev, rqd, buf); |
1a94b2d48 lightnvm: impleme... |
748 749 |
return ret; |
17912c49e lightnvm: submit ... |
750 |
} |
1a94b2d48 lightnvm: impleme... |
751 |
EXPORT_SYMBOL(nvm_submit_io_sync); |
10995c3dc lightnvm: collaps... |
752 |
|
06894efea lightnvm: use end... |
753 |
void nvm_end_io(struct nvm_rq *rqd) |
91276162d lightnvm: refacto... |
754 |
{ |
ade69e243 lightnvm: merge g... |
755 |
struct nvm_tgt_dev *tgt_dev = rqd->dev; |
ade69e243 lightnvm: merge g... |
756 757 758 |
/* Convert address space */ if (tgt_dev) |
dab8ee9e8 lightnvm: cleanup... |
759 |
nvm_rq_dev_to_tgt(tgt_dev, rqd); |
ade69e243 lightnvm: merge g... |
760 |
|
06894efea lightnvm: use end... |
761 762 |
if (rqd->end_io) rqd->end_io(rqd); |
91276162d lightnvm: refacto... |
763 764 |
} EXPORT_SYMBOL(nvm_end_io); |
aff3fb18f lightnvm: move ba... |
765 766 |
static int nvm_submit_io_sync_raw(struct nvm_dev *dev, struct nvm_rq *rqd) { |
98d87f70f lightnvm: remove ... |
767 |
if (!dev->ops->submit_io) |
aff3fb18f lightnvm: move ba... |
768 |
return -ENODEV; |
98d87f70f lightnvm: remove ... |
769 |
rqd->dev = NULL; |
aff3fb18f lightnvm: move ba... |
770 |
rqd->flags = nvm_set_flags(&dev->geo, rqd); |
48e5da725 lightnvm: move me... |
771 |
return nvm_submit_io_wait(dev, rqd, NULL); |
aff3fb18f lightnvm: move ba... |
772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 |
} static int nvm_bb_chunk_sense(struct nvm_dev *dev, struct ppa_addr ppa) { struct nvm_rq rqd = { NULL }; struct bio bio; struct bio_vec bio_vec; struct page *page; int ret; page = alloc_page(GFP_KERNEL); if (!page) return -ENOMEM; bio_init(&bio, &bio_vec, 1); bio_add_page(&bio, page, PAGE_SIZE, 0); bio_set_op_attrs(&bio, REQ_OP_READ, 0); rqd.bio = &bio; rqd.opcode = NVM_OP_PREAD; rqd.is_seq = 1; rqd.nr_ppas = 1; rqd.ppa_addr = generic_to_dev_addr(dev, ppa); ret = nvm_submit_io_sync_raw(dev, &rqd); if (ret) return ret; __free_page(page); return rqd.error; } |
22e8c9766 lightnvm: move bl... |
804 |
/* |
aff3fb18f lightnvm: move ba... |
805 806 807 |
* Scans a 1.2 chunk first and last page to determine if its state. * If the chunk is found to be open, also scan it to update the write * pointer. |
22e8c9766 lightnvm: move bl... |
808 |
*/ |
aff3fb18f lightnvm: move ba... |
809 810 |
static int nvm_bb_chunk_scan(struct nvm_dev *dev, struct ppa_addr ppa, struct nvm_chk_meta *meta) |
22e8c9766 lightnvm: move bl... |
811 |
{ |
8e79b5cb1 lightnvm: move bl... |
812 |
struct nvm_geo *geo = &dev->geo; |
aff3fb18f lightnvm: move ba... |
813 |
int ret, pg, pl; |
22e8c9766 lightnvm: move bl... |
814 |
|
aff3fb18f lightnvm: move ba... |
815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 |
/* sense first page */ ret = nvm_bb_chunk_sense(dev, ppa); if (ret < 0) /* io error */ return ret; else if (ret == 0) /* valid data */ meta->state = NVM_CHK_ST_OPEN; else if (ret > 0) { /* * If empty page, the chunk is free, else it is an * actual io error. In that case, mark it offline. */ switch (ret) { case NVM_RSP_ERR_EMPTYPAGE: meta->state = NVM_CHK_ST_FREE; return 0; case NVM_RSP_ERR_FAILCRC: case NVM_RSP_ERR_FAILECC: case NVM_RSP_WARN_HIGHECC: meta->state = NVM_CHK_ST_OPEN; goto scan; default: return -ret; /* other io error */ } } /* sense last page */ ppa.g.pg = geo->num_pg - 1; ppa.g.pl = geo->num_pln - 1; ret = nvm_bb_chunk_sense(dev, ppa); if (ret < 0) /* io error */ return ret; else if (ret == 0) { /* Chunk fully written */ meta->state = NVM_CHK_ST_CLOSED; meta->wp = geo->clba; return 0; } else if (ret > 0) { switch (ret) { case NVM_RSP_ERR_EMPTYPAGE: case NVM_RSP_ERR_FAILCRC: case NVM_RSP_ERR_FAILECC: case NVM_RSP_WARN_HIGHECC: meta->state = NVM_CHK_ST_OPEN; break; default: return -ret; /* other io error */ } } scan: /* * chunk is open, we scan sequentially to update the write pointer. * We make the assumption that targets write data across all planes * before moving to the next page. */ for (pg = 0; pg < geo->num_pg; pg++) { for (pl = 0; pl < geo->num_pln; pl++) { ppa.g.pg = pg; ppa.g.pl = pl; ret = nvm_bb_chunk_sense(dev, ppa); if (ret < 0) /* io error */ return ret; else if (ret == 0) { meta->wp += geo->ws_min; } else if (ret > 0) { switch (ret) { case NVM_RSP_ERR_EMPTYPAGE: return 0; case NVM_RSP_ERR_FAILCRC: case NVM_RSP_ERR_FAILECC: case NVM_RSP_WARN_HIGHECC: meta->wp += geo->ws_min; break; default: return -ret; /* other io error */ } } } } return 0; } /* * folds a bad block list from its plane representation to its * chunk representation. * * If any of the planes status are bad or grown bad, the chunk is marked * offline. If not bad, the first plane state acts as the chunk state. */ static int nvm_bb_to_chunk(struct nvm_dev *dev, struct ppa_addr ppa, u8 *blks, int nr_blks, struct nvm_chk_meta *meta) { struct nvm_geo *geo = &dev->geo; int ret, blk, pl, offset, blktype; |
22e8c9766 lightnvm: move bl... |
911 |
|
a40afad90 lightnvm: normali... |
912 913 |
for (blk = 0; blk < geo->num_chk; blk++) { offset = blk * geo->pln_mode; |
22e8c9766 lightnvm: move bl... |
914 |
blktype = blks[offset]; |
a40afad90 lightnvm: normali... |
915 |
for (pl = 0; pl < geo->pln_mode; pl++) { |
22e8c9766 lightnvm: move bl... |
916 917 918 919 920 921 |
if (blks[offset + pl] & (NVM_BLK_T_BAD|NVM_BLK_T_GRWN_BAD)) { blktype = blks[offset + pl]; break; } } |
aff3fb18f lightnvm: move ba... |
922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 |
ppa.g.blk = blk; meta->wp = 0; meta->type = NVM_CHK_TP_W_SEQ; meta->wi = 0; meta->slba = generic_to_dev_addr(dev, ppa).ppa; meta->cnlb = dev->geo.clba; if (blktype == NVM_BLK_T_FREE) { ret = nvm_bb_chunk_scan(dev, ppa, meta); if (ret) return ret; } else { meta->state = NVM_CHK_ST_OFFLINE; } meta++; |
22e8c9766 lightnvm: move bl... |
939 |
} |
aff3fb18f lightnvm: move ba... |
940 |
return 0; |
22e8c9766 lightnvm: move bl... |
941 |
} |
22e8c9766 lightnvm: move bl... |
942 |
|
aff3fb18f lightnvm: move ba... |
943 944 945 946 947 948 949 |
static int nvm_get_bb_meta(struct nvm_dev *dev, sector_t slba, int nchks, struct nvm_chk_meta *meta) { struct nvm_geo *geo = &dev->geo; struct ppa_addr ppa; u8 *blks; int ch, lun, nr_blks; |
55e58c5e7 lightnvm: Fix uni... |
950 |
int ret = 0; |
aff3fb18f lightnvm: move ba... |
951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 |
ppa.ppa = slba; ppa = dev_to_generic_addr(dev, ppa); if (ppa.g.blk != 0) return -EINVAL; if ((nchks % geo->num_chk) != 0) return -EINVAL; nr_blks = geo->num_chk * geo->pln_mode; blks = kmalloc(nr_blks, GFP_KERNEL); if (!blks) return -ENOMEM; for (ch = ppa.g.ch; ch < geo->num_ch; ch++) { for (lun = ppa.g.lun; lun < geo->num_lun; lun++) { struct ppa_addr ppa_gen, ppa_dev; if (!nchks) goto done; ppa_gen.ppa = 0; ppa_gen.g.ch = ch; ppa_gen.g.lun = lun; ppa_dev = generic_to_dev_addr(dev, ppa_gen); ret = dev->ops->get_bb_tbl(dev, ppa_dev, blks); if (ret) goto done; ret = nvm_bb_to_chunk(dev, ppa_gen, blks, nr_blks, meta); if (ret) goto done; meta += geo->num_chk; nchks -= geo->num_chk; } } done: kfree(blks); return ret; } int nvm_get_chunk_meta(struct nvm_tgt_dev *tgt_dev, struct ppa_addr ppa, int nchks, struct nvm_chk_meta *meta) |
333ba053d lightnvm: transfo... |
999 |
{ |
8f4fe008f lightnvm: remove ... |
1000 |
struct nvm_dev *dev = tgt_dev->parent; |
dab8ee9e8 lightnvm: cleanup... |
1001 |
nvm_ppa_tgt_to_dev(tgt_dev, &ppa, 1); |
333ba053d lightnvm: transfo... |
1002 |
|
aff3fb18f lightnvm: move ba... |
1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 |
if (dev->geo.version == NVM_OCSSD_SPEC_12) return nvm_get_bb_meta(dev, (sector_t)ppa.ppa, nchks, meta); return dev->ops->get_chk_meta(dev, (sector_t)ppa.ppa, nchks, meta); } EXPORT_SYMBOL_GPL(nvm_get_chunk_meta); int nvm_set_chunk_meta(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *ppas, int nr_ppas, int type) { struct nvm_dev *dev = tgt_dev->parent; struct nvm_rq rqd; int ret; if (dev->geo.version == NVM_OCSSD_SPEC_20) return 0; if (nr_ppas > NVM_MAX_VLBA) { |
43db059ea lightnvm: introdu... |
1021 1022 |
pr_err("unable to update all blocks atomically "); |
aff3fb18f lightnvm: move ba... |
1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 |
return -EINVAL; } memset(&rqd, 0, sizeof(struct nvm_rq)); nvm_set_rqd_ppalist(tgt_dev, &rqd, ppas, nr_ppas); nvm_rq_tgt_to_dev(tgt_dev, &rqd); ret = dev->ops->set_bb_tbl(dev, &rqd.ppa_addr, rqd.nr_ppas, type); nvm_free_rqd_ppalist(tgt_dev, &rqd); if (ret) return -EINVAL; return 0; |
333ba053d lightnvm: transfo... |
1037 |
} |
aff3fb18f lightnvm: move ba... |
1038 |
EXPORT_SYMBOL_GPL(nvm_set_chunk_meta); |
333ba053d lightnvm: transfo... |
1039 |
|
cd9e9808d lightnvm: Support... |
1040 1041 |
static int nvm_core_init(struct nvm_dev *dev) { |
8e79b5cb1 lightnvm: move bl... |
1042 |
struct nvm_geo *geo = &dev->geo; |
7f7c5d03c lightnvm: avoid m... |
1043 |
int ret; |
cd9e9808d lightnvm: Support... |
1044 |
|
fae7fae40 lightnvm: make ge... |
1045 |
dev->lun_map = kcalloc(BITS_TO_LONGS(geo->all_luns), |
da1e28491 lightnvm: add a b... |
1046 1047 1048 |
sizeof(unsigned long), GFP_KERNEL); if (!dev->lun_map) return -ENOMEM; |
7f7c5d03c lightnvm: avoid m... |
1049 |
|
ade69e243 lightnvm: merge g... |
1050 1051 |
INIT_LIST_HEAD(&dev->area_list); INIT_LIST_HEAD(&dev->targets); |
e3eb3799f lightnvm: core on... |
1052 |
mutex_init(&dev->mlock); |
4c9dacb82 lightnvm: specify... |
1053 |
spin_lock_init(&dev->lock); |
cd9e9808d lightnvm: Support... |
1054 |
|
ade69e243 lightnvm: merge g... |
1055 1056 1057 |
ret = nvm_register_map(dev); if (ret) goto err_fmtype; |
ac81bfa98 nvme: refactor na... |
1058 |
|
cd9e9808d lightnvm: Support... |
1059 |
return 0; |
7f7c5d03c lightnvm: avoid m... |
1060 1061 1062 |
err_fmtype: kfree(dev->lun_map); return ret; |
cd9e9808d lightnvm: Support... |
1063 |
} |
e69397ea0 lightnvm: track i... |
1064 |
static void nvm_free(struct kref *ref) |
cd9e9808d lightnvm: Support... |
1065 |
{ |
e69397ea0 lightnvm: track i... |
1066 |
struct nvm_dev *dev = container_of(ref, struct nvm_dev, ref); |
cd9e9808d lightnvm: Support... |
1067 |
|
40267efdd lightnvm: expose ... |
1068 1069 |
if (dev->dma_pool) dev->ops->destroy_dma_pool(dev->dma_pool); |
e69397ea0 lightnvm: track i... |
1070 1071 |
if (dev->rmap) nvm_unregister_map(dev); |
7f7c5d03c lightnvm: avoid m... |
1072 |
kfree(dev->lun_map); |
40267efdd lightnvm: expose ... |
1073 |
kfree(dev); |
cd9e9808d lightnvm: Support... |
1074 1075 1076 1077 |
} static int nvm_init(struct nvm_dev *dev) { |
8e79b5cb1 lightnvm: move bl... |
1078 |
struct nvm_geo *geo = &dev->geo; |
480fc0db8 lightnvm: wrong r... |
1079 |
int ret = -EINVAL; |
cd9e9808d lightnvm: Support... |
1080 |
|
e46f4e482 lightnvm: simplif... |
1081 |
if (dev->ops->identity(dev)) { |
43db059ea lightnvm: introdu... |
1082 1083 |
pr_err("device could not be identified "); |
cd9e9808d lightnvm: Support... |
1084 1085 |
goto err; } |
43db059ea lightnvm: introdu... |
1086 1087 1088 |
pr_debug("ver:%u.%u nvm_vendor:%x ", geo->major_ver_id, geo->minor_ver_id, geo->vmnt); |
cd9e9808d lightnvm: Support... |
1089 |
|
cd9e9808d lightnvm: Support... |
1090 1091 |
ret = nvm_core_init(dev); if (ret) { |
43db059ea lightnvm: introdu... |
1092 1093 |
pr_err("could not initialize core structures. "); |
cd9e9808d lightnvm: Support... |
1094 1095 |
goto err; } |
43db059ea lightnvm: introdu... |
1096 1097 |
pr_info("registered %s [%u/%u/%u/%u/%u] ", |
a40afad90 lightnvm: normali... |
1098 1099 1100 |
dev->name, dev->geo.ws_min, dev->geo.ws_opt, dev->geo.num_chk, dev->geo.all_luns, dev->geo.num_ch); |
cd9e9808d lightnvm: Support... |
1101 1102 |
return 0; err: |
43db059ea lightnvm: introdu... |
1103 1104 |
pr_err("failed to initialize nvm "); |
cd9e9808d lightnvm: Support... |
1105 1106 |
return ret; } |
b0b4e09c1 lightnvm: control... |
1107 |
struct nvm_dev *nvm_alloc_dev(int node) |
cd9e9808d lightnvm: Support... |
1108 |
{ |
e69397ea0 lightnvm: track i... |
1109 1110 1111 1112 1113 1114 1115 |
struct nvm_dev *dev; dev = kzalloc_node(sizeof(struct nvm_dev), GFP_KERNEL, node); if (dev) kref_init(&dev->ref); return dev; |
cd9e9808d lightnvm: Support... |
1116 |
} |
b0b4e09c1 lightnvm: control... |
1117 |
EXPORT_SYMBOL(nvm_alloc_dev); |
cd9e9808d lightnvm: Support... |
1118 |
|
b0b4e09c1 lightnvm: control... |
1119 |
int nvm_register(struct nvm_dev *dev) |
cd9e9808d lightnvm: Support... |
1120 |
{ |
24828d053 lightnvm: dynamic... |
1121 |
int ret, exp_pool_size; |
cd9e9808d lightnvm: Support... |
1122 |
|
e69397ea0 lightnvm: track i... |
1123 1124 |
if (!dev->q || !dev->ops) { kref_put(&dev->ref, nvm_free); |
ade69e243 lightnvm: merge g... |
1125 |
return -EINVAL; |
e69397ea0 lightnvm: track i... |
1126 |
} |
cd9e9808d lightnvm: Support... |
1127 |
|
85136c010 lightnvm: simplif... |
1128 |
ret = nvm_init(dev); |
e69397ea0 lightnvm: track i... |
1129 1130 |
if (ret) { kref_put(&dev->ref, nvm_free); |
85136c010 lightnvm: simplif... |
1131 |
return ret; |
e69397ea0 lightnvm: track i... |
1132 |
} |
85136c010 lightnvm: simplif... |
1133 |
|
24828d053 lightnvm: dynamic... |
1134 1135 1136 1137 1138 1139 |
exp_pool_size = max_t(int, PAGE_SIZE, (NVM_MAX_VLBA * (sizeof(u64) + dev->geo.sos))); exp_pool_size = round_up(exp_pool_size, PAGE_SIZE); dev->dma_pool = dev->ops->create_dma_pool(dev, "ppalist", exp_pool_size); |
89a09c564 lightnvm: remove ... |
1140 |
if (!dev->dma_pool) { |
43db059ea lightnvm: introdu... |
1141 1142 |
pr_err("could not create dma pool "); |
e69397ea0 lightnvm: track i... |
1143 |
kref_put(&dev->ref, nvm_free); |
89a09c564 lightnvm: remove ... |
1144 |
return -ENOMEM; |
cd9e9808d lightnvm: Support... |
1145 |
} |
762796bc9 lightnvm: fix med... |
1146 |
/* register device with a supported media manager */ |
edad2e660 lightnvm: prematu... |
1147 1148 1149 |
down_write(&nvm_lock); list_add(&dev->devices, &nvm_devices); up_write(&nvm_lock); |
cd9e9808d lightnvm: Support... |
1150 |
return 0; |
cd9e9808d lightnvm: Support... |
1151 1152 |
} EXPORT_SYMBOL(nvm_register); |
b0b4e09c1 lightnvm: control... |
1153 |
void nvm_unregister(struct nvm_dev *dev) |
cd9e9808d lightnvm: Support... |
1154 |
{ |
ade69e243 lightnvm: merge g... |
1155 1156 1157 1158 1159 1160 |
struct nvm_target *t, *tmp; mutex_lock(&dev->mlock); list_for_each_entry_safe(t, tmp, &dev->targets, list) { if (t->dev->parent != dev) continue; |
a7c9e9109 lightnvm: pass fl... |
1161 |
__nvm_remove_target(t, false); |
e69397ea0 lightnvm: track i... |
1162 |
kref_put(&dev->ref, nvm_free); |
ade69e243 lightnvm: merge g... |
1163 1164 |
} mutex_unlock(&dev->mlock); |
d0a712ceb lightnvm: missing... |
1165 |
down_write(&nvm_lock); |
cd9e9808d lightnvm: Support... |
1166 1167 |
list_del(&dev->devices); up_write(&nvm_lock); |
c1480ad59 lightnvm: prevent... |
1168 |
|
e69397ea0 lightnvm: track i... |
1169 |
kref_put(&dev->ref, nvm_free); |
cd9e9808d lightnvm: Support... |
1170 1171 |
} EXPORT_SYMBOL(nvm_unregister); |
cd9e9808d lightnvm: Support... |
1172 1173 1174 |
static int __nvm_configure_create(struct nvm_ioctl_create *create) { struct nvm_dev *dev; |
e69397ea0 lightnvm: track i... |
1175 |
int ret; |
cd9e9808d lightnvm: Support... |
1176 |
|
d0a712ceb lightnvm: missing... |
1177 |
down_write(&nvm_lock); |
cd9e9808d lightnvm: Support... |
1178 |
dev = nvm_find_nvm_dev(create->dev); |
d0a712ceb lightnvm: missing... |
1179 |
up_write(&nvm_lock); |
b76eb20bb lightnvm: move ta... |
1180 |
|
cd9e9808d lightnvm: Support... |
1181 |
if (!dev) { |
43db059ea lightnvm: introdu... |
1182 1183 |
pr_err("device not found "); |
cd9e9808d lightnvm: Support... |
1184 1185 |
return -EINVAL; } |
e69397ea0 lightnvm: track i... |
1186 1187 1188 1189 1190 1191 |
kref_get(&dev->ref); ret = nvm_create_tgt(dev, create); if (ret) kref_put(&dev->ref, nvm_free); return ret; |
cd9e9808d lightnvm: Support... |
1192 |
} |
cd9e9808d lightnvm: Support... |
1193 1194 1195 1196 1197 |
static long nvm_ioctl_info(struct file *file, void __user *arg) { struct nvm_ioctl_info *info; struct nvm_tgt_type *tt; int tgt_iter = 0; |
cd9e9808d lightnvm: Support... |
1198 1199 1200 1201 1202 1203 1204 |
info = memdup_user(arg, sizeof(struct nvm_ioctl_info)); if (IS_ERR(info)) return -EFAULT; info->version[0] = NVM_VERSION_MAJOR; info->version[1] = NVM_VERSION_MINOR; info->version[2] = NVM_VERSION_PATCH; |
88d31ea26 lightnvm: protect... |
1205 |
down_write(&nvm_tgtt_lock); |
6063fe399 lightnvm: rename ... |
1206 |
list_for_each_entry(tt, &nvm_tgt_types, list) { |
cd9e9808d lightnvm: Support... |
1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 |
struct nvm_ioctl_info_tgt *tgt = &info->tgts[tgt_iter]; tgt->version[0] = tt->version[0]; tgt->version[1] = tt->version[1]; tgt->version[2] = tt->version[2]; strncpy(tgt->tgtname, tt->name, NVM_TTYPE_NAME_MAX); tgt_iter++; } info->tgtsize = tgt_iter; |
88d31ea26 lightnvm: protect... |
1218 |
up_write(&nvm_tgtt_lock); |
cd9e9808d lightnvm: Support... |
1219 |
|
76e25081b lightnvm: fix ioc... |
1220 1221 |
if (copy_to_user(arg, info, sizeof(struct nvm_ioctl_info))) { kfree(info); |
cd9e9808d lightnvm: Support... |
1222 |
return -EFAULT; |
76e25081b lightnvm: fix ioc... |
1223 |
} |
cd9e9808d lightnvm: Support... |
1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 |
kfree(info); return 0; } static long nvm_ioctl_get_devices(struct file *file, void __user *arg) { struct nvm_ioctl_get_devices *devices; struct nvm_dev *dev; int i = 0; |
cd9e9808d lightnvm: Support... |
1234 1235 1236 1237 1238 1239 1240 |
devices = kzalloc(sizeof(struct nvm_ioctl_get_devices), GFP_KERNEL); if (!devices) return -ENOMEM; down_write(&nvm_lock); list_for_each_entry(dev, &nvm_devices, devices) { struct nvm_ioctl_device_info *info = &devices->info[i]; |
6eb082452 lightnvm: convert... |
1241 |
strlcpy(info->devname, dev->name, sizeof(info->devname)); |
cd9e9808d lightnvm: Support... |
1242 |
|
ade69e243 lightnvm: merge g... |
1243 1244 1245 1246 |
/* kept for compatibility */ info->bmversion[0] = 1; info->bmversion[1] = 0; info->bmversion[2] = 0; |
6eb082452 lightnvm: convert... |
1247 |
strlcpy(info->bmname, "gennvm", sizeof(info->bmname)); |
cd9e9808d lightnvm: Support... |
1248 |
i++; |
ade69e243 lightnvm: merge g... |
1249 |
|
a48faebe6 lightnvm: fix out... |
1250 1251 1252 1253 |
if (i >= ARRAY_SIZE(devices->info)) { pr_err("max %zd devices can be reported. ", ARRAY_SIZE(devices->info)); |
cd9e9808d lightnvm: Support... |
1254 1255 1256 1257 1258 1259 |
break; } } up_write(&nvm_lock); devices->nr_devices = i; |
76e25081b lightnvm: fix ioc... |
1260 1261 1262 |
if (copy_to_user(arg, devices, sizeof(struct nvm_ioctl_get_devices))) { kfree(devices); |
cd9e9808d lightnvm: Support... |
1263 |
return -EFAULT; |
76e25081b lightnvm: fix ioc... |
1264 |
} |
cd9e9808d lightnvm: Support... |
1265 1266 1267 1268 1269 1270 1271 1272 |
kfree(devices); return 0; } static long nvm_ioctl_dev_create(struct file *file, void __user *arg) { struct nvm_ioctl_create create; |
cd9e9808d lightnvm: Support... |
1273 1274 |
if (copy_from_user(&create, arg, sizeof(struct nvm_ioctl_create))) return -EFAULT; |
e53927393 lightnvm: set tar... |
1275 1276 |
if (create.conf.type == NVM_CONFIG_TYPE_EXTENDED && create.conf.e.rsv != 0) { |
43db059ea lightnvm: introdu... |
1277 1278 |
pr_err("reserved config field in use "); |
e53927393 lightnvm: set tar... |
1279 1280 |
return -EINVAL; } |
cd9e9808d lightnvm: Support... |
1281 1282 1283 1284 1285 |
create.dev[DISK_NAME_LEN - 1] = '\0'; create.tgttype[NVM_TTYPE_NAME_MAX - 1] = '\0'; create.tgtname[DISK_NAME_LEN - 1] = '\0'; if (create.flags != 0) { |
4af3f75d7 lightnvm: allow t... |
1286 1287 1288 1289 1290 1291 1292 |
__u32 flags = create.flags; /* Check for valid flags */ if (flags & NVM_TARGET_FACTORY) flags &= ~NVM_TARGET_FACTORY; if (flags) { |
43db059ea lightnvm: introdu... |
1293 1294 |
pr_err("flag not supported "); |
4af3f75d7 lightnvm: allow t... |
1295 1296 |
return -EINVAL; } |
cd9e9808d lightnvm: Support... |
1297 1298 1299 1300 1301 1302 1303 1304 |
} return __nvm_configure_create(&create); } static long nvm_ioctl_dev_remove(struct file *file, void __user *arg) { struct nvm_ioctl_remove remove; |
cd9e9808d lightnvm: Support... |
1305 1306 1307 1308 1309 1310 |
if (copy_from_user(&remove, arg, sizeof(struct nvm_ioctl_remove))) return -EFAULT; remove.tgtname[DISK_NAME_LEN - 1] = '\0'; if (remove.flags != 0) { |
43db059ea lightnvm: introdu... |
1311 1312 |
pr_err("no flags supported "); |
cd9e9808d lightnvm: Support... |
1313 1314 |
return -EINVAL; } |
843f2edbd lightnvm: do not ... |
1315 |
return nvm_remove_tgt(&remove); |
cd9e9808d lightnvm: Support... |
1316 |
} |
ade69e243 lightnvm: merge g... |
1317 |
/* kept for compatibility reasons */ |
556961542 lightnvm: introdu... |
1318 1319 1320 |
static long nvm_ioctl_dev_init(struct file *file, void __user *arg) { struct nvm_ioctl_dev_init init; |
556961542 lightnvm: introdu... |
1321 1322 1323 1324 |
if (copy_from_user(&init, arg, sizeof(struct nvm_ioctl_dev_init))) return -EFAULT; if (init.flags != 0) { |
43db059ea lightnvm: introdu... |
1325 1326 |
pr_err("no flags supported "); |
556961542 lightnvm: introdu... |
1327 1328 |
return -EINVAL; } |
ade69e243 lightnvm: merge g... |
1329 |
return 0; |
556961542 lightnvm: introdu... |
1330 |
} |
ade69e243 lightnvm: merge g... |
1331 |
/* Kept for compatibility reasons */ |
8b4970c41 lightnvm: introdu... |
1332 1333 1334 |
static long nvm_ioctl_dev_factory(struct file *file, void __user *arg) { struct nvm_ioctl_dev_factory fact; |
8b4970c41 lightnvm: introdu... |
1335 |
|
8b4970c41 lightnvm: introdu... |
1336 1337 1338 1339 1340 1341 1342 |
if (copy_from_user(&fact, arg, sizeof(struct nvm_ioctl_dev_factory))) return -EFAULT; fact.dev[DISK_NAME_LEN - 1] = '\0'; if (fact.flags & ~(NVM_FACTORY_NR_BITS - 1)) return -EINVAL; |
bf6431856 lightnvm: allow t... |
1343 |
return 0; |
8b4970c41 lightnvm: introdu... |
1344 |
} |
cd9e9808d lightnvm: Support... |
1345 1346 1347 |
static long nvm_ctl_ioctl(struct file *file, uint cmd, unsigned long arg) { void __user *argp = (void __user *)arg; |
40f962d78 lightnvm: central... |
1348 1349 |
if (!capable(CAP_SYS_ADMIN)) return -EPERM; |
cd9e9808d lightnvm: Support... |
1350 1351 1352 1353 1354 1355 1356 1357 1358 |
switch (cmd) { case NVM_INFO: return nvm_ioctl_info(file, argp); case NVM_GET_DEVICES: return nvm_ioctl_get_devices(file, argp); case NVM_DEV_CREATE: return nvm_ioctl_dev_create(file, argp); case NVM_DEV_REMOVE: return nvm_ioctl_dev_remove(file, argp); |
556961542 lightnvm: introdu... |
1359 1360 |
case NVM_DEV_INIT: return nvm_ioctl_dev_init(file, argp); |
8b4970c41 lightnvm: introdu... |
1361 1362 |
case NVM_DEV_FACTORY: return nvm_ioctl_dev_factory(file, argp); |
cd9e9808d lightnvm: Support... |
1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 |
} return 0; } static const struct file_operations _ctl_fops = { .open = nonseekable_open, .unlocked_ioctl = nvm_ctl_ioctl, .owner = THIS_MODULE, .llseek = noop_llseek, }; static struct miscdevice _nvm_misc = { .minor = MISC_DYNAMIC_MINOR, .name = "lightnvm", .nodename = "lightnvm/control", .fops = &_ctl_fops, }; |
389b2a1c0 lightnvm: make co... |
1380 |
builtin_misc_device(_nvm_misc); |