Blame view
drivers/lightnvm/pblk.h
39 KB
02a1520d5 lightnvm: pblk: a... |
1 |
/* SPDX-License-Identifier: GPL-2.0 */ |
a4bd217b4 lightnvm: physica... |
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 |
/* * Copyright (C) 2015 IT University of Copenhagen (rrpc.h) * Copyright (C) 2016 CNEX Labs * Initial release: Matias Bjorling <matias@cnexlabs.com> * Write buffering: Javier Gonzalez <javier@cnexlabs.com> * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License version * 2 as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * Implementation of a Physical Block-device target for Open-channel SSDs. * */ #ifndef PBLK_H_ #define PBLK_H_ #include <linux/blkdev.h> #include <linux/blk-mq.h> #include <linux/bio.h> #include <linux/module.h> #include <linux/kthread.h> #include <linux/vmalloc.h> #include <linux/crc32.h> #include <linux/uuid.h> #include <linux/lightnvm.h> /* Run only GC if less than 1/X blocks are free */ #define GC_LIMIT_INVERSE 5 #define GC_TIME_MSECS 1000 #define PBLK_SECTOR (512) #define PBLK_EXPOSED_PAGE_SIZE (4096) |
a4bd217b4 lightnvm: physica... |
41 |
|
ef5764946 lightnvm: pblk: s... |
42 |
#define PBLK_NR_CLOSE_JOBS (4) |
a4bd217b4 lightnvm: physica... |
43 |
#define PBLK_CACHE_NAME_LEN (DISK_NAME_LEN + 16) |
a4bd217b4 lightnvm: physica... |
44 45 46 47 |
/* Max 512 LUNs per device */ #define PBLK_MAX_LUNS_BITMAP (4) #define NR_PHY_IN_LOG (PBLK_EXPOSED_PAGE_SIZE / PBLK_SECTOR) |
0d880398c lightnvm: pblk: d... |
48 |
/* Static pool sizes */ |
b84ae4a8b lightnvm: pblk: s... |
49 |
#define PBLK_GEN_WS_POOL_SIZE (2) |
e53927393 lightnvm: set tar... |
50 |
#define PBLK_DEFAULT_OP (11) |
a4bd217b4 lightnvm: physica... |
51 |
enum { |
e2cddf208 lightnvm: pblk: i... |
52 53 54 |
PBLK_READ = READ, PBLK_WRITE = WRITE,/* Write from write buffer */ PBLK_WRITE_INT, /* Internal write - no write buffer */ |
8f554597e lightnvm: pblk: d... |
55 |
PBLK_READ_RECOV, /* Recovery read - errors allowed */ |
e2cddf208 lightnvm: pblk: i... |
56 57 58 59 |
PBLK_ERASE, }; enum { |
a4bd217b4 lightnvm: physica... |
60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 |
/* IO Types */ PBLK_IOTYPE_USER = 1 << 0, PBLK_IOTYPE_GC = 1 << 1, /* Write buffer flags */ PBLK_FLUSH_ENTRY = 1 << 2, PBLK_WRITTEN_DATA = 1 << 3, PBLK_SUBMITTED_ENTRY = 1 << 4, PBLK_WRITABLE_ENTRY = 1 << 5, }; enum { PBLK_BLK_ST_OPEN = 0x1, PBLK_BLK_ST_CLOSED = 0x2, }; |
4209c31c0 lightnvm: pblk: a... |
75 76 77 78 79 |
enum { PBLK_CHUNK_RESET_START, PBLK_CHUNK_RESET_DONE, PBLK_CHUNK_RESET_FAILED, }; |
b20ba1bc7 lightnvm: pblk: r... |
80 81 82 83 |
struct pblk_sec_meta { u64 reserved; __le64 lba; }; |
a4bd217b4 lightnvm: physica... |
84 85 86 |
/* The number of GC lists and the rate-limiter states go together. This way the * rate-limiter can dictate how much GC is needed based on resource utilization. */ |
48b8d2089 lightnvm: pblk: g... |
87 |
#define PBLK_GC_NR_LISTS 4 |
a4bd217b4 lightnvm: physica... |
88 89 |
enum { |
48b8d2089 lightnvm: pblk: g... |
90 91 92 93 94 |
PBLK_RL_OFF = 0, PBLK_RL_WERR = 1, PBLK_RL_HIGH = 2, PBLK_RL_MID = 3, PBLK_RL_LOW = 4 |
a4bd217b4 lightnvm: physica... |
95 |
}; |
afdc23c91 lightnvm: pblk: u... |
96 |
#define pblk_dma_ppa_size (sizeof(u64) * NVM_MAX_VLBA) |
a4bd217b4 lightnvm: physica... |
97 |
|
084ec9ba0 lightnvm: pblk: r... |
98 |
/* write buffer completion context */ |
a4bd217b4 lightnvm: physica... |
99 100 101 102 103 104 105 106 |
struct pblk_c_ctx { struct list_head list; /* Head for out-of-order completion */ unsigned long *lun_bitmap; /* Luns used on current request */ unsigned int sentry; unsigned int nr_valid; unsigned int nr_padded; }; |
a4809fee4 lightnvm: pblk: c... |
107 |
/* read context */ |
084ec9ba0 lightnvm: pblk: r... |
108 109 |
struct pblk_g_ctx { void *private; |
998ba6297 lightnvm: pblk: a... |
110 |
unsigned long start_time; |
a4809fee4 lightnvm: pblk: c... |
111 |
u64 lba; |
a4bd217b4 lightnvm: physica... |
112 |
}; |
ee8d5c1ad lightnvm: pblk: r... |
113 114 115 116 117 118 |
/* Pad context */ struct pblk_pad_rq { struct pblk *pblk; struct completion wait; struct kref ref; }; |
a4bd217b4 lightnvm: physica... |
119 120 121 122 |
/* Recovery context */ struct pblk_rec_ctx { struct pblk *pblk; struct nvm_rq *rqd; |
a4bd217b4 lightnvm: physica... |
123 124 125 126 127 128 129 130 |
struct work_struct ws_rec; }; /* Write context */ struct pblk_w_ctx { struct bio_list bios; /* Original bios - used for completion * in REQ_FUA, REQ_FLUSH case */ |
ef697902a lightnvm: assume ... |
131 |
u64 lba; /* Logic addr. associated with entry */ |
a4bd217b4 lightnvm: physica... |
132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 |
struct ppa_addr ppa; /* Physic addr. associated with entry */ int flags; /* Write context flags */ }; struct pblk_rb_entry { struct ppa_addr cacheline; /* Cacheline for this entry */ void *data; /* Pointer to data on this entry */ struct pblk_w_ctx w_ctx; /* Context for this entry */ struct list_head index; /* List head to enable indexes */ }; #define EMPTY_ENTRY (~0U) struct pblk_rb_pages { struct page *pages; int order; struct list_head list; }; struct pblk_rb { struct pblk_rb_entry *entries; /* Ring buffer entries */ unsigned int mem; /* Write offset - points to next * writable entry in memory */ unsigned int subm; /* Read offset - points to last entry * that has been submitted to the media * to be persisted */ unsigned int sync; /* Synced - backpointer that signals * the last submitted entry that has * been successfully persisted to media */ |
8154d296d lightnvm: pblk: r... |
164 |
unsigned int flush_point; /* Sync point - last entry that must be |
a4bd217b4 lightnvm: physica... |
165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 |
* flushed to the media. Used with * REQ_FLUSH and REQ_FUA */ unsigned int l2p_update; /* l2p update point - next entry for * which l2p mapping will be updated to * contain a device ppa address (instead * of a cacheline */ unsigned int nr_entries; /* Number of entries in write buffer - * must be a power of two */ unsigned int seg_size; /* Size of the data segments being * stored on each entry. Typically this * will be 4KB */ |
766c8ceb1 lightnvm: pblk: g... |
180 181 182 183 |
unsigned int back_thres; /* Threshold that shall be maintained by * the backpointer in order to respect * geo->mw_cunits on a per chunk basis */ |
a4bd217b4 lightnvm: physica... |
184 185 186 187 |
struct list_head pages; /* List of data pages */ spinlock_t w_lock; /* Write lock */ spinlock_t s_lock; /* Sync lock */ |
880eda544 lightnvm: move NV... |
188 |
#ifdef CONFIG_NVM_PBLK_DEBUG |
8154d296d lightnvm: pblk: r... |
189 |
atomic_t inflight_flush_point; /* Not served REQ_FLUSH | REQ_FUA */ |
a4bd217b4 lightnvm: physica... |
190 191 192 193 194 195 196 |
#endif }; #define PBLK_RECOVERY_SECTORS 16 struct pblk_lun { struct ppa_addr bppa; |
a4bd217b4 lightnvm: physica... |
197 198 199 200 201 202 |
struct semaphore wr_sem; }; struct pblk_gc_rq { struct pblk_line *line; void *data; |
afdc23c91 lightnvm: pblk: u... |
203 204 |
u64 paddr_list[NVM_MAX_VLBA]; u64 lba_list[NVM_MAX_VLBA]; |
a4bd217b4 lightnvm: physica... |
205 206 207 208 209 210 |
int nr_secs; int secs_to_gc; struct list_head list; }; struct pblk_gc { |
b20ba1bc7 lightnvm: pblk: r... |
211 212 213 |
/* These states are not protected by a lock since (i) they are in the * fast path, and (ii) they are not critical. */ |
a4bd217b4 lightnvm: physica... |
214 215 216 |
int gc_active; int gc_enabled; int gc_forced; |
a4bd217b4 lightnvm: physica... |
217 218 219 |
struct task_struct *gc_ts; struct task_struct *gc_writer_ts; |
b20ba1bc7 lightnvm: pblk: r... |
220 221 222 |
struct task_struct *gc_reader_ts; struct workqueue_struct *gc_line_reader_wq; |
a4bd217b4 lightnvm: physica... |
223 |
struct workqueue_struct *gc_reader_wq; |
b20ba1bc7 lightnvm: pblk: r... |
224 |
|
a4bd217b4 lightnvm: physica... |
225 |
struct timer_list gc_timer; |
b20ba1bc7 lightnvm: pblk: r... |
226 |
struct semaphore gc_sem; |
d6b992f7a lightnvm: pblk: g... |
227 228 229 230 |
atomic_t read_inflight_gc; /* Number of lines with inflight GC reads */ atomic_t pipeline_gc; /* Number of lines in the GC pipeline - * started reads to finished writes */ |
a4bd217b4 lightnvm: physica... |
231 |
int w_entries; |
b20ba1bc7 lightnvm: pblk: r... |
232 |
|
a4bd217b4 lightnvm: physica... |
233 |
struct list_head w_list; |
b20ba1bc7 lightnvm: pblk: r... |
234 |
struct list_head r_list; |
a4bd217b4 lightnvm: physica... |
235 236 237 |
spinlock_t lock; spinlock_t w_lock; |
b20ba1bc7 lightnvm: pblk: r... |
238 |
spinlock_t r_lock; |
a4bd217b4 lightnvm: physica... |
239 240 241 242 243 244 |
}; struct pblk_rl { unsigned int high; /* Upper threshold for rate limiter (free run - * user I/O rate limiter */ |
a4bd217b4 lightnvm: physica... |
245 |
unsigned int high_pw; /* High rounded up as a power of 2 */ |
b20ba1bc7 lightnvm: pblk: r... |
246 247 |
#define PBLK_USER_HIGH_THRS 8 /* Begin write limit at 12% available blks */ #define PBLK_USER_LOW_THRS 10 /* Aggressive GC at 10% available blocks */ |
a4bd217b4 lightnvm: physica... |
248 249 250 251 252 253 254 255 256 257 258 |
int rb_windows_pw; /* Number of rate windows in the write buffer * given as a power-of-2. This guarantees that * when user I/O is being rate limited, there * will be reserved enough space for the GC to * place its payload. A window is of * pblk->max_write_pgs size, which in NVMe is * 64, i.e., 256kb. */ int rb_budget; /* Total number of entries available for I/O */ int rb_user_max; /* Max buffer entries available for user I/O */ |
a4bd217b4 lightnvm: physica... |
259 260 261 |
int rb_gc_max; /* Max buffer entries available for GC I/O */ int rb_gc_rsv; /* Reserved buffer entries for GC I/O */ int rb_state; /* Rate-limiter current state */ |
da67e68fb lightnvm: pblk: a... |
262 |
int rb_max_io; /* Maximum size for an I/O giving the config */ |
588726d3e lightnvm: pblk: f... |
263 264 |
atomic_t rb_user_cnt; /* User I/O buffer counter */ |
a4bd217b4 lightnvm: physica... |
265 |
atomic_t rb_gc_cnt; /* GC I/O buffer counter */ |
588726d3e lightnvm: pblk: f... |
266 |
atomic_t rb_space; /* Space limit in case of reaching capacity */ |
a4bd217b4 lightnvm: physica... |
267 |
|
b20ba1bc7 lightnvm: pblk: r... |
268 |
int rsv_blocks; /* Reserved blocks for GC */ |
a4bd217b4 lightnvm: physica... |
269 |
int rb_user_active; |
b20ba1bc7 lightnvm: pblk: r... |
270 |
int rb_gc_active; |
48b8d2089 lightnvm: pblk: g... |
271 |
atomic_t werr_lines; /* Number of write error lines that needs gc */ |
a4bd217b4 lightnvm: physica... |
272 |
struct timer_list u_timer; |
a4bd217b4 lightnvm: physica... |
273 |
unsigned long total_blocks; |
a7689938e lightnvm: pblk: u... |
274 275 276 |
atomic_t free_blocks; /* Total number of free blocks (+ OP) */ atomic_t free_user_blocks; /* Number of user free blocks (no OP) */ |
a4bd217b4 lightnvm: physica... |
277 |
}; |
a4bd217b4 lightnvm: physica... |
278 279 280 281 282 283 284 285 286 |
#define PBLK_LINE_EMPTY (~0U) enum { /* Line Types */ PBLK_LINETYPE_FREE = 0, PBLK_LINETYPE_LOG = 1, PBLK_LINETYPE_DATA = 2, /* Line state */ |
32ef9412c lightnvm: pblk: i... |
287 |
PBLK_LINESTATE_NEW = 9, |
a4bd217b4 lightnvm: physica... |
288 289 290 291 292 293 294 295 296 297 298 299 300 301 |
PBLK_LINESTATE_FREE = 10, PBLK_LINESTATE_OPEN = 11, PBLK_LINESTATE_CLOSED = 12, PBLK_LINESTATE_GC = 13, PBLK_LINESTATE_BAD = 14, PBLK_LINESTATE_CORRUPT = 15, /* GC group */ PBLK_LINEGC_NONE = 20, PBLK_LINEGC_EMPTY = 21, PBLK_LINEGC_LOW = 22, PBLK_LINEGC_MID = 23, PBLK_LINEGC_HIGH = 24, PBLK_LINEGC_FULL = 25, |
48b8d2089 lightnvm: pblk: g... |
302 |
PBLK_LINEGC_WERR = 26 |
a4bd217b4 lightnvm: physica... |
303 304 305 |
}; #define PBLK_MAGIC 0x70626c6b /*pblk*/ |
d0ab0b1ab lightnvm: pblk: c... |
306 307 308 309 310 311 312 313 314 315 316 |
/* emeta/smeta persistent storage format versions: * Changes in major version requires offline migration. * Changes in minor version are handled automatically during * recovery. */ #define SMETA_VERSION_MAJOR (0) #define SMETA_VERSION_MINOR (1) #define EMETA_VERSION_MAJOR (0) |
76758390f lightnvm: pblk: e... |
317 |
#define EMETA_VERSION_MINOR (2) |
a4bd217b4 lightnvm: physica... |
318 319 320 321 322 323 |
struct line_header { __le32 crc; __le32 identifier; /* pblk identifier */ __u8 uuid[16]; /* instance uuid */ __le16 type; /* line type */ |
d0ab0b1ab lightnvm: pblk: c... |
324 325 |
__u8 version_major; /* version major */ __u8 version_minor; /* version minor */ |
a4bd217b4 lightnvm: physica... |
326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 |
__le32 id; /* line id for current line */ }; struct line_smeta { struct line_header header; __le32 crc; /* Full structure including struct crc */ /* Previous line metadata */ __le32 prev_id; /* Line id for previous line */ /* Current line metadata */ __le64 seq_nr; /* Sequence number for current line */ /* Active writers */ __le32 window_wr_lun; /* Number of parallel LUNs to write */ __le32 rsvd[2]; |
dd2a43437 lightnvm: pblk: s... |
343 344 |
__le64 lun_bitmap[]; |
a4bd217b4 lightnvm: physica... |
345 |
}; |
76758390f lightnvm: pblk: e... |
346 |
|
a4bd217b4 lightnvm: physica... |
347 |
/* |
dd2a43437 lightnvm: pblk: s... |
348 349 350 351 |
* Metadata layout in media: * First sector: * 1. struct line_emeta * 2. bad block bitmap (u64 * window_wr_lun) |
76758390f lightnvm: pblk: e... |
352 |
* 3. write amplification counters |
dd2a43437 lightnvm: pblk: s... |
353 354 355 356 |
* Mid sectors (start at lbas_sector): * 3. nr_lbas (u64) forming lba list * Last sectors (start at vsc_sector): * 4. u32 valid sector count (vsc) for all lines (~0U: free line) |
a4bd217b4 lightnvm: physica... |
357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 |
*/ struct line_emeta { struct line_header header; __le32 crc; /* Full structure including struct crc */ /* Previous line metadata */ __le32 prev_id; /* Line id for prev line */ /* Current line metadata */ __le64 seq_nr; /* Sequence number for current line */ /* Active writers */ __le32 window_wr_lun; /* Number of parallel LUNs to write */ /* Bookkeeping for recovery */ __le32 next_id; /* Line id for next line */ __le64 nr_lbas; /* Number of lbas mapped in line */ __le64 nr_valid_lbas; /* Number of valid lbas mapped in line */ |
76758390f lightnvm: pblk: e... |
376 377 378 379 380 381 382 383 384 |
__le64 bb_bitmap[]; /* Updated bad block bitmap for line */ }; /* Write amplification counters stored on media */ struct wa_counters { __le64 user; /* Number of user written sectors */ __le64 gc; /* Number of sectors written by GC*/ __le64 pad; /* Number of padded sectors */ |
dd2a43437 lightnvm: pblk: s... |
385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 |
}; struct pblk_emeta { struct line_emeta *buf; /* emeta buffer in media format */ int mem; /* Write offset - points to next * writable entry in memory */ atomic_t sync; /* Synced - backpointer that signals the * last entry that has been successfully * persisted to media */ unsigned int nr_entries; /* Number of emeta entries */ }; struct pblk_smeta { struct line_smeta *buf; /* smeta buffer in persistent format */ |
a4bd217b4 lightnvm: physica... |
401 |
}; |
48b8d2089 lightnvm: pblk: g... |
402 403 |
struct pblk_w_err_gc { int has_write_err; |
f2e024570 lightnvm: pblk: G... |
404 |
int has_gc_err; |
48b8d2089 lightnvm: pblk: g... |
405 406 |
__le64 *lba_list; }; |
a4bd217b4 lightnvm: physica... |
407 408 409 410 411 412 413 414 415 416 417 418 419 |
struct pblk_line { struct pblk *pblk; unsigned int id; /* Line number corresponds to the * block line */ unsigned int seq_nr; /* Unique line sequence number */ int state; /* PBLK_LINESTATE_X */ int type; /* PBLK_LINETYPE_X */ int gc_group; /* PBLK_LINEGC_X */ struct list_head list; /* Free, GC lists */ unsigned long *lun_bitmap; /* Bitmap for LUNs mapped in line */ |
32ef9412c lightnvm: pblk: i... |
420 |
struct nvm_chk_meta *chks; /* Chunks forming line */ |
dd2a43437 lightnvm: pblk: s... |
421 422 |
struct pblk_smeta *smeta; /* Start metadata */ struct pblk_emeta *emeta; /* End medatada */ |
a4bd217b4 lightnvm: physica... |
423 |
int meta_line; /* Metadata line id */ |
dd2a43437 lightnvm: pblk: s... |
424 |
int meta_distance; /* Distance between data and metadata */ |
a4bd217b4 lightnvm: physica... |
425 426 427 |
u64 emeta_ssec; /* Sector where emeta starts */ unsigned int sec_in_line; /* Number of usable secs in line */ |
a44f53faf lightnvm: pblk: f... |
428 |
atomic_t blk_in_line; /* Number of good blocks in line */ |
a4bd217b4 lightnvm: physica... |
429 430 431 432 433 |
unsigned long *blk_bitmap; /* Bitmap for valid/invalid blocks */ unsigned long *erase_bitmap; /* Bitmap for erased blocks */ unsigned long *map_bitmap; /* Bitmap for mapped sectors in line */ unsigned long *invalid_bitmap; /* Bitmap for invalid sectors in line */ |
a44f53faf lightnvm: pblk: f... |
434 |
atomic_t left_eblks; /* Blocks left for erasing */ |
a4bd217b4 lightnvm: physica... |
435 436 437 |
atomic_t left_seblks; /* Blocks left for sync erasing */ int left_msecs; /* Sectors left for mapping */ |
a4bd217b4 lightnvm: physica... |
438 |
unsigned int cur_sec; /* Sector map pointer */ |
dd2a43437 lightnvm: pblk: s... |
439 440 441 |
unsigned int nr_valid_lbas; /* Number of valid lbas in line */ __le32 *vsc; /* Valid sector count in line */ |
a4bd217b4 lightnvm: physica... |
442 443 |
struct kref ref; /* Write buffer L2P references */ |
0586942f0 lightnvm: pblk: f... |
444 |
atomic_t sec_to_update; /* Outstanding L2P updates to ppa */ |
a4bd217b4 lightnvm: physica... |
445 |
|
48b8d2089 lightnvm: pblk: g... |
446 |
struct pblk_w_err_gc *w_err_gc; /* Write error gc recovery metadata */ |
a4bd217b4 lightnvm: physica... |
447 448 |
spinlock_t lock; /* Necessary for invalid_bitmap only */ }; |
a44f53faf lightnvm: pblk: f... |
449 |
#define PBLK_DATA_LINES 4 |
a4bd217b4 lightnvm: physica... |
450 |
|
dd2a43437 lightnvm: pblk: s... |
451 |
enum { |
dd2a43437 lightnvm: pblk: s... |
452 453 454 |
PBLK_EMETA_TYPE_HEADER = 1, /* struct line_emeta first sector */ PBLK_EMETA_TYPE_LLBA = 2, /* lba list - type: __le64 */ PBLK_EMETA_TYPE_VSC = 3, /* vsc list - type: __le32 */ |
a4bd217b4 lightnvm: physica... |
455 456 457 458 459 460 461 462 463 464 465 466 |
}; struct pblk_line_mgmt { int nr_lines; /* Total number of full lines */ int nr_free_lines; /* Number of full lines in free list */ /* Free lists - use free_lock */ struct list_head free_list; /* Full lines ready to use */ struct list_head corrupt_list; /* Full lines corrupted */ struct list_head bad_list; /* Full lines bad */ /* GC lists - use gc_lock */ |
b20ba1bc7 lightnvm: pblk: r... |
467 |
struct list_head *gc_lists[PBLK_GC_NR_LISTS]; |
a4bd217b4 lightnvm: physica... |
468 469 470 |
struct list_head gc_high_list; /* Full lines ready to GC, high isc */ struct list_head gc_mid_list; /* Full lines ready to GC, mid isc */ struct list_head gc_low_list; /* Full lines ready to GC, low isc */ |
48b8d2089 lightnvm: pblk: g... |
471 |
struct list_head gc_werr_list; /* Write err recovery list */ |
a4bd217b4 lightnvm: physica... |
472 473 474 475 476 477 478 |
struct list_head gc_full_list; /* Full lines ready to GC, no valid */ struct list_head gc_empty_list; /* Full lines close, all valid */ struct pblk_line *log_line; /* Current FTL log line */ struct pblk_line *data_line; /* Current data line */ struct pblk_line *log_next; /* Next FTL log line */ struct pblk_line *data_next; /* Next data line */ |
dd2a43437 lightnvm: pblk: s... |
479 480 481 |
struct list_head emeta_list; /* Lines queued to schedule emeta */ __le32 *vsc_list; /* Valid sector counts for all lines */ |
a4bd217b4 lightnvm: physica... |
482 |
/* Pre-allocated metadata for data lines */ |
dd2a43437 lightnvm: pblk: s... |
483 484 |
struct pblk_smeta *sline_meta[PBLK_DATA_LINES]; struct pblk_emeta *eline_meta[PBLK_DATA_LINES]; |
a4bd217b4 lightnvm: physica... |
485 |
unsigned long meta_bitmap; |
53d82db69 lightnvm: pblk: a... |
486 487 488 |
/* Cache and mempool for map/invalid bitmaps */ struct kmem_cache *bitmap_cache; mempool_t *bitmap_pool; |
a4bd217b4 lightnvm: physica... |
489 490 491 492 493 494 495 496 |
/* Helpers for fast bitmap calculations */ unsigned long *bb_template; unsigned long *bb_aux; unsigned long d_seq_nr; /* Data line unique sequence number */ unsigned long l_seq_nr; /* Log line unique sequence number */ spinlock_t free_lock; |
dd2a43437 lightnvm: pblk: s... |
497 |
spinlock_t close_lock; |
a4bd217b4 lightnvm: physica... |
498 499 500 501 502 |
spinlock_t gc_lock; }; struct pblk_line_meta { unsigned int smeta_len; /* Total length for smeta */ |
dd2a43437 lightnvm: pblk: s... |
503 504 505 |
unsigned int smeta_sec; /* Sectors needed for smeta */ unsigned int emeta_len[4]; /* Lengths for emeta: |
76758390f lightnvm: pblk: e... |
506 507 508 509 510 |
* [0]: Total * [1]: struct line_emeta + * bb_bitmap + struct wa_counters * [2]: L2P portion * [3]: vsc |
dd2a43437 lightnvm: pblk: s... |
511 512 513 514 |
*/ unsigned int emeta_sec[4]; /* Sectors needed for emeta. Same layout * as emeta_len */ |
a4bd217b4 lightnvm: physica... |
515 |
unsigned int emeta_bb; /* Boundary for bb that affects emeta */ |
dd2a43437 lightnvm: pblk: s... |
516 517 |
unsigned int vsc_list_len; /* Length for vsc list */ |
a4bd217b4 lightnvm: physica... |
518 519 520 521 522 523 |
unsigned int sec_bitmap_len; /* Length for sector bitmap in line */ unsigned int blk_bitmap_len; /* Length for block bitmap in line */ unsigned int lun_bitmap_len; /* Length for lun bitmap in line */ unsigned int blk_per_line; /* Number of blocks in a full line */ unsigned int sec_per_line; /* Number of sectors in a line */ |
dd2a43437 lightnvm: pblk: s... |
524 |
unsigned int dsec_per_line; /* Number of data sectors in a line */ |
a4bd217b4 lightnvm: physica... |
525 526 527 528 |
unsigned int min_blk_line; /* Min. number of good blocks in line */ unsigned int mid_thrs; /* Threshold for GC mid list */ unsigned int high_thrs; /* Threshold for GC high list */ |
dd2a43437 lightnvm: pblk: s... |
529 530 |
unsigned int meta_distance; /* Distance between data and metadata */ |
a4bd217b4 lightnvm: physica... |
531 |
}; |
588726d3e lightnvm: pblk: f... |
532 533 534 535 536 537 |
enum { PBLK_STATE_RUNNING = 0, PBLK_STATE_STOPPING = 1, PBLK_STATE_RECOVERING = 2, PBLK_STATE_STOPPED = 3, }; |
3b2a3ad11 lightnvm: pblk: i... |
538 539 540 541 542 543 544 545 546 547 548 |
/* Internal format to support not power-of-2 device formats */ struct pblk_addrf { /* gen to dev */ int sec_stripe; int ch_stripe; int lun_stripe; /* dev to gen */ int sec_lun_stripe; int sec_ws_stripe; }; |
a4bd217b4 lightnvm: physica... |
549 550 551 552 553 554 555 556 557 558 559 |
struct pblk { struct nvm_tgt_dev *dev; struct gendisk *disk; struct kobject kobj; struct pblk_lun *luns; struct pblk_line *lines; /* Line array */ struct pblk_line_mgmt l_mg; /* Line management */ struct pblk_line_meta lm; /* Line metadata */ |
3b2a3ad11 lightnvm: pblk: i... |
560 561 |
struct nvm_addrf addrf; /* Aligned address format */ struct pblk_addrf uaddrf; /* Unaligned address format */ |
bb845ae45 lightnvm: pblk: r... |
562 |
int addrf_len; |
a4bd217b4 lightnvm: physica... |
563 564 |
struct pblk_rb rwb; |
588726d3e lightnvm: pblk: f... |
565 |
int state; /* pblk line state */ |
a4bd217b4 lightnvm: physica... |
566 |
int min_write_pgs; /* Minimum amount of pages required by controller */ |
55d8ec353 lightnvm: pblk: s... |
567 |
int min_write_pgs_data; /* Minimum amount of payload pages */ |
a4bd217b4 lightnvm: physica... |
568 |
int max_write_pgs; /* Maximum amount of pages supported by controller */ |
faa79f27f lightnvm: pblk: a... |
569 |
int oob_meta_size; /* Size of OOB sector metadata */ |
a4bd217b4 lightnvm: physica... |
570 571 |
sector_t capacity; /* Device capacity when bad blocks are subtracted */ |
a7689938e lightnvm: pblk: u... |
572 573 574 |
int op; /* Percentage of device used for over-provisioning */ int op_blks; /* Number of blocks used for over-provisioning */ |
a4bd217b4 lightnvm: physica... |
575 576 577 |
/* pblk provisioning values. Used by rate limiter */ struct pblk_rl rl; |
c2e9f5d45 lightnvm: pblk: e... |
578 |
int sec_per_write; |
a4bd217b4 lightnvm: physica... |
579 |
|
7e0a0847e lightnvm: pblk: S... |
580 |
guid_t instance_uuid; |
76758390f lightnvm: pblk: e... |
581 582 583 584 585 586 587 588 589 590 |
/* Persistent write amplification counters, 4kb sector I/Os */ atomic64_t user_wa; /* Sectors written by user */ atomic64_t gc_wa; /* Sectors written by GC */ atomic64_t pad_wa; /* Padded sectors written */ /* Reset values for delta write amplification measurements */ u64 user_rst_wa; u64 gc_rst_wa; u64 pad_rst_wa; |
5d149bfab lightnvm: pblk: a... |
591 592 593 594 |
/* Counters used for calculating padding distribution */ atomic64_t *pad_dist; /* Padding distribution buckets */ u64 nr_flush_rst; /* Flushes reset value for pad dist.*/ atomic64_t nr_flush; /* Number of flush/fua I/O */ |
880eda544 lightnvm: move NV... |
595 |
#ifdef CONFIG_NVM_PBLK_DEBUG |
76758390f lightnvm: pblk: e... |
596 |
/* Non-persistent debug counters, 4kb sector I/Os */ |
a4bd217b4 lightnvm: physica... |
597 598 599 |
atomic_long_t inflight_writes; /* Inflight writes (user and gc) */ atomic_long_t padded_writes; /* Sectors padded due to flush/fua */ atomic_long_t padded_wb; /* Sectors padded in write buffer */ |
a4bd217b4 lightnvm: physica... |
600 601 602 |
atomic_long_t req_writes; /* Sectors stored on write buffer */ atomic_long_t sub_writes; /* Sectors submitted from buffer */ atomic_long_t sync_writes; /* Sectors synced to media */ |
a4bd217b4 lightnvm: physica... |
603 |
atomic_long_t inflight_reads; /* Inflight sector read requests */ |
db7ada33c lightnvm: pblk: a... |
604 |
atomic_long_t cache_reads; /* Read requests that hit the cache */ |
a4bd217b4 lightnvm: physica... |
605 606 607 608 609 610 611 612 613 614 615 616 617 618 |
atomic_long_t sync_reads; /* Completed sector read requests */ atomic_long_t recov_writes; /* Sectors submitted from recovery */ atomic_long_t recov_gc_writes; /* Sectors submitted from write GC */ atomic_long_t recov_gc_reads; /* Sectors submitted from read GC */ #endif spinlock_t lock; atomic_long_t read_failed; atomic_long_t read_empty; atomic_long_t read_high_ecc; atomic_long_t read_failed_gc; atomic_long_t write_failed; atomic_long_t erase_failed; |
588726d3e lightnvm: pblk: f... |
619 |
atomic_t inflight_io; /* General inflight I/O counter */ |
a4bd217b4 lightnvm: physica... |
620 621 622 623 624 625 626 627 628 629 |
struct task_struct *writer_ts; /* Simple translation map of logical addresses to physical addresses. * The logical addresses is known by the host system, while the physical * addresses are used when writing to the disk block device. */ unsigned char *trans_map; spinlock_t trans_lock; struct list_head compl_list; |
6a3abf5be lightnvm: pblk: r... |
630 631 |
spinlock_t resubmit_lock; /* Resubmit list lock */ struct list_head resubmit_list; /* Resubmit list for failed writes*/ |
b906bbb69 lightnvm: convert... |
632 633 634 635 636 637 |
mempool_t page_bio_pool; mempool_t gen_ws_pool; mempool_t rec_pool; mempool_t r_rq_pool; mempool_t w_rq_pool; mempool_t e_rq_pool; |
a4bd217b4 lightnvm: physica... |
638 |
|
ef5764946 lightnvm: pblk: s... |
639 640 |
struct workqueue_struct *close_wq; struct workqueue_struct *bb_wq; |
7bd4d370d lightnvm: pblk: g... |
641 |
struct workqueue_struct *r_end_wq; |
ef5764946 lightnvm: pblk: s... |
642 |
|
a4bd217b4 lightnvm: physica... |
643 644 645 646 647 648 649 650 651 652 653 |
struct timer_list wtimer; struct pblk_gc gc; }; struct pblk_line_ws { struct pblk *pblk; struct pblk_line *line; void *priv; struct work_struct ws; }; |
084ec9ba0 lightnvm: pblk: r... |
654 |
#define pblk_g_rq_size (sizeof(struct nvm_rq) + sizeof(struct pblk_g_ctx)) |
a4bd217b4 lightnvm: physica... |
655 |
#define pblk_w_rq_size (sizeof(struct nvm_rq) + sizeof(struct pblk_c_ctx)) |
4e495a46b lightnvm: pblk: e... |
656 657 658 659 660 661 662 663 |
#define pblk_err(pblk, fmt, ...) \ pr_err("pblk %s: " fmt, pblk->disk->disk_name, ##__VA_ARGS__) #define pblk_info(pblk, fmt, ...) \ pr_info("pblk %s: " fmt, pblk->disk->disk_name, ##__VA_ARGS__) #define pblk_warn(pblk, fmt, ...) \ pr_warn("pblk %s: " fmt, pblk->disk->disk_name, ##__VA_ARGS__) #define pblk_debug(pblk, fmt, ...) \ pr_debug("pblk %s: " fmt, pblk->disk->disk_name, ##__VA_ARGS__) |
a4bd217b4 lightnvm: physica... |
664 665 666 |
/* * pblk ring buffer operations */ |
766c8ceb1 lightnvm: pblk: g... |
667 668 |
int pblk_rb_init(struct pblk_rb *rb, unsigned int size, unsigned int threshold, unsigned int seg_sz); |
a4bd217b4 lightnvm: physica... |
669 670 671 672 673 674 675 |
int pblk_rb_may_write_user(struct pblk_rb *rb, struct bio *bio, unsigned int nr_entries, unsigned int *pos); int pblk_rb_may_write_gc(struct pblk_rb *rb, unsigned int nr_entries, unsigned int *pos); void pblk_rb_write_entry_user(struct pblk_rb *rb, void *data, struct pblk_w_ctx w_ctx, unsigned int pos); void pblk_rb_write_entry_gc(struct pblk_rb *rb, void *data, |
d340121eb lightnvm: pblk: s... |
676 677 |
struct pblk_w_ctx w_ctx, struct pblk_line *line, u64 paddr, unsigned int pos); |
a4bd217b4 lightnvm: physica... |
678 |
struct pblk_w_ctx *pblk_rb_w_ctx(struct pblk_rb *rb, unsigned int pos); |
588726d3e lightnvm: pblk: f... |
679 |
void pblk_rb_flush(struct pblk_rb *rb); |
a4bd217b4 lightnvm: physica... |
680 681 |
void pblk_rb_sync_l2p(struct pblk_rb *rb); |
d624f371d lightnvm: pblk: g... |
682 |
unsigned int pblk_rb_read_to_bio(struct pblk_rb *rb, struct nvm_rq *rqd, |
875d94f3a lightnvm: pblk: a... |
683 684 |
unsigned int pos, unsigned int nr_entries, unsigned int count); |
a4bd217b4 lightnvm: physica... |
685 |
int pblk_rb_copy_to_bio(struct pblk_rb *rb, struct bio *bio, sector_t lba, |
a96de64a2 lightnvm: pblk: s... |
686 |
struct ppa_addr ppa); |
a4bd217b4 lightnvm: physica... |
687 688 689 690 |
unsigned int pblk_rb_read_commit(struct pblk_rb *rb, unsigned int entries); unsigned int pblk_rb_sync_init(struct pblk_rb *rb, unsigned long *flags); unsigned int pblk_rb_sync_advance(struct pblk_rb *rb, unsigned int nr_entries); |
40b8657dc lightnvm: pblk: e... |
691 692 |
unsigned int pblk_rb_ptr_wrap(struct pblk_rb *rb, unsigned int p, unsigned int nr_entries); |
a4bd217b4 lightnvm: physica... |
693 |
void pblk_rb_sync_end(struct pblk_rb *rb, unsigned long *flags); |
8154d296d lightnvm: pblk: r... |
694 |
unsigned int pblk_rb_flush_point_count(struct pblk_rb *rb); |
a4bd217b4 lightnvm: physica... |
695 696 |
unsigned int pblk_rb_read_count(struct pblk_rb *rb); |
ee8d5c1ad lightnvm: pblk: r... |
697 |
unsigned int pblk_rb_sync_count(struct pblk_rb *rb); |
a4bd217b4 lightnvm: physica... |
698 699 700 701 |
unsigned int pblk_rb_wrap_pos(struct pblk_rb *rb, unsigned int pos); int pblk_rb_tear_down_check(struct pblk_rb *rb); int pblk_rb_pos_oob(struct pblk_rb *rb, u64 pos); |
9bd1f875c lightnvm: pblk: m... |
702 |
void pblk_rb_free(struct pblk_rb *rb); |
a4bd217b4 lightnvm: physica... |
703 704 705 706 707 |
ssize_t pblk_rb_sysfs(struct pblk_rb *rb, char *buf); /* * pblk core */ |
67bf26a32 lightnvm: pblk: r... |
708 709 |
struct nvm_rq *pblk_alloc_rqd(struct pblk *pblk, int type); void pblk_free_rqd(struct pblk *pblk, struct nvm_rq *rqd, int type); |
45dcf29b9 lightnvm: pblk: e... |
710 711 |
int pblk_alloc_rqd_meta(struct pblk *pblk, struct nvm_rq *rqd); void pblk_free_rqd_meta(struct pblk *pblk, struct nvm_rq *rqd); |
c2e9f5d45 lightnvm: pblk: e... |
712 |
void pblk_set_sec_per_write(struct pblk *pblk, int sec_per_write); |
a4bd217b4 lightnvm: physica... |
713 714 |
int pblk_setup_w_rec_rq(struct pblk *pblk, struct nvm_rq *rqd, struct pblk_c_ctx *c_ctx); |
a4bd217b4 lightnvm: physica... |
715 |
void pblk_discard(struct pblk *pblk, struct bio *bio); |
aff3fb18f lightnvm: move ba... |
716 |
struct nvm_chk_meta *pblk_get_chunk_meta(struct pblk *pblk); |
32ef9412c lightnvm: pblk: i... |
717 718 719 |
struct nvm_chk_meta *pblk_chunk_get_off(struct pblk *pblk, struct nvm_chk_meta *lp, struct ppa_addr ppa); |
a4bd217b4 lightnvm: physica... |
720 721 |
void pblk_log_write_err(struct pblk *pblk, struct nvm_rq *rqd); void pblk_log_read_err(struct pblk *pblk, struct nvm_rq *rqd); |
48e5da725 lightnvm: move me... |
722 723 |
int pblk_submit_io(struct pblk *pblk, struct nvm_rq *rqd, void *buf); int pblk_submit_io_sync(struct pblk *pblk, struct nvm_rq *rqd, void *buf); |
dd2a43437 lightnvm: pblk: s... |
724 |
int pblk_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line); |
4c44abf43 lightnvm: pblk: a... |
725 |
void pblk_check_chunk_state_update(struct pblk *pblk, struct nvm_rq *rqd); |
a4bd217b4 lightnvm: physica... |
726 727 |
struct pblk_line *pblk_line_get(struct pblk *pblk); struct pblk_line *pblk_line_get_first_data(struct pblk *pblk); |
21d228711 lightnvm: pblk: e... |
728 |
struct pblk_line *pblk_line_replace_data(struct pblk *pblk); |
ae14cc044 lightnvm: pblk: r... |
729 730 |
void pblk_ppa_to_line_put(struct pblk *pblk, struct ppa_addr ppa); void pblk_rq_to_line_put(struct pblk *pblk, struct nvm_rq *rqd); |
a4bd217b4 lightnvm: physica... |
731 732 733 |
int pblk_line_recov_alloc(struct pblk *pblk, struct pblk_line *line); void pblk_line_recov_close(struct pblk *pblk, struct pblk_line *line); struct pblk_line *pblk_line_get_data(struct pblk *pblk); |
d624f371d lightnvm: pblk: g... |
734 |
struct pblk_line *pblk_line_get_erase(struct pblk *pblk); |
a4bd217b4 lightnvm: physica... |
735 736 |
int pblk_line_erase(struct pblk *pblk, struct pblk_line *line); int pblk_line_is_full(struct pblk_line *line); |
8e55c07b2 lightnvm: pblk: r... |
737 |
void pblk_line_free(struct pblk_line *line); |
dd2a43437 lightnvm: pblk: s... |
738 |
void pblk_line_close_meta(struct pblk *pblk, struct pblk_line *line); |
a4bd217b4 lightnvm: physica... |
739 |
void pblk_line_close(struct pblk *pblk, struct pblk_line *line); |
dd2a43437 lightnvm: pblk: s... |
740 |
void pblk_line_close_ws(struct work_struct *work); |
588726d3e lightnvm: pblk: f... |
741 |
void pblk_pipeline_stop(struct pblk *pblk); |
a7c9e9109 lightnvm: pass fl... |
742 743 |
void __pblk_pipeline_stop(struct pblk *pblk); void __pblk_pipeline_flush(struct pblk *pblk); |
b84ae4a8b lightnvm: pblk: s... |
744 745 746 |
void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv, void (*work)(struct work_struct *), gfp_t gfp_mask, struct workqueue_struct *wq); |
a4bd217b4 lightnvm: physica... |
747 |
u64 pblk_line_smeta_start(struct pblk *pblk, struct pblk_line *line); |
af3fac166 lightnvm: pblk: r... |
748 749 |
int pblk_line_smeta_read(struct pblk *pblk, struct pblk_line *line); int pblk_line_emeta_read(struct pblk *pblk, struct pblk_line *line, |
dd2a43437 lightnvm: pblk: s... |
750 |
void *emeta_buf); |
a4bd217b4 lightnvm: physica... |
751 752 |
int pblk_blk_erase_async(struct pblk *pblk, struct ppa_addr erase_ppa); void pblk_line_put(struct kref *ref); |
7bd4d370d lightnvm: pblk: g... |
753 |
void pblk_line_put_wq(struct kref *ref); |
a4bd217b4 lightnvm: physica... |
754 |
struct list_head *pblk_line_gc_list(struct pblk *pblk, struct pblk_line *line); |
dd2a43437 lightnvm: pblk: s... |
755 756 |
u64 pblk_lookup_page(struct pblk *pblk, struct pblk_line *line); void pblk_dealloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs); |
a4bd217b4 lightnvm: physica... |
757 |
u64 pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs); |
dd2a43437 lightnvm: pblk: s... |
758 |
u64 __pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs); |
a4bd217b4 lightnvm: physica... |
759 |
int pblk_calc_secs(struct pblk *pblk, unsigned long secs_avail, |
55d8ec353 lightnvm: pblk: s... |
760 |
unsigned long secs_to_flush, bool skip_meta); |
43241cfe4 lightnvm: pblk: r... |
761 |
void pblk_down_rq(struct pblk *pblk, struct ppa_addr ppa, |
a4bd217b4 lightnvm: physica... |
762 |
unsigned long *lun_bitmap); |
43241cfe4 lightnvm: pblk: r... |
763 764 |
void pblk_down_chunk(struct pblk *pblk, struct ppa_addr ppa); void pblk_up_chunk(struct pblk *pblk, struct ppa_addr ppa); |
e99e802fc lightnvm: pblk: r... |
765 |
void pblk_up_rq(struct pblk *pblk, unsigned long *lun_bitmap); |
a4bd217b4 lightnvm: physica... |
766 767 |
int pblk_bio_add_pages(struct pblk *pblk, struct bio *bio, gfp_t flags, int nr_pages); |
a4bd217b4 lightnvm: physica... |
768 769 770 |
void pblk_bio_free_pages(struct pblk *pblk, struct bio *bio, int off, int nr_pages); void pblk_map_invalidate(struct pblk *pblk, struct ppa_addr ppa); |
0880a9aa2 lightnvm: pblk: d... |
771 772 |
void __pblk_map_invalidate(struct pblk *pblk, struct pblk_line *line, u64 paddr); |
a4bd217b4 lightnvm: physica... |
773 774 775 776 777 778 |
void pblk_update_map(struct pblk *pblk, sector_t lba, struct ppa_addr ppa); void pblk_update_map_cache(struct pblk *pblk, sector_t lba, struct ppa_addr ppa); void pblk_update_map_dev(struct pblk *pblk, sector_t lba, struct ppa_addr ppa, struct ppa_addr entry_line); int pblk_update_map_gc(struct pblk *pblk, sector_t lba, struct ppa_addr ppa, |
d340121eb lightnvm: pblk: s... |
779 |
struct pblk_line *gc_line, u64 paddr); |
a4bd217b4 lightnvm: physica... |
780 781 |
void pblk_lookup_l2p_rand(struct pblk *pblk, struct ppa_addr *ppas, u64 *lba_list, int nr_secs); |
a96de64a2 lightnvm: pblk: s... |
782 783 |
int pblk_lookup_l2p_seq(struct pblk *pblk, struct ppa_addr *ppas, sector_t blba, int nr_secs, bool *from_cache); |
55d8ec353 lightnvm: pblk: s... |
784 785 |
void *pblk_get_meta_for_writes(struct pblk *pblk, struct nvm_rq *rqd); void pblk_get_packed_meta(struct pblk *pblk, struct nvm_rq *rqd); |
a4bd217b4 lightnvm: physica... |
786 787 788 789 |
/* * pblk user I/O write path */ |
3e03f6322 lightnvm: pblk: I... |
790 |
void pblk_write_to_cache(struct pblk *pblk, struct bio *bio, |
a4bd217b4 lightnvm: physica... |
791 |
unsigned long flags); |
d340121eb lightnvm: pblk: s... |
792 |
int pblk_write_gc_to_cache(struct pblk *pblk, struct pblk_gc_rq *gc_rq); |
a4bd217b4 lightnvm: physica... |
793 794 795 796 |
/* * pblk map */ |
525f7bb2c lightnvm: pblk: s... |
797 |
int pblk_map_erase_rq(struct pblk *pblk, struct nvm_rq *rqd, |
a4bd217b4 lightnvm: physica... |
798 799 |
unsigned int sentry, unsigned long *lun_bitmap, unsigned int valid_secs, struct ppa_addr *erase_ppa); |
525f7bb2c lightnvm: pblk: s... |
800 |
int pblk_map_rq(struct pblk *pblk, struct nvm_rq *rqd, unsigned int sentry, |
a4bd217b4 lightnvm: physica... |
801 802 803 804 805 806 807 |
unsigned long *lun_bitmap, unsigned int valid_secs, unsigned int off); /* * pblk write thread */ int pblk_write_ts(void *data); |
87c1d2d37 lightnvm: Convert... |
808 |
void pblk_write_timer_fn(struct timer_list *t); |
a4bd217b4 lightnvm: physica... |
809 |
void pblk_write_should_kick(struct pblk *pblk); |
cc9c9a00b lightnvm: pblk: k... |
810 |
void pblk_write_kick(struct pblk *pblk); |
a4bd217b4 lightnvm: physica... |
811 812 813 814 |
/* * pblk read path */ |
b906bbb69 lightnvm: convert... |
815 |
extern struct bio_set pblk_bio_set; |
3e03f6322 lightnvm: pblk: I... |
816 |
void pblk_submit_read(struct pblk *pblk, struct bio *bio); |
d340121eb lightnvm: pblk: s... |
817 |
int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq); |
a4bd217b4 lightnvm: physica... |
818 819 820 |
/* * pblk recovery */ |
a4bd217b4 lightnvm: physica... |
821 |
struct pblk_line *pblk_recov_l2p(struct pblk *pblk); |
588726d3e lightnvm: pblk: f... |
822 |
int pblk_recov_pad(struct pblk *pblk); |
06bc072b3 lightnvm: pblk: r... |
823 |
int pblk_recov_check_emeta(struct pblk *pblk, struct line_emeta *emeta); |
a4bd217b4 lightnvm: physica... |
824 825 826 827 |
/* * pblk gc */ |
b20ba1bc7 lightnvm: pblk: r... |
828 |
#define PBLK_GC_MAX_READERS 8 /* Max number of outstanding GC reader jobs */ |
3627896a4 lightnvm: pblk: u... |
829 |
#define PBLK_GC_RQ_QD 128 /* Queue depth for inflight GC requests */ |
b20ba1bc7 lightnvm: pblk: r... |
830 |
#define PBLK_GC_L_QD 4 /* Queue depth for inflight GC lines */ |
a4bd217b4 lightnvm: physica... |
831 832 |
int pblk_gc_init(struct pblk *pblk); |
a7c9e9109 lightnvm: pass fl... |
833 |
void pblk_gc_exit(struct pblk *pblk, bool graceful); |
a4bd217b4 lightnvm: physica... |
834 835 |
void pblk_gc_should_start(struct pblk *pblk); void pblk_gc_should_stop(struct pblk *pblk); |
03661b5f7 lightnvm: pblk: s... |
836 |
void pblk_gc_should_kick(struct pblk *pblk); |
37ce33d57 lightnvm: pblk: f... |
837 |
void pblk_gc_free_full_lines(struct pblk *pblk); |
a4bd217b4 lightnvm: physica... |
838 839 |
void pblk_gc_sysfs_state_show(struct pblk *pblk, int *gc_enabled, int *gc_active); |
b20ba1bc7 lightnvm: pblk: r... |
840 |
int pblk_gc_sysfs_force(struct pblk *pblk, int force); |
f2e024570 lightnvm: pblk: G... |
841 |
void pblk_put_line_back(struct pblk *pblk, struct pblk_line *line); |
a4bd217b4 lightnvm: physica... |
842 843 844 845 |
/* * pblk rate limiter */ |
b4cdc4260 lightnvm: pblk: p... |
846 |
void pblk_rl_init(struct pblk_rl *rl, int budget, int threshold); |
a4bd217b4 lightnvm: physica... |
847 |
void pblk_rl_free(struct pblk_rl *rl); |
03661b5f7 lightnvm: pblk: s... |
848 |
void pblk_rl_update_rates(struct pblk_rl *rl); |
b20ba1bc7 lightnvm: pblk: r... |
849 |
int pblk_rl_high_thrs(struct pblk_rl *rl); |
a4bd217b4 lightnvm: physica... |
850 |
unsigned long pblk_rl_nr_free_blks(struct pblk_rl *rl); |
a7689938e lightnvm: pblk: u... |
851 |
unsigned long pblk_rl_nr_user_free_blks(struct pblk_rl *rl); |
a4bd217b4 lightnvm: physica... |
852 |
int pblk_rl_user_may_insert(struct pblk_rl *rl, int nr_entries); |
588726d3e lightnvm: pblk: f... |
853 |
void pblk_rl_inserted(struct pblk_rl *rl, int nr_entries); |
a4bd217b4 lightnvm: physica... |
854 855 856 857 |
void pblk_rl_user_in(struct pblk_rl *rl, int nr_entries); int pblk_rl_gc_may_insert(struct pblk_rl *rl, int nr_entries); void pblk_rl_gc_in(struct pblk_rl *rl, int nr_entries); void pblk_rl_out(struct pblk_rl *rl, int nr_user, int nr_gc); |
da67e68fb lightnvm: pblk: a... |
858 |
int pblk_rl_max_io(struct pblk_rl *rl); |
a4bd217b4 lightnvm: physica... |
859 |
void pblk_rl_free_lines_inc(struct pblk_rl *rl, struct pblk_line *line); |
a7689938e lightnvm: pblk: u... |
860 861 |
void pblk_rl_free_lines_dec(struct pblk_rl *rl, struct pblk_line *line, bool used); |
588726d3e lightnvm: pblk: f... |
862 |
int pblk_rl_is_limit(struct pblk_rl *rl); |
a4bd217b4 lightnvm: physica... |
863 |
|
48b8d2089 lightnvm: pblk: g... |
864 865 |
void pblk_rl_werr_line_in(struct pblk_rl *rl); void pblk_rl_werr_line_out(struct pblk_rl *rl); |
a4bd217b4 lightnvm: physica... |
866 867 868 869 870 |
/* * pblk sysfs */ int pblk_sysfs_init(struct gendisk *tdisk); void pblk_sysfs_exit(struct gendisk *tdisk); |
a4bd217b4 lightnvm: physica... |
871 872 873 874 |
static inline struct nvm_rq *nvm_rq_from_c_ctx(void *c_ctx) { return c_ctx - sizeof(struct nvm_rq); } |
dd2a43437 lightnvm: pblk: s... |
875 876 877 878 |
static inline void *emeta_to_bb(struct line_emeta *emeta) { return emeta->bb_bitmap; } |
76758390f lightnvm: pblk: e... |
879 880 881 882 883 |
static inline void *emeta_to_wa(struct pblk_line_meta *lm, struct line_emeta *emeta) { return emeta->bb_bitmap + lm->blk_bitmap_len; } |
dd2a43437 lightnvm: pblk: s... |
884 885 886 887 888 889 |
static inline void *emeta_to_lbas(struct pblk *pblk, struct line_emeta *emeta) { return ((void *)emeta + pblk->lm.emeta_len[1]); } static inline void *emeta_to_vsc(struct pblk *pblk, struct line_emeta *emeta) |
a4bd217b4 lightnvm: physica... |
890 |
{ |
dd2a43437 lightnvm: pblk: s... |
891 |
return (emeta_to_lbas(pblk, emeta) + pblk->lm.emeta_len[2]); |
a4bd217b4 lightnvm: physica... |
892 |
} |
b20ba1bc7 lightnvm: pblk: r... |
893 894 |
static inline int pblk_line_vsc(struct pblk_line *line) { |
d340121eb lightnvm: pblk: s... |
895 |
return le32_to_cpu(*line->vsc); |
b20ba1bc7 lightnvm: pblk: r... |
896 |
} |
cb21665c8 lightnvm: pblk: i... |
897 |
static inline int pblk_ppa_to_line_id(struct ppa_addr p) |
a4bd217b4 lightnvm: physica... |
898 |
{ |
694715137 lightnvm: add sup... |
899 |
return p.a.blk; |
a4bd217b4 lightnvm: physica... |
900 |
} |
cb21665c8 lightnvm: pblk: i... |
901 902 903 904 905 |
static inline struct pblk_line *pblk_ppa_to_line(struct pblk *pblk, struct ppa_addr p) { return &pblk->lines[pblk_ppa_to_line_id(p)]; } |
b1bcfda10 lightnvm: pblk: c... |
906 |
static inline int pblk_ppa_to_pos(struct nvm_geo *geo, struct ppa_addr p) |
a4bd217b4 lightnvm: physica... |
907 |
{ |
694715137 lightnvm: add sup... |
908 |
return p.a.lun * geo->num_ch + p.a.ch; |
a4bd217b4 lightnvm: physica... |
909 |
} |
b1bcfda10 lightnvm: pblk: c... |
910 911 |
static inline struct ppa_addr addr_to_gen_ppa(struct pblk *pblk, u64 paddr, u64 line_id) |
a4bd217b4 lightnvm: physica... |
912 |
{ |
3b2a3ad11 lightnvm: pblk: i... |
913 914 |
struct nvm_tgt_dev *dev = pblk->dev; struct nvm_geo *geo = &dev->geo; |
b1bcfda10 lightnvm: pblk: c... |
915 |
struct ppa_addr ppa; |
3b2a3ad11 lightnvm: pblk: i... |
916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 |
if (geo->version == NVM_OCSSD_SPEC_12) { struct nvm_addrf_12 *ppaf = (struct nvm_addrf_12 *)&pblk->addrf; ppa.ppa = 0; ppa.g.blk = line_id; ppa.g.pg = (paddr & ppaf->pg_mask) >> ppaf->pg_offset; ppa.g.lun = (paddr & ppaf->lun_mask) >> ppaf->lun_offset; ppa.g.ch = (paddr & ppaf->ch_mask) >> ppaf->ch_offset; ppa.g.pl = (paddr & ppaf->pln_mask) >> ppaf->pln_offset; ppa.g.sec = (paddr & ppaf->sec_mask) >> ppaf->sec_offset; } else { struct pblk_addrf *uaddrf = &pblk->uaddrf; int secs, chnls, luns; ppa.ppa = 0; ppa.m.chk = line_id; paddr = div_u64_rem(paddr, uaddrf->sec_stripe, &secs); ppa.m.sec = secs; paddr = div_u64_rem(paddr, uaddrf->ch_stripe, &chnls); ppa.m.grp = chnls; paddr = div_u64_rem(paddr, uaddrf->lun_stripe, &luns); ppa.m.pu = luns; ppa.m.sec += uaddrf->sec_stripe * paddr; } |
b1bcfda10 lightnvm: pblk: c... |
945 946 |
return ppa; |
a4bd217b4 lightnvm: physica... |
947 |
} |
2cf99bbd1 lightnvm: pblk: a... |
948 949 950 951 952 |
static inline struct nvm_chk_meta *pblk_dev_ppa_to_chunk(struct pblk *pblk, struct ppa_addr p) { struct nvm_tgt_dev *dev = pblk->dev; struct nvm_geo *geo = &dev->geo; |
cb21665c8 lightnvm: pblk: i... |
953 |
struct pblk_line *line = pblk_ppa_to_line(pblk, p); |
2cf99bbd1 lightnvm: pblk: a... |
954 955 956 957 958 959 960 961 962 963 964 965 |
int pos = pblk_ppa_to_pos(geo, p); return &line->chks[pos]; } static inline u64 pblk_dev_ppa_to_chunk_addr(struct pblk *pblk, struct ppa_addr p) { struct nvm_tgt_dev *dev = pblk->dev; return dev_to_chunk_addr(dev->parent, &pblk->addrf, p); } |
b1bcfda10 lightnvm: pblk: c... |
966 967 |
static inline u64 pblk_dev_ppa_to_line_addr(struct pblk *pblk, struct ppa_addr p) |
a4bd217b4 lightnvm: physica... |
968 |
{ |
3b2a3ad11 lightnvm: pblk: i... |
969 970 |
struct nvm_tgt_dev *dev = pblk->dev; struct nvm_geo *geo = &dev->geo; |
b1bcfda10 lightnvm: pblk: c... |
971 |
u64 paddr; |
3b2a3ad11 lightnvm: pblk: i... |
972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 |
if (geo->version == NVM_OCSSD_SPEC_12) { struct nvm_addrf_12 *ppaf = (struct nvm_addrf_12 *)&pblk->addrf; paddr = (u64)p.g.ch << ppaf->ch_offset; paddr |= (u64)p.g.lun << ppaf->lun_offset; paddr |= (u64)p.g.pg << ppaf->pg_offset; paddr |= (u64)p.g.pl << ppaf->pln_offset; paddr |= (u64)p.g.sec << ppaf->sec_offset; } else { struct pblk_addrf *uaddrf = &pblk->uaddrf; u64 secs = p.m.sec; int sec_stripe; paddr = (u64)p.m.grp * uaddrf->sec_stripe; paddr += (u64)p.m.pu * uaddrf->sec_lun_stripe; secs = div_u64_rem(secs, uaddrf->sec_stripe, &sec_stripe); paddr += secs * uaddrf->sec_ws_stripe; paddr += sec_stripe; } |
b1bcfda10 lightnvm: pblk: c... |
992 993 |
return paddr; |
a4bd217b4 lightnvm: physica... |
994 995 996 997 |
} static inline struct ppa_addr pblk_ppa32_to_ppa64(struct pblk *pblk, u32 ppa32) { |
7f985f9a6 lightnvm: move pp... |
998 |
struct nvm_tgt_dev *dev = pblk->dev; |
a4bd217b4 lightnvm: physica... |
999 |
|
7f985f9a6 lightnvm: move pp... |
1000 |
return nvm_ppa32_to_ppa64(dev->parent, &pblk->addrf, ppa32); |
a4bd217b4 lightnvm: physica... |
1001 |
} |
a4bd217b4 lightnvm: physica... |
1002 1003 |
static inline u32 pblk_ppa64_to_ppa32(struct pblk *pblk, struct ppa_addr ppa64) { |
7f985f9a6 lightnvm: move pp... |
1004 |
struct nvm_tgt_dev *dev = pblk->dev; |
a4bd217b4 lightnvm: physica... |
1005 |
|
7f985f9a6 lightnvm: move pp... |
1006 |
return nvm_ppa64_to_ppa32(dev->parent, &pblk->addrf, ppa64); |
a4bd217b4 lightnvm: physica... |
1007 |
} |
b1bcfda10 lightnvm: pblk: c... |
1008 1009 |
static inline struct ppa_addr pblk_trans_map_get(struct pblk *pblk, sector_t lba) |
a4bd217b4 lightnvm: physica... |
1010 |
{ |
b1bcfda10 lightnvm: pblk: c... |
1011 |
struct ppa_addr ppa; |
bb845ae45 lightnvm: pblk: r... |
1012 |
if (pblk->addrf_len < 32) { |
a4bd217b4 lightnvm: physica... |
1013 |
u32 *map = (u32 *)pblk->trans_map; |
b1bcfda10 lightnvm: pblk: c... |
1014 |
ppa = pblk_ppa32_to_ppa64(pblk, map[lba]); |
a4bd217b4 lightnvm: physica... |
1015 |
} else { |
b1bcfda10 lightnvm: pblk: c... |
1016 |
struct ppa_addr *map = (struct ppa_addr *)pblk->trans_map; |
a4bd217b4 lightnvm: physica... |
1017 |
|
b1bcfda10 lightnvm: pblk: c... |
1018 |
ppa = map[lba]; |
a4bd217b4 lightnvm: physica... |
1019 |
} |
b1bcfda10 lightnvm: pblk: c... |
1020 1021 |
return ppa; |
a4bd217b4 lightnvm: physica... |
1022 |
} |
b1bcfda10 lightnvm: pblk: c... |
1023 1024 |
static inline void pblk_trans_map_set(struct pblk *pblk, sector_t lba, struct ppa_addr ppa) |
a4bd217b4 lightnvm: physica... |
1025 |
{ |
bb845ae45 lightnvm: pblk: r... |
1026 |
if (pblk->addrf_len < 32) { |
b1bcfda10 lightnvm: pblk: c... |
1027 |
u32 *map = (u32 *)pblk->trans_map; |
a4bd217b4 lightnvm: physica... |
1028 |
|
b1bcfda10 lightnvm: pblk: c... |
1029 1030 1031 |
map[lba] = pblk_ppa64_to_ppa32(pblk, ppa); } else { u64 *map = (u64 *)pblk->trans_map; |
a4bd217b4 lightnvm: physica... |
1032 |
|
b1bcfda10 lightnvm: pblk: c... |
1033 1034 |
map[lba] = ppa.ppa; } |
a4bd217b4 lightnvm: physica... |
1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 |
} static inline int pblk_ppa_empty(struct ppa_addr ppa_addr) { return (ppa_addr.ppa == ADDR_EMPTY); } static inline void pblk_ppa_set_empty(struct ppa_addr *ppa_addr) { ppa_addr->ppa = ADDR_EMPTY; } |
076984669 lightnvm: pblk: v... |
1046 1047 |
static inline bool pblk_ppa_comp(struct ppa_addr lppa, struct ppa_addr rppa) { |
8b7bc8498 lightnvm: pblk: r... |
1048 |
return (lppa.ppa == rppa.ppa); |
076984669 lightnvm: pblk: v... |
1049 |
} |
a4bd217b4 lightnvm: physica... |
1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 |
static inline int pblk_addr_in_cache(struct ppa_addr ppa) { return (ppa.ppa != ADDR_EMPTY && ppa.c.is_cached); } static inline int pblk_addr_to_cacheline(struct ppa_addr ppa) { return ppa.c.line; } static inline struct ppa_addr pblk_cacheline_to_addr(int addr) { struct ppa_addr p; p.c.line = addr; p.c.is_cached = 1; return p; } |
a4bd217b4 lightnvm: physica... |
1069 |
static inline u32 pblk_calc_meta_header_crc(struct pblk *pblk, |
dd2a43437 lightnvm: pblk: s... |
1070 |
struct line_header *header) |
a4bd217b4 lightnvm: physica... |
1071 1072 |
{ u32 crc = ~(u32)0; |
dd2a43437 lightnvm: pblk: s... |
1073 |
crc = crc32_le(crc, (unsigned char *)header + sizeof(crc), |
a4bd217b4 lightnvm: physica... |
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 |
sizeof(struct line_header) - sizeof(crc)); return crc; } static inline u32 pblk_calc_smeta_crc(struct pblk *pblk, struct line_smeta *smeta) { struct pblk_line_meta *lm = &pblk->lm; u32 crc = ~(u32)0; crc = crc32_le(crc, (unsigned char *)smeta + sizeof(struct line_header) + sizeof(crc), lm->smeta_len - sizeof(struct line_header) - sizeof(crc)); return crc; } static inline u32 pblk_calc_emeta_crc(struct pblk *pblk, struct line_emeta *emeta) { struct pblk_line_meta *lm = &pblk->lm; u32 crc = ~(u32)0; crc = crc32_le(crc, (unsigned char *)emeta + sizeof(struct line_header) + sizeof(crc), |
dd2a43437 lightnvm: pblk: s... |
1101 |
lm->emeta_len[0] - |
a4bd217b4 lightnvm: physica... |
1102 1103 1104 1105 |
sizeof(struct line_header) - sizeof(crc)); return crc; } |
f9c101523 lightnvm: pblk: i... |
1106 |
static inline int pblk_io_aligned(struct pblk *pblk, int nr_secs) |
a4bd217b4 lightnvm: physica... |
1107 |
{ |
f9c101523 lightnvm: pblk: i... |
1108 |
return !(nr_secs % pblk->min_write_pgs); |
a4bd217b4 lightnvm: physica... |
1109 |
} |
880eda544 lightnvm: move NV... |
1110 |
#ifdef CONFIG_NVM_PBLK_DEBUG |
4e495a46b lightnvm: pblk: e... |
1111 |
static inline void print_ppa(struct pblk *pblk, struct ppa_addr *p, |
3b2a3ad11 lightnvm: pblk: i... |
1112 |
char *msg, int error) |
a4bd217b4 lightnvm: physica... |
1113 |
{ |
4e495a46b lightnvm: pblk: e... |
1114 |
struct nvm_geo *geo = &pblk->dev->geo; |
a4bd217b4 lightnvm: physica... |
1115 |
if (p->c.is_cached) { |
4e495a46b lightnvm: pblk: e... |
1116 1117 |
pblk_err(pblk, "ppa: (%s: %x) cache line: %llu ", |
a4bd217b4 lightnvm: physica... |
1118 |
msg, error, (u64)p->c.line); |
3b2a3ad11 lightnvm: pblk: i... |
1119 |
} else if (geo->version == NVM_OCSSD_SPEC_12) { |
4e495a46b lightnvm: pblk: e... |
1120 1121 |
pblk_err(pblk, "ppa: (%s: %x):ch:%d,lun:%d,blk:%d,pg:%d,pl:%d,sec:%d ", |
a4bd217b4 lightnvm: physica... |
1122 1123 1124 |
msg, error, p->g.ch, p->g.lun, p->g.blk, p->g.pg, p->g.pl, p->g.sec); |
3b2a3ad11 lightnvm: pblk: i... |
1125 |
} else { |
4e495a46b lightnvm: pblk: e... |
1126 1127 |
pblk_err(pblk, "ppa: (%s: %x):ch:%d,lun:%d,chk:%d,sec:%d ", |
3b2a3ad11 lightnvm: pblk: i... |
1128 1129 |
msg, error, p->m.grp, p->m.pu, p->m.chk, p->m.sec); |
a4bd217b4 lightnvm: physica... |
1130 1131 1132 1133 1134 1135 1136 1137 1138 |
} } static inline void pblk_print_failed_rqd(struct pblk *pblk, struct nvm_rq *rqd, int error) { int bit = -1; if (rqd->nr_ppas == 1) { |
4e495a46b lightnvm: pblk: e... |
1139 |
print_ppa(pblk, &rqd->ppa_addr, "rqd", error); |
a4bd217b4 lightnvm: physica... |
1140 1141 1142 1143 1144 |
return; } while ((bit = find_next_bit((void *)&rqd->ppa_status, rqd->nr_ppas, bit + 1)) < rqd->nr_ppas) { |
4e495a46b lightnvm: pblk: e... |
1145 |
print_ppa(pblk, &rqd->ppa_list[bit], "rqd", error); |
a4bd217b4 lightnvm: physica... |
1146 |
} |
4e495a46b lightnvm: pblk: e... |
1147 1148 |
pblk_err(pblk, "error:%d, ppa_status:%llx ", error, rqd->ppa_status); |
a4bd217b4 lightnvm: physica... |
1149 |
} |
a4bd217b4 lightnvm: physica... |
1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 |
static inline int pblk_boundary_ppa_checks(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *ppas, int nr_ppas) { struct nvm_geo *geo = &tgt_dev->geo; struct ppa_addr *ppa; int i; for (i = 0; i < nr_ppas; i++) { ppa = &ppas[i]; |
3b2a3ad11 lightnvm: pblk: i... |
1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 |
if (geo->version == NVM_OCSSD_SPEC_12) { if (!ppa->c.is_cached && ppa->g.ch < geo->num_ch && ppa->g.lun < geo->num_lun && ppa->g.pl < geo->num_pln && ppa->g.blk < geo->num_chk && ppa->g.pg < geo->num_pg && ppa->g.sec < geo->ws_min) continue; } else { if (!ppa->c.is_cached && ppa->m.grp < geo->num_ch && ppa->m.pu < geo->num_lun && ppa->m.chk < geo->num_chk && ppa->m.sec < geo->clba) continue; } |
4e495a46b lightnvm: pblk: e... |
1177 |
print_ppa(tgt_dev->q->queuedata, ppa, "boundary", i); |
1a94b2d48 lightnvm: impleme... |
1178 |
|
a4bd217b4 lightnvm: physica... |
1179 1180 1181 1182 |
return 1; } return 0; } |
1a94b2d48 lightnvm: impleme... |
1183 1184 1185 |
static inline int pblk_check_io(struct pblk *pblk, struct nvm_rq *rqd) { struct nvm_tgt_dev *dev = pblk->dev; |
d68a93440 lightnvm: introdu... |
1186 |
struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); |
1a94b2d48 lightnvm: impleme... |
1187 1188 1189 1190 1191 1192 1193 1194 |
if (pblk_boundary_ppa_checks(dev, ppa_list, rqd->nr_ppas)) { WARN_ON(1); return -EINVAL; } if (rqd->opcode == NVM_OP_PWRITE) { struct pblk_line *line; |
1a94b2d48 lightnvm: impleme... |
1195 1196 1197 |
int i; for (i = 0; i < rqd->nr_ppas; i++) { |
cb21665c8 lightnvm: pblk: i... |
1198 |
line = pblk_ppa_to_line(pblk, ppa_list[i]); |
1a94b2d48 lightnvm: impleme... |
1199 1200 1201 |
spin_lock(&line->lock); if (line->state != PBLK_LINESTATE_OPEN) { |
4e495a46b lightnvm: pblk: e... |
1202 1203 |
pblk_err(pblk, "bad ppa: line:%d,state:%d ", |
1a94b2d48 lightnvm: impleme... |
1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 |
line->id, line->state); WARN_ON(1); spin_unlock(&line->lock); return -EINVAL; } spin_unlock(&line->lock); } } return 0; } #endif |
a4bd217b4 lightnvm: physica... |
1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 |
static inline int pblk_boundary_paddr_checks(struct pblk *pblk, u64 paddr) { struct pblk_line_meta *lm = &pblk->lm; if (paddr > lm->sec_per_line) return 1; return 0; } static inline unsigned int pblk_get_bi_idx(struct bio *bio) { return bio->bi_iter.bi_idx; } static inline sector_t pblk_get_lba(struct bio *bio) { return bio->bi_iter.bi_sector / NR_PHY_IN_LOG; } static inline unsigned int pblk_get_secs(struct bio *bio) { return bio->bi_iter.bi_size / PBLK_EXPOSED_PAGE_SIZE; } |
4c44abf43 lightnvm: pblk: a... |
1240 1241 1242 1243 1244 1245 |
static inline char *pblk_disk_name(struct pblk *pblk) { struct gendisk *disk = pblk->disk; return disk->disk_name; } |
3bcebc5ba lightnvm: pblk: s... |
1246 1247 1248 1249 1250 1251 1252 1253 1254 |
static inline unsigned int pblk_get_min_chks(struct pblk *pblk) { struct pblk_line_meta *lm = &pblk->lm; /* In a worst-case scenario every line will have OP invalid sectors. * We will then need a minimum of 1/OP lines to free up a single line */ return DIV_ROUND_UP(100, pblk->op) * lm->blk_per_line; |
faa79f27f lightnvm: pblk: a... |
1255 |
} |
3bcebc5ba lightnvm: pblk: s... |
1256 |
|
faa79f27f lightnvm: pblk: a... |
1257 1258 1259 |
static inline struct pblk_sec_meta *pblk_get_meta(struct pblk *pblk, void *meta, int index) { |
2c4d5356e lightnvm: pblk: d... |
1260 1261 1262 |
return meta + max_t(int, sizeof(struct pblk_sec_meta), pblk->oob_meta_size) * index; |
3bcebc5ba lightnvm: pblk: s... |
1263 |
} |
24828d053 lightnvm: dynamic... |
1264 1265 1266 |
static inline int pblk_dma_meta_size(struct pblk *pblk) { |
2c4d5356e lightnvm: pblk: d... |
1267 1268 |
return max_t(int, sizeof(struct pblk_sec_meta), pblk->oob_meta_size) * NVM_MAX_VLBA; |
24828d053 lightnvm: dynamic... |
1269 |
} |
55d8ec353 lightnvm: pblk: s... |
1270 1271 1272 1273 1274 |
static inline int pblk_is_oob_meta_supported(struct pblk *pblk) { return pblk->oob_meta_size >= sizeof(struct pblk_sec_meta); } |
a4bd217b4 lightnvm: physica... |
1275 |
#endif /* PBLK_H_ */ |