Blame view

lib/scatterlist.c 19.5 KB
0db9299f4   Jens Axboe   SG: Move function...
1
2
3
4
5
6
7
8
  /*
   * Copyright (C) 2007 Jens Axboe <jens.axboe@oracle.com>
   *
   * Scatterlist handling helpers.
   *
   * This source code is licensed under the GNU General Public License,
   * Version 2. See the file COPYING for more details.
   */
8bc3bcc93   Paul Gortmaker   lib: reduce the u...
9
  #include <linux/export.h>
5a0e3ad6a   Tejun Heo   include cleanup: ...
10
  #include <linux/slab.h>
0db9299f4   Jens Axboe   SG: Move function...
11
  #include <linux/scatterlist.h>
b1adaf65b   FUJITA Tomonori   [SCSI] block: add...
12
  #include <linux/highmem.h>
b94de9bb7   Chris Wilson   lib/scatterlist: ...
13
  #include <linux/kmemleak.h>
0db9299f4   Jens Axboe   SG: Move function...
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
  
  /**
   * sg_next - return the next scatterlist entry in a list
   * @sg:		The current sg entry
   *
   * Description:
   *   Usually the next entry will be @sg@ + 1, but if this sg element is part
   *   of a chained scatterlist, it could jump to the start of a new
   *   scatterlist array.
   *
   **/
  struct scatterlist *sg_next(struct scatterlist *sg)
  {
  #ifdef CONFIG_DEBUG_SG
  	BUG_ON(sg->sg_magic != SG_MAGIC);
  #endif
  	if (sg_is_last(sg))
  		return NULL;
  
  	sg++;
  	if (unlikely(sg_is_chain(sg)))
  		sg = sg_chain_ptr(sg);
  
  	return sg;
  }
  EXPORT_SYMBOL(sg_next);
  
  /**
2e4846102   Maxim Levitsky   scatterlist: add ...
42
43
44
45
46
47
48
49
50
51
   * sg_nents - return total count of entries in scatterlist
   * @sg:		The scatterlist
   *
   * Description:
   * Allows to know how many entries are in sg, taking into acount
   * chaining as well
   *
   **/
  int sg_nents(struct scatterlist *sg)
  {
232f1b510   Maxim Levitsky   scatterlist: refa...
52
53
  	int nents;
  	for (nents = 0; sg; sg = sg_next(sg))
2e4846102   Maxim Levitsky   scatterlist: add ...
54
  		nents++;
2e4846102   Maxim Levitsky   scatterlist: add ...
55
56
57
  	return nents;
  }
  EXPORT_SYMBOL(sg_nents);
cfaed10d1   Tom Lendacky   scatterlist: intr...
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
  /**
   * sg_nents_for_len - return total count of entries in scatterlist
   *                    needed to satisfy the supplied length
   * @sg:		The scatterlist
   * @len:	The total required length
   *
   * Description:
   * Determines the number of entries in sg that are required to meet
   * the supplied length, taking into acount chaining as well
   *
   * Returns:
   *   the number of sg entries needed, negative error on failure
   *
   **/
  int sg_nents_for_len(struct scatterlist *sg, u64 len)
  {
  	int nents;
  	u64 total;
  
  	if (!len)
  		return 0;
  
  	for (nents = 0, total = 0; sg; sg = sg_next(sg)) {
  		nents++;
  		total += sg->length;
  		if (total >= len)
  			return nents;
  	}
  
  	return -EINVAL;
  }
  EXPORT_SYMBOL(sg_nents_for_len);
2e4846102   Maxim Levitsky   scatterlist: add ...
90
91
  
  /**
0db9299f4   Jens Axboe   SG: Move function...
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
   * sg_last - return the last scatterlist entry in a list
   * @sgl:	First entry in the scatterlist
   * @nents:	Number of entries in the scatterlist
   *
   * Description:
   *   Should only be used casually, it (currently) scans the entire list
   *   to get the last entry.
   *
   *   Note that the @sgl@ pointer passed in need not be the first one,
   *   the important bit is that @nents@ denotes the number of entries that
   *   exist from @sgl@.
   *
   **/
  struct scatterlist *sg_last(struct scatterlist *sgl, unsigned int nents)
  {
0db9299f4   Jens Axboe   SG: Move function...
107
108
109
110
111
  	struct scatterlist *sg, *ret = NULL;
  	unsigned int i;
  
  	for_each_sg(sgl, sg, nents, i)
  		ret = sg;
0db9299f4   Jens Axboe   SG: Move function...
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
  #ifdef CONFIG_DEBUG_SG
  	BUG_ON(sgl[0].sg_magic != SG_MAGIC);
  	BUG_ON(!sg_is_last(ret));
  #endif
  	return ret;
  }
  EXPORT_SYMBOL(sg_last);
  
  /**
   * sg_init_table - Initialize SG table
   * @sgl:	   The SG table
   * @nents:	   Number of entries in table
   *
   * Notes:
   *   If this is part of a chained sg table, sg_mark_end() should be
   *   used only on the last table part.
   *
   **/
  void sg_init_table(struct scatterlist *sgl, unsigned int nents)
  {
  	memset(sgl, 0, sizeof(*sgl) * nents);
  #ifdef CONFIG_DEBUG_SG
  	{
  		unsigned int i;
  		for (i = 0; i < nents; i++)
  			sgl[i].sg_magic = SG_MAGIC;
  	}
  #endif
  	sg_mark_end(&sgl[nents - 1]);
  }
  EXPORT_SYMBOL(sg_init_table);
  
  /**
   * sg_init_one - Initialize a single entry sg list
   * @sg:		 SG entry
   * @buf:	 Virtual address for IO
   * @buflen:	 IO length
   *
   **/
  void sg_init_one(struct scatterlist *sg, const void *buf, unsigned int buflen)
  {
  	sg_init_table(sg, 1);
  	sg_set_buf(sg, buf, buflen);
  }
  EXPORT_SYMBOL(sg_init_one);
  
  /*
   * The default behaviour of sg_alloc_table() is to use these kmalloc/kfree
   * helpers.
   */
  static struct scatterlist *sg_kmalloc(unsigned int nents, gfp_t gfp_mask)
  {
b94de9bb7   Chris Wilson   lib/scatterlist: ...
164
165
166
167
168
169
170
171
172
173
174
175
176
177
  	if (nents == SG_MAX_SINGLE_ALLOC) {
  		/*
  		 * Kmemleak doesn't track page allocations as they are not
  		 * commonly used (in a raw form) for kernel data structures.
  		 * As we chain together a list of pages and then a normal
  		 * kmalloc (tracked by kmemleak), in order to for that last
  		 * allocation not to become decoupled (and thus a
  		 * false-positive) we need to inform kmemleak of all the
  		 * intermediate allocations.
  		 */
  		void *ptr = (void *) __get_free_page(gfp_mask);
  		kmemleak_alloc(ptr, PAGE_SIZE, 1, gfp_mask);
  		return ptr;
  	} else
0db9299f4   Jens Axboe   SG: Move function...
178
179
180
181
182
  		return kmalloc(nents * sizeof(struct scatterlist), gfp_mask);
  }
  
  static void sg_kfree(struct scatterlist *sg, unsigned int nents)
  {
b94de9bb7   Chris Wilson   lib/scatterlist: ...
183
184
  	if (nents == SG_MAX_SINGLE_ALLOC) {
  		kmemleak_free(sg);
0db9299f4   Jens Axboe   SG: Move function...
185
  		free_page((unsigned long) sg);
b94de9bb7   Chris Wilson   lib/scatterlist: ...
186
  	} else
0db9299f4   Jens Axboe   SG: Move function...
187
188
189
190
191
192
  		kfree(sg);
  }
  
  /**
   * __sg_free_table - Free a previously mapped sg table
   * @table:	The sg table header to use
7cedb1f17   James Bottomley   SG: work with the...
193
   * @max_ents:	The maximum number of entries per single scatterlist
c53c6d6a6   Christoph Hellwig   scatterlist: allo...
194
   * @skip_first_chunk: don't free the (preallocated) first scatterlist chunk
0db9299f4   Jens Axboe   SG: Move function...
195
196
197
   * @free_fn:	Free function
   *
   *  Description:
7cedb1f17   James Bottomley   SG: work with the...
198
199
200
   *    Free an sg table previously allocated and setup with
   *    __sg_alloc_table().  The @max_ents value must be identical to
   *    that previously used with __sg_alloc_table().
0db9299f4   Jens Axboe   SG: Move function...
201
202
   *
   **/
7cedb1f17   James Bottomley   SG: work with the...
203
  void __sg_free_table(struct sg_table *table, unsigned int max_ents,
c53c6d6a6   Christoph Hellwig   scatterlist: allo...
204
  		     bool skip_first_chunk, sg_free_fn *free_fn)
0db9299f4   Jens Axboe   SG: Move function...
205
206
207
208
209
210
211
212
213
214
215
216
  {
  	struct scatterlist *sgl, *next;
  
  	if (unlikely(!table->sgl))
  		return;
  
  	sgl = table->sgl;
  	while (table->orig_nents) {
  		unsigned int alloc_size = table->orig_nents;
  		unsigned int sg_size;
  
  		/*
7cedb1f17   James Bottomley   SG: work with the...
217
  		 * If we have more than max_ents segments left,
0db9299f4   Jens Axboe   SG: Move function...
218
219
220
221
  		 * then assign 'next' to the sg table after the current one.
  		 * sg_size is then one less than alloc size, since the last
  		 * element is the chain pointer.
  		 */
7cedb1f17   James Bottomley   SG: work with the...
222
223
224
  		if (alloc_size > max_ents) {
  			next = sg_chain_ptr(&sgl[max_ents - 1]);
  			alloc_size = max_ents;
0db9299f4   Jens Axboe   SG: Move function...
225
226
227
228
229
230
231
  			sg_size = alloc_size - 1;
  		} else {
  			sg_size = alloc_size;
  			next = NULL;
  		}
  
  		table->orig_nents -= sg_size;
c21e59d8d   Tony Battersby   lib/scatterlist: ...
232
  		if (skip_first_chunk)
c53c6d6a6   Christoph Hellwig   scatterlist: allo...
233
  			skip_first_chunk = false;
c21e59d8d   Tony Battersby   lib/scatterlist: ...
234
235
  		else
  			free_fn(sgl, alloc_size);
0db9299f4   Jens Axboe   SG: Move function...
236
237
238
239
240
241
242
243
244
245
246
247
248
249
  		sgl = next;
  	}
  
  	table->sgl = NULL;
  }
  EXPORT_SYMBOL(__sg_free_table);
  
  /**
   * sg_free_table - Free a previously allocated sg table
   * @table:	The mapped sg table header
   *
   **/
  void sg_free_table(struct sg_table *table)
  {
c53c6d6a6   Christoph Hellwig   scatterlist: allo...
250
  	__sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree);
0db9299f4   Jens Axboe   SG: Move function...
251
252
253
254
255
256
257
  }
  EXPORT_SYMBOL(sg_free_table);
  
  /**
   * __sg_alloc_table - Allocate and initialize an sg table with given allocator
   * @table:	The sg table header to use
   * @nents:	Number of entries in sg list
7cedb1f17   James Bottomley   SG: work with the...
258
   * @max_ents:	The maximum number of entries the allocator returns per call
0db9299f4   Jens Axboe   SG: Move function...
259
260
261
   * @gfp_mask:	GFP allocation mask
   * @alloc_fn:	Allocator to use
   *
7cedb1f17   James Bottomley   SG: work with the...
262
263
264
265
266
267
   * Description:
   *   This function returns a @table @nents long. The allocator is
   *   defined to return scatterlist chunks of maximum size @max_ents.
   *   Thus if @nents is bigger than @max_ents, the scatterlists will be
   *   chained in units of @max_ents.
   *
0db9299f4   Jens Axboe   SG: Move function...
268
269
270
271
272
   * Notes:
   *   If this function returns non-0 (eg failure), the caller must call
   *   __sg_free_table() to cleanup any leftover allocations.
   *
   **/
7cedb1f17   James Bottomley   SG: work with the...
273
  int __sg_alloc_table(struct sg_table *table, unsigned int nents,
c53c6d6a6   Christoph Hellwig   scatterlist: allo...
274
275
  		     unsigned int max_ents, struct scatterlist *first_chunk,
  		     gfp_t gfp_mask, sg_alloc_fn *alloc_fn)
0db9299f4   Jens Axboe   SG: Move function...
276
277
278
  {
  	struct scatterlist *sg, *prv;
  	unsigned int left;
27daabd9b   Dan Carpenter   lib/scatterlist: ...
279
280
281
282
  	memset(table, 0, sizeof(*table));
  
  	if (nents == 0)
  		return -EINVAL;
308c09f17   Laura Abbott   lib/scatterlist: ...
283
  #ifndef CONFIG_ARCH_HAS_SG_CHAIN
6fd59a83b   Nick Bowler   scatterlist: don'...
284
285
  	if (WARN_ON_ONCE(nents > max_ents))
  		return -EINVAL;
0db9299f4   Jens Axboe   SG: Move function...
286
  #endif
0db9299f4   Jens Axboe   SG: Move function...
287
288
289
290
  	left = nents;
  	prv = NULL;
  	do {
  		unsigned int sg_size, alloc_size = left;
7cedb1f17   James Bottomley   SG: work with the...
291
292
  		if (alloc_size > max_ents) {
  			alloc_size = max_ents;
0db9299f4   Jens Axboe   SG: Move function...
293
294
295
296
297
  			sg_size = alloc_size - 1;
  		} else
  			sg_size = alloc_size;
  
  		left -= sg_size;
c53c6d6a6   Christoph Hellwig   scatterlist: allo...
298
299
300
301
302
303
  		if (first_chunk) {
  			sg = first_chunk;
  			first_chunk = NULL;
  		} else {
  			sg = alloc_fn(alloc_size, gfp_mask);
  		}
edce6820a   Jeffrey Carlyle   scatterlist: prev...
304
305
306
307
308
309
310
311
312
313
314
315
  		if (unlikely(!sg)) {
  			/*
  			 * Adjust entry count to reflect that the last
  			 * entry of the previous table won't be used for
  			 * linkage.  Without this, sg_kfree() may get
  			 * confused.
  			 */
  			if (prv)
  				table->nents = ++table->orig_nents;
  
   			return -ENOMEM;
  		}
0db9299f4   Jens Axboe   SG: Move function...
316
317
318
319
320
321
322
323
324
  
  		sg_init_table(sg, alloc_size);
  		table->nents = table->orig_nents += sg_size;
  
  		/*
  		 * If this is the first mapping, assign the sg table header.
  		 * If this is not the first mapping, chain previous part.
  		 */
  		if (prv)
7cedb1f17   James Bottomley   SG: work with the...
325
  			sg_chain(prv, max_ents, sg);
0db9299f4   Jens Axboe   SG: Move function...
326
327
328
329
330
331
332
333
  		else
  			table->sgl = sg;
  
  		/*
  		 * If no more entries after this one, mark the end
  		 */
  		if (!left)
  			sg_mark_end(&sg[sg_size - 1]);
0db9299f4   Jens Axboe   SG: Move function...
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
  		prv = sg;
  	} while (left);
  
  	return 0;
  }
  EXPORT_SYMBOL(__sg_alloc_table);
  
  /**
   * sg_alloc_table - Allocate and initialize an sg table
   * @table:	The sg table header to use
   * @nents:	Number of entries in sg list
   * @gfp_mask:	GFP allocation mask
   *
   *  Description:
   *    Allocate and initialize an sg table. If @nents@ is larger than
   *    SG_MAX_SINGLE_ALLOC a chained sg table will be setup.
   *
   **/
  int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
  {
  	int ret;
7cedb1f17   James Bottomley   SG: work with the...
355
  	ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC,
c53c6d6a6   Christoph Hellwig   scatterlist: allo...
356
  			       NULL, gfp_mask, sg_kmalloc);
0db9299f4   Jens Axboe   SG: Move function...
357
  	if (unlikely(ret))
c53c6d6a6   Christoph Hellwig   scatterlist: allo...
358
  		__sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree);
0db9299f4   Jens Axboe   SG: Move function...
359
360
361
362
  
  	return ret;
  }
  EXPORT_SYMBOL(sg_alloc_table);
b1adaf65b   FUJITA Tomonori   [SCSI] block: add...
363
364
  
  /**
efc42bc98   Tomasz Stanislawski   scatterlist: add ...
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
   * sg_alloc_table_from_pages - Allocate and initialize an sg table from
   *			       an array of pages
   * @sgt:	The sg table header to use
   * @pages:	Pointer to an array of page pointers
   * @n_pages:	Number of pages in the pages array
   * @offset:     Offset from start of the first page to the start of a buffer
   * @size:       Number of valid bytes in the buffer (after offset)
   * @gfp_mask:	GFP allocation mask
   *
   *  Description:
   *    Allocate and initialize an sg table from a list of pages. Contiguous
   *    ranges of the pages are squashed into a single scatterlist node. A user
   *    may provide an offset at a start and a size of valid data in a buffer
   *    specified by the page array. The returned sg table is released by
   *    sg_free_table.
   *
   * Returns:
   *   0 on success, negative error on failure
   */
  int sg_alloc_table_from_pages(struct sg_table *sgt,
  	struct page **pages, unsigned int n_pages,
  	unsigned long offset, unsigned long size,
  	gfp_t gfp_mask)
  {
  	unsigned int chunks;
  	unsigned int i;
  	unsigned int cur_page;
  	int ret;
  	struct scatterlist *s;
  
  	/* compute number of contiguous chunks */
  	chunks = 1;
  	for (i = 1; i < n_pages; ++i)
  		if (page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1)
  			++chunks;
  
  	ret = sg_alloc_table(sgt, chunks, gfp_mask);
  	if (unlikely(ret))
  		return ret;
  
  	/* merging chunks and putting them into the scatterlist */
  	cur_page = 0;
  	for_each_sg(sgt->sgl, s, sgt->orig_nents, i) {
  		unsigned long chunk_size;
  		unsigned int j;
  
  		/* look for the end of the current chunk */
  		for (j = cur_page + 1; j < n_pages; ++j)
  			if (page_to_pfn(pages[j]) !=
  			    page_to_pfn(pages[j - 1]) + 1)
  				break;
  
  		chunk_size = ((j - cur_page) << PAGE_SHIFT) - offset;
  		sg_set_page(s, pages[cur_page], min(size, chunk_size), offset);
  		size -= chunk_size;
  		offset = 0;
  		cur_page = j;
  	}
  
  	return 0;
  }
  EXPORT_SYMBOL(sg_alloc_table_from_pages);
a321e91b6   Imre Deak   lib/scatterlist: ...
427
428
429
430
431
432
  void __sg_page_iter_start(struct sg_page_iter *piter,
  			  struct scatterlist *sglist, unsigned int nents,
  			  unsigned long pgoffset)
  {
  	piter->__pg_advance = 0;
  	piter->__nents = nents;
a321e91b6   Imre Deak   lib/scatterlist: ...
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
  	piter->sg = sglist;
  	piter->sg_pgoffset = pgoffset;
  }
  EXPORT_SYMBOL(__sg_page_iter_start);
  
  static int sg_page_count(struct scatterlist *sg)
  {
  	return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT;
  }
  
  bool __sg_page_iter_next(struct sg_page_iter *piter)
  {
  	if (!piter->__nents || !piter->sg)
  		return false;
  
  	piter->sg_pgoffset += piter->__pg_advance;
  	piter->__pg_advance = 1;
  
  	while (piter->sg_pgoffset >= sg_page_count(piter->sg)) {
  		piter->sg_pgoffset -= sg_page_count(piter->sg);
  		piter->sg = sg_next(piter->sg);
  		if (!--piter->__nents || !piter->sg)
  			return false;
  	}
a321e91b6   Imre Deak   lib/scatterlist: ...
457
458
459
460
  
  	return true;
  }
  EXPORT_SYMBOL(__sg_page_iter_next);
efc42bc98   Tomasz Stanislawski   scatterlist: add ...
461
  /**
137d3edb4   Tejun Heo   sg: reimplement s...
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
   * sg_miter_start - start mapping iteration over a sg list
   * @miter: sg mapping iter to be started
   * @sgl: sg list to iterate over
   * @nents: number of sg entries
   *
   * Description:
   *   Starts mapping iterator @miter.
   *
   * Context:
   *   Don't care.
   */
  void sg_miter_start(struct sg_mapping_iter *miter, struct scatterlist *sgl,
  		    unsigned int nents, unsigned int flags)
  {
  	memset(miter, 0, sizeof(struct sg_mapping_iter));
4225fc855   Imre Deak   lib/scatterlist: ...
477
  	__sg_page_iter_start(&miter->piter, sgl, nents, 0);
6de7e356f   Sebastian Andrzej Siewior   lib/scatterlist: ...
478
  	WARN_ON(!(flags & (SG_MITER_TO_SG | SG_MITER_FROM_SG)));
137d3edb4   Tejun Heo   sg: reimplement s...
479
480
481
  	miter->__flags = flags;
  }
  EXPORT_SYMBOL(sg_miter_start);
110520048   Akinobu Mita   lib/scatterlist: ...
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
  static bool sg_miter_get_next_page(struct sg_mapping_iter *miter)
  {
  	if (!miter->__remaining) {
  		struct scatterlist *sg;
  		unsigned long pgoffset;
  
  		if (!__sg_page_iter_next(&miter->piter))
  			return false;
  
  		sg = miter->piter.sg;
  		pgoffset = miter->piter.sg_pgoffset;
  
  		miter->__offset = pgoffset ? 0 : sg->offset;
  		miter->__remaining = sg->offset + sg->length -
  				(pgoffset << PAGE_SHIFT) - miter->__offset;
  		miter->__remaining = min_t(unsigned long, miter->__remaining,
  					   PAGE_SIZE - miter->__offset);
  	}
  
  	return true;
  }
137d3edb4   Tejun Heo   sg: reimplement s...
503
  /**
df642cea2   Akinobu Mita   lib/scatterlist: ...
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
   * sg_miter_skip - reposition mapping iterator
   * @miter: sg mapping iter to be skipped
   * @offset: number of bytes to plus the current location
   *
   * Description:
   *   Sets the offset of @miter to its current location plus @offset bytes.
   *   If mapping iterator @miter has been proceeded by sg_miter_next(), this
   *   stops @miter.
   *
   * Context:
   *   Don't care if @miter is stopped, or not proceeded yet.
   *   Otherwise, preemption disabled if the SG_MITER_ATOMIC is set.
   *
   * Returns:
   *   true if @miter contains the valid mapping.  false if end of sg
   *   list is reached.
   */
0d6077f8b   Ming Lei   lib/scatterlist: ...
521
  bool sg_miter_skip(struct sg_mapping_iter *miter, off_t offset)
df642cea2   Akinobu Mita   lib/scatterlist: ...
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
  {
  	sg_miter_stop(miter);
  
  	while (offset) {
  		off_t consumed;
  
  		if (!sg_miter_get_next_page(miter))
  			return false;
  
  		consumed = min_t(off_t, offset, miter->__remaining);
  		miter->__offset += consumed;
  		miter->__remaining -= consumed;
  		offset -= consumed;
  	}
  
  	return true;
  }
0d6077f8b   Ming Lei   lib/scatterlist: ...
539
  EXPORT_SYMBOL(sg_miter_skip);
df642cea2   Akinobu Mita   lib/scatterlist: ...
540
541
  
  /**
137d3edb4   Tejun Heo   sg: reimplement s...
542
543
544
545
   * sg_miter_next - proceed mapping iterator to the next mapping
   * @miter: sg mapping iter to proceed
   *
   * Description:
8290e2d2d   Tejun Heo   scatterlist: atom...
546
547
548
   *   Proceeds @miter to the next mapping.  @miter should have been started
   *   using sg_miter_start().  On successful return, @miter->page,
   *   @miter->addr and @miter->length point to the current mapping.
137d3edb4   Tejun Heo   sg: reimplement s...
549
550
   *
   * Context:
8290e2d2d   Tejun Heo   scatterlist: atom...
551
552
   *   Preemption disabled if SG_MITER_ATOMIC.  Preemption must stay disabled
   *   till @miter is stopped.  May sleep if !SG_MITER_ATOMIC.
137d3edb4   Tejun Heo   sg: reimplement s...
553
554
555
556
557
558
559
   *
   * Returns:
   *   true if @miter contains the next mapping.  false if end of sg
   *   list is reached.
   */
  bool sg_miter_next(struct sg_mapping_iter *miter)
  {
137d3edb4   Tejun Heo   sg: reimplement s...
560
  	sg_miter_stop(miter);
4225fc855   Imre Deak   lib/scatterlist: ...
561
562
563
564
  	/*
  	 * Get to the next page if necessary.
  	 * __remaining, __offset is adjusted by sg_miter_stop
  	 */
110520048   Akinobu Mita   lib/scatterlist: ...
565
566
  	if (!sg_miter_get_next_page(miter))
  		return false;
137d3edb4   Tejun Heo   sg: reimplement s...
567

2db76d7c3   Imre Deak   lib/scatterlist: ...
568
  	miter->page = sg_page_iter_page(&miter->piter);
4225fc855   Imre Deak   lib/scatterlist: ...
569
  	miter->consumed = miter->length = miter->__remaining;
137d3edb4   Tejun Heo   sg: reimplement s...
570
571
  
  	if (miter->__flags & SG_MITER_ATOMIC)
4225fc855   Imre Deak   lib/scatterlist: ...
572
  		miter->addr = kmap_atomic(miter->page) + miter->__offset;
137d3edb4   Tejun Heo   sg: reimplement s...
573
  	else
4225fc855   Imre Deak   lib/scatterlist: ...
574
  		miter->addr = kmap(miter->page) + miter->__offset;
137d3edb4   Tejun Heo   sg: reimplement s...
575
576
577
578
579
580
581
582
583
584
585
  
  	return true;
  }
  EXPORT_SYMBOL(sg_miter_next);
  
  /**
   * sg_miter_stop - stop mapping iteration
   * @miter: sg mapping iter to be stopped
   *
   * Description:
   *   Stops mapping iterator @miter.  @miter should have been started
4ba6a2b28   Masahiro Yamada   scatterlist: fix ...
586
587
588
   *   using sg_miter_start().  A stopped iteration can be resumed by
   *   calling sg_miter_next() on it.  This is useful when resources (kmap)
   *   need to be released during iteration.
137d3edb4   Tejun Heo   sg: reimplement s...
589
590
   *
   * Context:
8290e2d2d   Tejun Heo   scatterlist: atom...
591
592
   *   Preemption disabled if the SG_MITER_ATOMIC is set.  Don't care
   *   otherwise.
137d3edb4   Tejun Heo   sg: reimplement s...
593
594
595
596
597
598
599
600
   */
  void sg_miter_stop(struct sg_mapping_iter *miter)
  {
  	WARN_ON(miter->consumed > miter->length);
  
  	/* drop resources from the last iteration */
  	if (miter->addr) {
  		miter->__offset += miter->consumed;
4225fc855   Imre Deak   lib/scatterlist: ...
601
  		miter->__remaining -= miter->consumed;
137d3edb4   Tejun Heo   sg: reimplement s...
602

3d77b50c5   Ming Lei   lib/scatterlist.c...
603
604
  		if ((miter->__flags & SG_MITER_TO_SG) &&
  		    !PageSlab(miter->page))
6de7e356f   Sebastian Andrzej Siewior   lib/scatterlist: ...
605
  			flush_kernel_dcache_page(miter->page);
137d3edb4   Tejun Heo   sg: reimplement s...
606
  		if (miter->__flags & SG_MITER_ATOMIC) {
8290e2d2d   Tejun Heo   scatterlist: atom...
607
  			WARN_ON_ONCE(preemptible());
c3eede8e0   Cong Wang   lib: remove the s...
608
  			kunmap_atomic(miter->addr);
137d3edb4   Tejun Heo   sg: reimplement s...
609
  		} else
f652c521e   Arjan van de Ven   lib/scatterlist.c...
610
  			kunmap(miter->page);
137d3edb4   Tejun Heo   sg: reimplement s...
611
612
613
614
615
616
617
618
619
620
  
  		miter->page = NULL;
  		miter->addr = NULL;
  		miter->length = 0;
  		miter->consumed = 0;
  	}
  }
  EXPORT_SYMBOL(sg_miter_stop);
  
  /**
b1adaf65b   FUJITA Tomonori   [SCSI] block: add...
621
622
623
624
625
   * sg_copy_buffer - Copy data between a linear buffer and an SG list
   * @sgl:		 The SG list
   * @nents:		 Number of SG entries
   * @buf:		 Where to copy from
   * @buflen:		 The number of bytes to copy
df642cea2   Akinobu Mita   lib/scatterlist: ...
626
627
628
   * @skip:		 Number of bytes to skip before copying
   * @to_buffer:		 transfer direction (true == from an sg list to a
   *			 buffer, false == from a buffer to an sg list
b1adaf65b   FUJITA Tomonori   [SCSI] block: add...
629
630
631
632
   *
   * Returns the number of copied bytes.
   *
   **/
386ecb121   Dave Gordon   drivers/scsi/scsi...
633
634
  size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf,
  		      size_t buflen, off_t skip, bool to_buffer)
b1adaf65b   FUJITA Tomonori   [SCSI] block: add...
635
  {
137d3edb4   Tejun Heo   sg: reimplement s...
636
637
  	unsigned int offset = 0;
  	struct sg_mapping_iter miter;
6de7e356f   Sebastian Andrzej Siewior   lib/scatterlist: ...
638
639
640
641
642
643
  	unsigned int sg_flags = SG_MITER_ATOMIC;
  
  	if (to_buffer)
  		sg_flags |= SG_MITER_FROM_SG;
  	else
  		sg_flags |= SG_MITER_TO_SG;
137d3edb4   Tejun Heo   sg: reimplement s...
644

6de7e356f   Sebastian Andrzej Siewior   lib/scatterlist: ...
645
  	sg_miter_start(&miter, sgl, nents, sg_flags);
137d3edb4   Tejun Heo   sg: reimplement s...
646

df642cea2   Akinobu Mita   lib/scatterlist: ...
647
648
  	if (!sg_miter_skip(&miter, skip))
  		return false;
1d5210ef7   Gilad Ben-Yossef   scatterlist: reor...
649
  	while ((offset < buflen) && sg_miter_next(&miter)) {
137d3edb4   Tejun Heo   sg: reimplement s...
650
651
652
653
654
655
  		unsigned int len;
  
  		len = min(miter.length, buflen - offset);
  
  		if (to_buffer)
  			memcpy(buf + offset, miter.addr, len);
6de7e356f   Sebastian Andrzej Siewior   lib/scatterlist: ...
656
  		else
137d3edb4   Tejun Heo   sg: reimplement s...
657
  			memcpy(miter.addr, buf + offset, len);
b1adaf65b   FUJITA Tomonori   [SCSI] block: add...
658

137d3edb4   Tejun Heo   sg: reimplement s...
659
  		offset += len;
b1adaf65b   FUJITA Tomonori   [SCSI] block: add...
660
  	}
137d3edb4   Tejun Heo   sg: reimplement s...
661
662
663
  	sg_miter_stop(&miter);
  
  	return offset;
b1adaf65b   FUJITA Tomonori   [SCSI] block: add...
664
  }
386ecb121   Dave Gordon   drivers/scsi/scsi...
665
  EXPORT_SYMBOL(sg_copy_buffer);
b1adaf65b   FUJITA Tomonori   [SCSI] block: add...
666
667
668
669
670
671
672
673
674
675
676
677
  
  /**
   * sg_copy_from_buffer - Copy from a linear buffer to an SG list
   * @sgl:		 The SG list
   * @nents:		 Number of SG entries
   * @buf:		 Where to copy from
   * @buflen:		 The number of bytes to copy
   *
   * Returns the number of copied bytes.
   *
   **/
  size_t sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents,
2a1bf8f93   Dave Gordon   lib/scatterlist: ...
678
  			   const void *buf, size_t buflen)
b1adaf65b   FUJITA Tomonori   [SCSI] block: add...
679
  {
2a1bf8f93   Dave Gordon   lib/scatterlist: ...
680
  	return sg_copy_buffer(sgl, nents, (void *)buf, buflen, 0, false);
b1adaf65b   FUJITA Tomonori   [SCSI] block: add...
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
  }
  EXPORT_SYMBOL(sg_copy_from_buffer);
  
  /**
   * sg_copy_to_buffer - Copy from an SG list to a linear buffer
   * @sgl:		 The SG list
   * @nents:		 Number of SG entries
   * @buf:		 Where to copy to
   * @buflen:		 The number of bytes to copy
   *
   * Returns the number of copied bytes.
   *
   **/
  size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents,
  			 void *buf, size_t buflen)
  {
df642cea2   Akinobu Mita   lib/scatterlist: ...
697
  	return sg_copy_buffer(sgl, nents, buf, buflen, 0, true);
b1adaf65b   FUJITA Tomonori   [SCSI] block: add...
698
699
  }
  EXPORT_SYMBOL(sg_copy_to_buffer);
df642cea2   Akinobu Mita   lib/scatterlist: ...
700
701
702
703
704
705
  
  /**
   * sg_pcopy_from_buffer - Copy from a linear buffer to an SG list
   * @sgl:		 The SG list
   * @nents:		 Number of SG entries
   * @buf:		 Where to copy from
df642cea2   Akinobu Mita   lib/scatterlist: ...
706
   * @buflen:		 The number of bytes to copy
4dc7daf84   Dave Gordon   lib/scatterlist.c...
707
   * @skip:		 Number of bytes to skip before copying
df642cea2   Akinobu Mita   lib/scatterlist: ...
708
709
710
711
712
   *
   * Returns the number of copied bytes.
   *
   **/
  size_t sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents,
2a1bf8f93   Dave Gordon   lib/scatterlist: ...
713
  			    const void *buf, size_t buflen, off_t skip)
df642cea2   Akinobu Mita   lib/scatterlist: ...
714
  {
2a1bf8f93   Dave Gordon   lib/scatterlist: ...
715
  	return sg_copy_buffer(sgl, nents, (void *)buf, buflen, skip, false);
df642cea2   Akinobu Mita   lib/scatterlist: ...
716
717
718
719
720
721
722
723
  }
  EXPORT_SYMBOL(sg_pcopy_from_buffer);
  
  /**
   * sg_pcopy_to_buffer - Copy from an SG list to a linear buffer
   * @sgl:		 The SG list
   * @nents:		 Number of SG entries
   * @buf:		 Where to copy to
df642cea2   Akinobu Mita   lib/scatterlist: ...
724
   * @buflen:		 The number of bytes to copy
4dc7daf84   Dave Gordon   lib/scatterlist.c...
725
   * @skip:		 Number of bytes to skip before copying
df642cea2   Akinobu Mita   lib/scatterlist: ...
726
727
728
729
730
731
732
733
734
735
   *
   * Returns the number of copied bytes.
   *
   **/
  size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents,
  			  void *buf, size_t buflen, off_t skip)
  {
  	return sg_copy_buffer(sgl, nents, buf, buflen, skip, true);
  }
  EXPORT_SYMBOL(sg_pcopy_to_buffer);
0945e5699   Johannes Thumshirn   scatterlist: add ...
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
  
  /**
   * sg_zero_buffer - Zero-out a part of a SG list
   * @sgl:		 The SG list
   * @nents:		 Number of SG entries
   * @buflen:		 The number of bytes to zero out
   * @skip:		 Number of bytes to skip before zeroing
   *
   * Returns the number of bytes zeroed.
   **/
  size_t sg_zero_buffer(struct scatterlist *sgl, unsigned int nents,
  		       size_t buflen, off_t skip)
  {
  	unsigned int offset = 0;
  	struct sg_mapping_iter miter;
  	unsigned int sg_flags = SG_MITER_ATOMIC | SG_MITER_TO_SG;
  
  	sg_miter_start(&miter, sgl, nents, sg_flags);
  
  	if (!sg_miter_skip(&miter, skip))
  		return false;
  
  	while (offset < buflen && sg_miter_next(&miter)) {
  		unsigned int len;
  
  		len = min(miter.length, buflen - offset);
  		memset(miter.addr, 0, len);
  
  		offset += len;
  	}
  
  	sg_miter_stop(&miter);
  	return offset;
  }
  EXPORT_SYMBOL(sg_zero_buffer);