Commit c53c6d6a authored by Christoph Hellwig's avatar Christoph Hellwig

scatterlist: allow chaining to preallocated chunks

Blk-mq drivers usually preallocate their S/G list as part of the request,
but if we want to support the very large S/G lists currently supported by
the SCSI code that would tie up a lot of memory in the preallocated request
pool.  Add support to the scatterlist code so that it can initialize a
S/G list that uses a preallocated first chunks and dynamically allocated
additional chunks.  That way the scsi-mq code can preallocate a first
page worth of S/G entries as part of the request, and dynamically extend
the S/G list when needed.
Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Reviewed-by: default avatarMartin K. Petersen <martin.petersen@oracle.com>
Reviewed-by: default avatarHannes Reinecke <hare@suse.de>
Reviewed-by: default avatarWebb Scales <webbnh@hp.com>
Acked-by: default avatarJens Axboe <axboe@kernel.dk>
Tested-by: default avatarBart Van Assche <bvanassche@acm.org>
Tested-by: default avatarRobert Elliott <elliott@hp.com>
parent f6d47e74
...@@ -564,6 +564,11 @@ static struct scatterlist *scsi_sg_alloc(unsigned int nents, gfp_t gfp_mask) ...@@ -564,6 +564,11 @@ static struct scatterlist *scsi_sg_alloc(unsigned int nents, gfp_t gfp_mask)
return mempool_alloc(sgp->pool, gfp_mask); return mempool_alloc(sgp->pool, gfp_mask);
} }
static void scsi_free_sgtable(struct scsi_data_buffer *sdb)
{
__sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS, false, scsi_sg_free);
}
static int scsi_alloc_sgtable(struct scsi_data_buffer *sdb, int nents, static int scsi_alloc_sgtable(struct scsi_data_buffer *sdb, int nents,
gfp_t gfp_mask) gfp_t gfp_mask)
{ {
...@@ -572,19 +577,12 @@ static int scsi_alloc_sgtable(struct scsi_data_buffer *sdb, int nents, ...@@ -572,19 +577,12 @@ static int scsi_alloc_sgtable(struct scsi_data_buffer *sdb, int nents,
BUG_ON(!nents); BUG_ON(!nents);
ret = __sg_alloc_table(&sdb->table, nents, SCSI_MAX_SG_SEGMENTS, ret = __sg_alloc_table(&sdb->table, nents, SCSI_MAX_SG_SEGMENTS,
gfp_mask, scsi_sg_alloc); NULL, gfp_mask, scsi_sg_alloc);
if (unlikely(ret)) if (unlikely(ret))
__sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS, scsi_free_sgtable(sdb);
scsi_sg_free);
return ret; return ret;
} }
static void scsi_free_sgtable(struct scsi_data_buffer *sdb)
{
__sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS, scsi_sg_free);
}
/* /*
* Function: scsi_release_buffers() * Function: scsi_release_buffers()
* *
......
...@@ -229,10 +229,10 @@ void sg_init_one(struct scatterlist *, const void *, unsigned int); ...@@ -229,10 +229,10 @@ void sg_init_one(struct scatterlist *, const void *, unsigned int);
typedef struct scatterlist *(sg_alloc_fn)(unsigned int, gfp_t); typedef struct scatterlist *(sg_alloc_fn)(unsigned int, gfp_t);
typedef void (sg_free_fn)(struct scatterlist *, unsigned int); typedef void (sg_free_fn)(struct scatterlist *, unsigned int);
void __sg_free_table(struct sg_table *, unsigned int, sg_free_fn *); void __sg_free_table(struct sg_table *, unsigned int, bool, sg_free_fn *);
void sg_free_table(struct sg_table *); void sg_free_table(struct sg_table *);
int __sg_alloc_table(struct sg_table *, unsigned int, unsigned int, gfp_t, int __sg_alloc_table(struct sg_table *, unsigned int, unsigned int,
sg_alloc_fn *); struct scatterlist *, gfp_t, sg_alloc_fn *);
int sg_alloc_table(struct sg_table *, unsigned int, gfp_t); int sg_alloc_table(struct sg_table *, unsigned int, gfp_t);
int sg_alloc_table_from_pages(struct sg_table *sgt, int sg_alloc_table_from_pages(struct sg_table *sgt,
struct page **pages, unsigned int n_pages, struct page **pages, unsigned int n_pages,
......
...@@ -165,6 +165,7 @@ static void sg_kfree(struct scatterlist *sg, unsigned int nents) ...@@ -165,6 +165,7 @@ static void sg_kfree(struct scatterlist *sg, unsigned int nents)
* __sg_free_table - Free a previously mapped sg table * __sg_free_table - Free a previously mapped sg table
* @table: The sg table header to use * @table: The sg table header to use
* @max_ents: The maximum number of entries per single scatterlist * @max_ents: The maximum number of entries per single scatterlist
* @skip_first_chunk: don't free the (preallocated) first scatterlist chunk
* @free_fn: Free function * @free_fn: Free function
* *
* Description: * Description:
...@@ -174,7 +175,7 @@ static void sg_kfree(struct scatterlist *sg, unsigned int nents) ...@@ -174,7 +175,7 @@ static void sg_kfree(struct scatterlist *sg, unsigned int nents)
* *
**/ **/
void __sg_free_table(struct sg_table *table, unsigned int max_ents, void __sg_free_table(struct sg_table *table, unsigned int max_ents,
sg_free_fn *free_fn) bool skip_first_chunk, sg_free_fn *free_fn)
{ {
struct scatterlist *sgl, *next; struct scatterlist *sgl, *next;
...@@ -202,7 +203,10 @@ void __sg_free_table(struct sg_table *table, unsigned int max_ents, ...@@ -202,7 +203,10 @@ void __sg_free_table(struct sg_table *table, unsigned int max_ents,
} }
table->orig_nents -= sg_size; table->orig_nents -= sg_size;
if (!skip_first_chunk) {
free_fn(sgl, alloc_size); free_fn(sgl, alloc_size);
skip_first_chunk = false;
}
sgl = next; sgl = next;
} }
...@@ -217,7 +221,7 @@ EXPORT_SYMBOL(__sg_free_table); ...@@ -217,7 +221,7 @@ EXPORT_SYMBOL(__sg_free_table);
**/ **/
void sg_free_table(struct sg_table *table) void sg_free_table(struct sg_table *table)
{ {
__sg_free_table(table, SG_MAX_SINGLE_ALLOC, sg_kfree); __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree);
} }
EXPORT_SYMBOL(sg_free_table); EXPORT_SYMBOL(sg_free_table);
...@@ -241,8 +245,8 @@ EXPORT_SYMBOL(sg_free_table); ...@@ -241,8 +245,8 @@ EXPORT_SYMBOL(sg_free_table);
* *
**/ **/
int __sg_alloc_table(struct sg_table *table, unsigned int nents, int __sg_alloc_table(struct sg_table *table, unsigned int nents,
unsigned int max_ents, gfp_t gfp_mask, unsigned int max_ents, struct scatterlist *first_chunk,
sg_alloc_fn *alloc_fn) gfp_t gfp_mask, sg_alloc_fn *alloc_fn)
{ {
struct scatterlist *sg, *prv; struct scatterlist *sg, *prv;
unsigned int left; unsigned int left;
...@@ -269,7 +273,12 @@ int __sg_alloc_table(struct sg_table *table, unsigned int nents, ...@@ -269,7 +273,12 @@ int __sg_alloc_table(struct sg_table *table, unsigned int nents,
left -= sg_size; left -= sg_size;
if (first_chunk) {
sg = first_chunk;
first_chunk = NULL;
} else {
sg = alloc_fn(alloc_size, gfp_mask); sg = alloc_fn(alloc_size, gfp_mask);
}
if (unlikely(!sg)) { if (unlikely(!sg)) {
/* /*
* Adjust entry count to reflect that the last * Adjust entry count to reflect that the last
...@@ -324,9 +333,9 @@ int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) ...@@ -324,9 +333,9 @@ int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask)
int ret; int ret;
ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC,
gfp_mask, sg_kmalloc); NULL, gfp_mask, sg_kmalloc);
if (unlikely(ret)) if (unlikely(ret))
__sg_free_table(table, SG_MAX_SINGLE_ALLOC, sg_kfree); __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree);
return ret; return ret;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment