[PATCH v4 09/10] xen/blkfront: make persistent grants per-queue
From: Bob Liu
Date: Sun Nov 01 2015 - 23:22:53 EST
Make persistent grants per-queue/ring instead of per-device, so that we can
drop the 'dev_lock' and get better scalability.
Signed-off-by: Bob Liu <bob.liu@xxxxxxxxxx>
---
drivers/block/xen-blkfront.c | 89 +++++++++++++++++---------------------------
1 file changed, 34 insertions(+), 55 deletions(-)
diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c
index 23096d7..eb19f08 100644
--- a/drivers/block/xen-blkfront.c
+++ b/drivers/block/xen-blkfront.c
@@ -133,6 +133,8 @@ struct blkfront_ring_info {
struct gnttab_free_callback callback;
struct blk_shadow shadow[BLK_MAX_RING_SIZE];
struct list_head indirect_pages;
+ struct list_head grants;
+ unsigned int persistent_gnts_c;
unsigned long shadow_free;
struct blkfront_info *dev_info;
};
@@ -144,8 +146,6 @@ struct blkfront_ring_info {
*/
struct blkfront_info
{
- /* Lock to proect info->grants list shared by multi rings */
- spinlock_t dev_lock;
struct mutex mutex;
struct xenbus_device *xbdev;
struct gendisk *gd;
@@ -155,8 +155,6 @@ struct blkfront_info
/* Number of pages per ring buffer */
unsigned int nr_ring_pages;
struct request_queue *rq;
- struct list_head grants;
- unsigned int persistent_gnts_c;
unsigned int feature_flush;
unsigned int feature_discard:1;
unsigned int feature_secdiscard:1;
@@ -231,7 +229,6 @@ static int fill_grant_buffer(struct blkfront_ring_info *rinfo, int num)
struct grant *gnt_list_entry, *n;
int i = 0;
- spin_lock_irq(&info->dev_lock);
while(i < num) {
gnt_list_entry = kzalloc(sizeof(struct grant), GFP_NOIO);
if (!gnt_list_entry)
@@ -247,35 +244,32 @@ static int fill_grant_buffer(struct blkfront_ring_info *rinfo, int num)
}
gnt_list_entry->gref = GRANT_INVALID_REF;
- list_add(&gnt_list_entry->node, &info->grants);
+ list_add(&gnt_list_entry->node, &rinfo->grants);
i++;
}
- spin_unlock_irq(&info->dev_lock);
return 0;
out_of_memory:
list_for_each_entry_safe(gnt_list_entry, n,
- &info->grants, node) {
+ &rinfo->grants, node) {
list_del(&gnt_list_entry->node);
if (info->feature_persistent)
__free_page(pfn_to_page(gnt_list_entry->pfn));
kfree(gnt_list_entry);
i--;
}
- spin_unlock_irq(&info->dev_lock);
BUG_ON(i != 0);
return -ENOMEM;
}
static struct grant *get_grant(grant_ref_t *gref_head,
unsigned long pfn,
- struct blkfront_info *info)
+ struct blkfront_ring_info *info)
{
struct grant *gnt_list_entry;
unsigned long buffer_gfn;
- spin_lock(&info->dev_lock);
BUG_ON(list_empty(&info->grants));
gnt_list_entry = list_first_entry(&info->grants, struct grant,
node);
@@ -283,21 +277,19 @@ static struct grant *get_grant(grant_ref_t *gref_head,
if (gnt_list_entry->gref != GRANT_INVALID_REF) {
info->persistent_gnts_c--;
- spin_unlock(&info->dev_lock);
return gnt_list_entry;
}
- spin_unlock(&info->dev_lock);
/* Assign a gref to this page */
gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head);
BUG_ON(gnt_list_entry->gref == -ENOSPC);
- if (!info->feature_persistent) {
+ if (!info->dev_info->feature_persistent) {
BUG_ON(!pfn);
gnt_list_entry->pfn = pfn;
}
buffer_gfn = pfn_to_gfn(gnt_list_entry->pfn);
gnttab_grant_foreign_access_ref(gnt_list_entry->gref,
- info->xbdev->otherend_id,
+ info->dev_info->xbdev->otherend_id,
buffer_gfn, 0);
return gnt_list_entry;
}
@@ -559,13 +551,13 @@ static int blkif_queue_request(struct request *req, struct blkfront_ring_info *r
list_del(&indirect_page->lru);
pfn = page_to_pfn(indirect_page);
}
- gnt_list_entry = get_grant(&gref_head, pfn, info);
+ gnt_list_entry = get_grant(&gref_head, pfn, rinfo);
rinfo->shadow[id].indirect_grants[n] = gnt_list_entry;
segments = kmap_atomic(pfn_to_page(gnt_list_entry->pfn));
ring_req->u.indirect.indirect_grefs[n] = gnt_list_entry->gref;
}
- gnt_list_entry = get_grant(&gref_head, page_to_pfn(sg_page(sg)), info);
+ gnt_list_entry = get_grant(&gref_head, page_to_pfn(sg_page(sg)), rinfo);
ref = gnt_list_entry->gref;
rinfo->shadow[id].grants_used[i] = gnt_list_entry;
@@ -992,7 +984,7 @@ static void blkif_restart_queue(struct work_struct *work)
static void blkif_free_ring(struct blkfront_ring_info *rinfo)
{
- struct grant *persistent_gnt;
+ struct grant *persistent_gnt, *n;
struct blkfront_info *info = rinfo->dev_info;
int i, j, segs;
@@ -1010,6 +1002,23 @@ static void blkif_free_ring(struct blkfront_ring_info *rinfo)
}
}
+ /* Remove all persistent grants */
+ if (!list_empty(&rinfo->grants)) {
+ list_for_each_entry_safe(persistent_gnt, n,
+ &rinfo->grants, node) {
+ list_del(&persistent_gnt->node);
+ if (persistent_gnt->gref != GRANT_INVALID_REF) {
+ gnttab_end_foreign_access(persistent_gnt->gref,
+ 0, 0UL);
+ rinfo->persistent_gnts_c--;
+ }
+ if (info->feature_persistent)
+ __free_page(pfn_to_page(persistent_gnt->pfn));
+ kfree(persistent_gnt);
+ }
+ }
+ BUG_ON(rinfo->persistent_gnts_c != 0);
+
for (i = 0; i < BLK_RING_SIZE(info); i++) {
/*
* Clear persistent grants present in requests already
@@ -1075,37 +1084,17 @@ free_shadow:
static void blkif_free(struct blkfront_info *info, int suspend)
{
- struct grant *persistent_gnt, *n;
int i;
/* Prevent new requests being issued until we fix things up. */
- spin_lock_irq(&info->dev_lock);
info->connected = suspend ?
BLKIF_STATE_SUSPENDED : BLKIF_STATE_DISCONNECTED;
/* No more blkif_request(). */
if (info->rq)
blk_mq_stop_hw_queues(info->rq);
- /* Remove all persistent grants */
- if (!list_empty(&info->grants)) {
- list_for_each_entry_safe(persistent_gnt, n,
- &info->grants, node) {
- list_del(&persistent_gnt->node);
- if (persistent_gnt->gref != GRANT_INVALID_REF) {
- gnttab_end_foreign_access(persistent_gnt->gref,
- 0, 0UL);
- info->persistent_gnts_c--;
- }
- if (info->feature_persistent)
- __free_page(pfn_to_page(persistent_gnt->pfn));
- kfree(persistent_gnt);
- }
- }
- BUG_ON(info->persistent_gnts_c != 0);
-
for (i = 0; i < info->nr_rings; i++)
blkif_free_ring(&info->rinfo[i]);
- spin_unlock_irq(&info->dev_lock);
}
static void blkif_completion(struct blk_shadow *s, struct blkfront_ring_info *rinfo,
@@ -1117,7 +1106,6 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_ring_info *ri
void *shared_data;
int nseg;
struct blkfront_info *info = rinfo->dev_info;
- unsigned long flags;
nseg = s->req.operation == BLKIF_OP_INDIRECT ?
s->req.u.indirect.nr_segments : s->req.u.rw.nr_segments;
@@ -1148,10 +1136,8 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_ring_info *ri
if (!info->feature_persistent)
pr_alert_ratelimited("backed has not unmapped grant: %u\n",
s->grants_used[i]->gref);
- spin_lock_irqsave(&info->dev_lock, flags);
- list_add(&s->grants_used[i]->node, &info->grants);
- info->persistent_gnts_c++;
- spin_unlock_irqrestore(&info->dev_lock, flags);
+ list_add(&s->grants_used[i]->node, &rinfo->grants);
+ rinfo->persistent_gnts_c++;
} else {
/*
* If the grant is not mapped by the backend we end the
@@ -1161,9 +1147,7 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_ring_info *ri
*/
gnttab_end_foreign_access(s->grants_used[i]->gref, 0, 0UL);
s->grants_used[i]->gref = GRANT_INVALID_REF;
- spin_lock_irqsave(&info->dev_lock, flags);
- list_add_tail(&s->grants_used[i]->node, &info->grants);
- spin_unlock_irqrestore(&info->dev_lock, flags);
+ list_add_tail(&s->grants_used[i]->node, &rinfo->grants);
}
}
if (s->req.operation == BLKIF_OP_INDIRECT) {
@@ -1172,10 +1156,8 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_ring_info *ri
if (!info->feature_persistent)
pr_alert_ratelimited("backed has not unmapped grant: %u\n",
s->indirect_grants[i]->gref);
- spin_lock_irqsave(&info->dev_lock, flags);
- list_add(&s->indirect_grants[i]->node, &info->grants);
- info->persistent_gnts_c++;
- spin_unlock_irqrestore(&info->dev_lock, flags);
+ list_add(&s->indirect_grants[i]->node, &rinfo->grants);
+ rinfo->persistent_gnts_c++;
} else {
struct page *indirect_page;
@@ -1184,14 +1166,12 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_ring_info *ri
* Add the used indirect page back to the list of
* available pages for indirect grefs.
*/
- spin_lock_irqsave(&info->dev_lock, flags);
if (!info->feature_persistent) {
indirect_page = pfn_to_page(s->indirect_grants[i]->pfn);
list_add(&indirect_page->lru, &rinfo->indirect_pages);
}
s->indirect_grants[i]->gref = GRANT_INVALID_REF;
- list_add_tail(&s->indirect_grants[i]->node, &info->grants);
- spin_unlock_irqrestore(&info->dev_lock, flags);
+ list_add_tail(&s->indirect_grants[i]->node, &rinfo->grants);
}
}
}
@@ -1587,10 +1567,8 @@ static int blkfront_probe(struct xenbus_device *dev,
}
mutex_init(&info->mutex);
- spin_lock_init(&info->dev_lock);
info->xbdev = dev;
info->vdevice = vdevice;
- INIT_LIST_HEAD(&info->grants);
info->connected = BLKIF_STATE_DISCONNECTED;
/* Check if backend supports multiple queues */
@@ -1616,6 +1594,7 @@ static int blkfront_probe(struct xenbus_device *dev,
struct blkfront_ring_info *rinfo;
rinfo = &info->rinfo[r_index];
+ INIT_LIST_HEAD(&rinfo->grants);
INIT_LIST_HEAD(&rinfo->indirect_pages);
rinfo->dev_info = info;
INIT_WORK(&rinfo->work, blkif_restart_queue);
--
1.8.3.1
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/