Re: [PATCH net-next v3 12/18] net/smc: implement DMB-related operations of loopback

[Wen Gu]

On 2023/9/22 07:31, Wenjia Zhang wrote:
>

<...>

> > +static int smc_lo_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb,
> > +                   void *client_priv)
> > +{
> > +    struct smc_lo_dmb_node *dmb_node, *tmp_node;
> > +    struct smc_lo_dev *ldev = smcd->priv;
> > +    int sba_idx, rc;
> > +
> > +    /* check space for new dmb */
> > +    for_each_clear_bit(sba_idx, ldev->sba_idx_mask, SMC_LODEV_MAX_DMBS) {
> > +        if (!test_and_set_bit(sba_idx, ldev->sba_idx_mask))
> > +            break;
> > +    }
> > +    if (sba_idx == SMC_LODEV_MAX_DMBS)
> > +        return -ENOSPC;
> > +
> > +    dmb_node = kzalloc(sizeof(*dmb_node), GFP_KERNEL);
> > +    if (!dmb_node) {
> > +        rc = -ENOMEM;
> > +        goto err_bit;
> > +    }
> > +
> > +    dmb_node->sba_idx = sba_idx;
> > +    dmb_node->cpu_addr = kzalloc(dmb->dmb_len, GFP_KERNEL |
> > +                     __GFP_NOWARN | __GFP_NORETRY |
> > +                     __GFP_NOMEMALLOC);
> kzalloc()/kmalloc() allocates physically contigueous memory. Are you sure it is suitable for allocating the dmb?

Yes, physically contigueous memory is little expensive here. I initially wanted to see the best performance.

I tried using vzalloc here, and the performance dropped a bit (2%~8%) compared to kzalloc. I think it is acceptable.

- ipc-benchmark
                       kzalloc                vzalloc
Message
rate (msg/s)            152076                 145753(-4.16%)

- sockperf
                       kzalloc                vzalloc
Bandwidth(MBps)       8491.638               8002.380(-5.76%)
Latency(us)              3.222                  3.508(+8.88%)

- nginx/wrk
                       kzalloc                vzalloc
Requests/s           272519.36              256490.94(-5.88%)

- redis-benchmark
                       kzalloc                vzalloc
GET(Requests/s)      123304.56              120084.05(-2.61%)
SET(Requests/s)      122062.87              118800.12(-2.67%)


> > +    if (!dmb_node->cpu_addr) {
> > +        rc = -ENOMEM;
> > +        goto err_node;
> > +    }
> > +    dmb_node->len = dmb->dmb_len;
> > +    dmb_node->dma_addr = (dma_addr_t)dmb_node->cpu_addr;
> > +
> > +again:
> > +    /* add new dmb into hash table */
> > +    get_random_bytes(&dmb_node->token, sizeof(dmb_node->token));
> > +    write_lock(&ldev->dmb_ht_lock);
> > +    hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_node->token) {
> > +        if (tmp_node->token == dmb_node->token) {
> > +            write_unlock(&ldev->dmb_ht_lock);
> > +            goto again;
> > +        }
> > +    }
> > +    hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token);
> > +    write_unlock(&ldev->dmb_ht_lock);
> > +
> > +    dmb->sba_idx = dmb_node->sba_idx;
> > +    dmb->dmb_tok = dmb_node->token;
> > +    dmb->cpu_addr = dmb_node->cpu_addr;
> > +    dmb->dma_addr = dmb_node->dma_addr;
> > +    dmb->dmb_len = dmb_node->len;
> > +
> > +    return 0;
> > +
> > +err_node:
> > +    kfree(dmb_node);
> > +err_bit:
> > +    clear_bit(sba_idx, ldev->sba_idx_mask);
> > +    return rc;
> > +}
> > +
> > +static int smc_lo_unregister_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb)
> > +{
> > +    struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node;
> > +    struct smc_lo_dev *ldev = smcd->priv;
> > +
> > +    /* remove dmb from hash table */
> > +    write_lock(&ldev->dmb_ht_lock);
> > +    hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
> > +        if (tmp_node->token == dmb->dmb_tok) {
> > +            dmb_node = tmp_node;
> > +            break;
> > +        }
> > +    }
> > +    if (!dmb_node) {
> > +        write_unlock(&ldev->dmb_ht_lock);
> > +        return -EINVAL;
> > +    }
> > +    hash_del(&dmb_node->list);
> > +    write_unlock(&ldev->dmb_ht_lock);
> > +
> > +    clear_bit(dmb_node->sba_idx, ldev->sba_idx_mask);
> > +    kfree(dmb_node->cpu_addr);
> > +    kfree(dmb_node);
> > +
> > +    return 0;
> > +}
> > +