Re: [PATCH v7 06/10] PCI: endpoint: pci-epf-vntb: Allow DMA and MW to share a BAR

From: Koichiro Den

Date: Wed Oct 21 2026 - 12:35:32 EST

On Wed, Oct 21, 2026 at 10:52:32AM -0500, Frank Li wrote:
> On Wed, Oct 21, 2026 at 01:22:25PM +0900, Koichiro Den wrote:
> > Some endpoint configurations have no spare BAR for DMA resources. Allow
> > dma_bar to select an MW BAR and place the DMA ranges after that MW.
> >
> > DMA submaps have fixed BAR offsets, so a shared MW translation must
> > cover its configured size. Report that size as the translation alignment
> > and require it to be a power of two.
> >
> > Older hosts treat the whole BAR as an MW. Require the host to advertise
> > MW/DMA boundary support in LINK_UP before enabling a shared-BAR link.
> >
> > Signed-off-by: Koichiro Den <den@xxxxxxxxxxxxx>
> > ---
> > Changes since v6:
> > - Split out the host-side changes. (Frank)
> > - Require MW/DMA BAR boundary support in LINK_UP for shared BARs.
> >
> > drivers/pci/endpoint/functions/pci-epf-vntb.c | 154 ++++++++++++++----
> > include/linux/pci-epf-ntb.h | 3 +
> > 2 files changed, 122 insertions(+), 35 deletions(-)
> >
> > diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c
> > index ac83af54122f..721cd841a1dd 100644
> > --- a/drivers/pci/endpoint/functions/pci-epf-vntb.c
> > +++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c
> > @@ -84,12 +84,13 @@ enum epf_irq_slot {
> > #define MIN_DB_COUNT (EPF_IRQ_DB_START + 1)
> >
> > struct epf_ntb_dma {
> > + struct mutex lock; /* Serialize DMA submap and MW updates */
> > struct epf_ntb_dma_ctrl ctrl;
> > struct dma_chan *dchan[EDMA_MAX_RD_CH];
> > void *bar_scratch;
> > dma_addr_t bar_scratch_phys;
> > size_t bar_scratch_size;
> > - struct pci_epf_bar_submap submap[EDMA_MAX_RD_CH + 2];
> > + struct pci_epf_bar_submap submap[EDMA_MAX_RD_CH + 3];
> > struct pci_epf_bar_submap *reg_submap;
> > unsigned int num_submap;
> > u16 rd_ch_cnt;
> > @@ -229,6 +230,22 @@ static bool epf_ntb_is_bar_used(struct epf_ntb *ntb,
> > return false;
> > }
> >
> > +static u64 epf_ntb_dma_bar_offset(struct epf_ntb *ntb, enum pci_barno barno)
> > +{
> > + unsigned int i;
> > +
> > + for (i = 0; i < ntb->num_mws; i++)
> > + if (ntb->epf_ntb_bar[BAR_MW1 + i] == barno)
> > + return ntb->mws_size[i];
> > +
> > + return 0;
> > +}
> > +
> > +static bool epf_ntb_dma_shares_bar(struct epf_ntb *ntb, enum pci_barno barno)
> > +{
> > + return ntb->dma && ntb->epf_ntb_bar[BAR_DMA] == barno;
> > +}
> > +
> > struct epf_ntb_dma_filter {
> > struct device *dev;
> > int chan_id;
> > @@ -353,6 +370,7 @@ static int epf_ntb_dw_edma_collect(struct epf_ntb *ntb,
> > unsigned int i;
> > size_t align;
> > u32 next = 0;
> > + u64 offset;
> > int ret;
> >
> > if (ctrl->u.dma_ctrl.reg_layout_data != EDMA_MF_EDMA_UNROLL)
> > @@ -371,9 +389,30 @@ static int epf_ntb_dw_edma_collect(struct epf_ntb *ntb,
> > if (!is_power_of_2(align))
> > return -EINVAL;
> >
> > - if (epf_ntb_is_bar_used(ntb, barno) ||
> > - pci_epc_get_next_free_bar(features, barno) != barno)
> > + if (barno == ntb->epf_ntb_bar[BAR_CONFIG] ||
> > + barno == ntb->epf_ntb_bar[BAR_DB])
> > + return -EINVAL;
> > +
> > + offset = epf_ntb_dma_bar_offset(ntb, barno);
> > + if (!offset &&
> > + (epf_ntb_is_bar_used(ntb, barno) ||
> > + pci_epc_get_next_free_bar(features, barno) != barno))
> > + return -EINVAL;
> > + if (offset > U32_MAX)
> > + return -EOVERFLOW;
> > + if (offset && !is_power_of_2(offset))
> > return -EINVAL;
> > + dma->ctrl.submap.offset = offset;
> > + next = offset;
> > + if (next) {
> > + /*
> > + * submap[0] covers the MW prefix. Keep its temporary
> > + * target at address 0 until the MW translation is
> > + * installed.
> > + */
> > + dma->submap[0].size = next;
> > + dma->num_submap = 1;
> > + }
> >
> > dma->ctrl.magic = EPF_NTB_DMA_MAGIC;
> > dma->ctrl.revision = EPF_NTB_DMA_REVISION;
> > @@ -408,7 +447,7 @@ static int epf_ntb_dw_edma_collect(struct epf_ntb *ntb,
> > chan->desc_addr_lo = lower_32_bits(dma_addr);
> > chan->desc_addr_hi = upper_32_bits(dma_addr);
> > }
> > - dma->ctrl.submap.size = next;
> > + dma->ctrl.submap.size = next - dma->ctrl.submap.offset;
> >
> > dma_dev = ntb->epf->epc->dev.parent;
> > for (i = 0; i < dma->rd_ch_cnt; i++) {
> > @@ -481,6 +520,8 @@ static int epf_ntb_dma_collect(struct epf_ntb *ntb)
> > if (!dma)
> > return -ENOMEM;
> >
> > + mutex_init(&dma->lock);
> > +
> > switch (ctrl->u.dma_ctrl.reg_layout) {
> > case PCI_EPC_AUX_DMA_REG_LAYOUT_DW_EDMA:
> > ret = epf_ntb_dw_edma_collect(ntb, dma, ctrl, resources, count);
> > @@ -551,7 +592,7 @@ static int epf_ntb_dma_bar_init(struct epf_ntb *ntb)
> > return -EOPNOTSUPP;
> >
> > barno = ntb->epf_ntb_bar[BAR_DMA];
> > - mapped_size = dma->ctrl.submap.size;
> > + mapped_size = dma->ctrl.submap.offset + dma->ctrl.submap.size;
> > /*
> > * Submaps cannot be installed until the host assigns the BAR address.
> > * Use address 0 for the temporary BAR Match Mode mapping, as is done
> > @@ -607,21 +648,25 @@ static void epf_ntb_dma_bar_clear(struct epf_ntb *ntb)
> > memset(bar, 0, sizeof(*bar));
> > }
> >
> > -static int epf_ntb_dma_set_bar(struct epf_ntb *ntb)
> > +static int epf_ntb_dma_set_bar_locked(struct epf_ntb *ntb,
> > + const dma_addr_t *mw_addr)
> > {
> > struct pci_epf_bar_submap *old_submap;
> > struct epf_ntb_dma *dma = ntb->dma;
> > unsigned int old_num_submap;
> > struct pci_epf_bar *bar;
> > + dma_addr_t old_mw_addr;
> > int restore, ret;
> > -
> > - if (!dma)
> > - return 0;
> > + lockdep_assert_held(&dma->lock);
> >
> > bar = &ntb->epf->bar[ntb->epf_ntb_bar[BAR_DMA]];
> >
> > old_submap = bar->submap;
> > old_num_submap = bar->num_submap;
> > + if (mw_addr) {
> > + old_mw_addr = dma->submap[0].phys_addr;
> > + dma->submap[0].phys_addr = *mw_addr;
> > + }
> > bar->submap = dma->submap;
> > bar->num_submap = dma->num_submap;
> >
> > @@ -631,6 +676,8 @@ static int epf_ntb_dma_set_bar(struct epf_ntb *ntb)
> > return 0;
> >
> > /* A failed dynamic update may have already removed the old mapping. */
> > + if (mw_addr)
> > + dma->submap[0].phys_addr = old_mw_addr;
> > bar->submap = old_submap;
> > bar->num_submap = old_num_submap;
> > restore = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no,
> > @@ -642,6 +689,18 @@ static int epf_ntb_dma_set_bar(struct epf_ntb *ntb)
> > return ret;
> > }
> >
> > +static int epf_ntb_dma_set_bar(struct epf_ntb *ntb)
> > +{
> > + struct epf_ntb_dma *dma = ntb->dma;
> > +
> > + if (!dma)
> > + return 0;
> > +
> > + guard(mutex)(&dma->lock);
> > +
> > + return epf_ntb_dma_set_bar_locked(ntb, NULL);
> > +}
> > +
> > /**
> > * epf_ntb_configure_mw() - Configure the Outbound Address Space for VHOST
> > * to access the memory window of HOST
> > @@ -780,6 +839,13 @@ static void epf_ntb_cmd_handler(struct work_struct *work)
> > EPF_NTB_COMMAND_STATUS_OK;
> > break;
> > case EPF_NTB_COMMAND_LINK_UP:
> > + if (ntb->dma && ntb->dma->ctrl.submap.offset &&
> > + !(argument & EPF_NTB_LINK_UP_DMA_MW_SUPPORTED)) {
> > + epf_ntb_set_link_enabled(ntb, false);
> > + dev_err(dev, "Host does not support MW/DMA BAR sharing\n");
> > + ctrl->command_status = EPF_NTB_COMMAND_STATUS_ERROR;
>
> suppose it should be fallback to old method?
> EPF_NTB_LINK_UP_DMA_MW_SUPPORTED is new feature.

It's possible (by using the whole BAR as an MW), but I personally think the
extra complexity would not really be worthwhile for this case.

If a "New" endpoint is paired with an "Old" host whose kernel cannot be
upgraded, the DMA feature cannot be used anyway. For that setup, I think
disabling DMA export with dma_bar = -1 (the default) and rebinding the endpoint
would be sufficient, with host re-enumeration as needed.

I can respin to add automatic fallback if you think it is necessary. WDYT?

Thanks for the review.
Koichiro

>
> Frank
>
> > + break;
> > + }
> > ret = epf_ntb_set_link_enabled(ntb, true);
> > if (ret < 0)
> > ctrl->command_status = EPF_NTB_COMMAND_STATUS_ERROR;
> > @@ -1269,6 +1335,7 @@ static void epf_ntb_db_bar_clear(struct epf_ntb *ntb)
> > */
> > static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
> > {
> > + bool shared;
> > int ret = 0;
> > int i;
> > u64 size;
> > @@ -1278,22 +1345,25 @@ static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
> > for (i = 0; i < ntb->num_mws; i++) {
> > size = ntb->mws_size[i];
> > barno = ntb->epf_ntb_bar[BAR_MW1 + i];
> > -
> > - ntb->epf->bar[barno].barno = barno;
> > - ntb->epf->bar[barno].size = size;
> > - ntb->epf->bar[barno].addr = NULL;
> > - ntb->epf->bar[barno].phys_addr = 0;
> > - ntb->epf->bar[barno].flags |= upper_32_bits(size) ?
> > - PCI_BASE_ADDRESS_MEM_TYPE_64 :
> > - PCI_BASE_ADDRESS_MEM_TYPE_32;
> > -
> > - ret = pci_epc_set_bar(ntb->epf->epc,
> > - ntb->epf->func_no,
> > - ntb->epf->vfunc_no,
> > - &ntb->epf->bar[barno]);
> > - if (ret) {
> > - dev_err(dev, "MW set failed\n");
> > - goto err_alloc_mem;
> > + shared = epf_ntb_dma_shares_bar(ntb, barno);
> > +
> > + if (!shared) {
> > + ntb->epf->bar[barno].barno = barno;
> > + ntb->epf->bar[barno].size = size;
> > + ntb->epf->bar[barno].addr = NULL;
> > + ntb->epf->bar[barno].phys_addr = 0;
> > + ntb->epf->bar[barno].flags |= upper_32_bits(size) ?
> > + PCI_BASE_ADDRESS_MEM_TYPE_64 :
> > + PCI_BASE_ADDRESS_MEM_TYPE_32;
> > +
> > + ret = pci_epc_set_bar(ntb->epf->epc,
> > + ntb->epf->func_no,
> > + ntb->epf->vfunc_no,
> > + &ntb->epf->bar[barno]);
> > + if (ret) {
> > + dev_err(dev, "MW set failed\n");
> > + goto err_alloc_mem;
> > + }
> > }
> >
> > /* Allocate EPC outbound memory windows to vpci vntb device */
> > @@ -1310,10 +1380,11 @@ static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
> > return ret;
> >
> > err_set_bar:
> > - pci_epc_clear_bar(ntb->epf->epc,
> > - ntb->epf->func_no,
> > - ntb->epf->vfunc_no,
> > - &ntb->epf->bar[barno]);
> > + if (!shared)
> > + pci_epc_clear_bar(ntb->epf->epc,
> > + ntb->epf->func_no,
> > + ntb->epf->vfunc_no,
> > + &ntb->epf->bar[barno]);
> > err_alloc_mem:
> > epf_ntb_mw_bar_clear(ntb, i);
> > return ret;
> > @@ -1331,10 +1402,11 @@ static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws)
> >
> > for (i = 0; i < num_mws; i++) {
> > barno = ntb->epf_ntb_bar[BAR_MW1 + i];
> > - pci_epc_clear_bar(ntb->epf->epc,
> > - ntb->epf->func_no,
> > - ntb->epf->vfunc_no,
> > - &ntb->epf->bar[barno]);
> > + if (!epf_ntb_dma_shares_bar(ntb, barno))
> > + pci_epc_clear_bar(ntb->epf->epc,
> > + ntb->epf->func_no,
> > + ntb->epf->vfunc_no,
> > + &ntb->epf->bar[barno]);
> >
> > pci_epc_mem_free_addr(ntb->epf->epc,
> > ntb->vpci_mw_phy[i],
> > @@ -1936,6 +2008,16 @@ static int vntb_epf_mw_set_trans(struct ntb_dev *ndev, int pidx, int idx,
> > dev = &ndev->dev;
> > barno = ntb->epf_ntb_bar[BAR_MW1 + idx];
> > epf_bar = &ntb->epf->bar[barno];
> > + if (epf_ntb_dma_shares_bar(ntb, barno)) {
> > + /* DMA submaps start after the configured MW size. */
> > + if (size != ntb->mws_size[idx])
> > + return -EINVAL;
> > +
> > + guard(mutex)(&ntb->dma->lock);
> > +
> > + return epf_ntb_dma_set_bar_locked(ntb, &addr);
> > + }
> > +
> > epf_bar->phys_addr = addr;
> > epf_bar->barno = barno;
> > epf_bar->size = size;
> > @@ -2134,15 +2216,17 @@ static int vntb_epf_mw_get_align(struct ntb_dev *ndev, int pidx, int idx,
> > resource_size_t *size_max)
> > {
> > struct epf_ntb *ntb = ntb_ndev(ndev);
> > + enum pci_barno barno = ntb->epf_ntb_bar[BAR_MW1 + idx];
> > + u64 size = ntb->mws_size[idx];
> >
> > if (addr_align)
> > *addr_align = SZ_4K;
> >
> > if (size_align)
> > - *size_align = 1;
> > + *size_align = epf_ntb_dma_shares_bar(ntb, barno) ? size : 1;
> >
> > if (size_max)
> > - *size_max = ntb->mws_size[idx];
> > + *size_max = size;
> >
> > return 0;
> > }
> > diff --git a/include/linux/pci-epf-ntb.h b/include/linux/pci-epf-ntb.h
> > index 5cc0f0badae0..8131ed4c6ac9 100644
> > --- a/include/linux/pci-epf-ntb.h
> > +++ b/include/linux/pci-epf-ntb.h
> > @@ -24,6 +24,9 @@
> > #define EPF_NTB_DB_COUNT_MASK GENMASK(15, 0)
> > #define EPF_NTB_MSIX_ENABLE BIT(16)
> >
> > +/* LINK_UP argument: the host honors the MW/DMA BAR boundary. */
> > +#define EPF_NTB_LINK_UP_DMA_MW_SUPPORTED BIT(0)
> > +
> > #define EPF_NTB_COMMAND_STATUS_OK 1
> > #define EPF_NTB_COMMAND_STATUS_ERROR 2
> >
> > --
> > 2.54.0
> >