Re: [PATCH v3 01/11] powerpc/pseries/iommu: Replace hard-coded page shift
From: Leonardo Bras
Date: Fri Apr 30 2021 - 11:58:14 EST
Thanks Alexey!
On Fri, 2021-04-23 at 17:27 +1000, Alexey Kardashevskiy wrote:
>
> On 22/04/2021 17:07, Leonardo Bras wrote:
> > Some functions assume IOMMU page size can only be 4K (pageshift == 12).
> > Update them to accept any page size passed, so we can use 64K pages.
> >
> > In the process, some defines like TCE_SHIFT were made obsolete, and then
> > removed.
> >
> > IODA3 Revision 3.0_prd1 (OpenPowerFoundation), Figures 3.4 and 3.5 show
> > a RPN of 52-bit, and considers a 12-bit pageshift, so there should be
> > no need of using TCE_RPN_MASK, which masks out any bit after 40 in rpn.
> > It's usage removed from tce_build_pSeries(), tce_build_pSeriesLP(), and
> > tce_buildmulti_pSeriesLP().
>
>
> After rereading the patch, I wonder why we had this TCE_RPN_MASK at all
> but what is certain is that this has nothing to do with IODA3 as these
> TCEs are guest phys addresses in pseries and IODA3 is bare metal. Except...
>
>
> > Most places had a tbl struct, so using tbl->it_page_shift was simple.
> > tce_free_pSeriesLP() was a special case, since callers not always have a
> > tbl struct, so adding a tceshift parameter seems the right thing to do.
> >
> > Signed-off-by: Leonardo Bras <leobras.c@xxxxxxxxx>
> > Reviewed-by: Alexey Kardashevskiy <aik@xxxxxxxxx>
> > ---
> > arch/powerpc/include/asm/tce.h | 8 ------
> > arch/powerpc/platforms/pseries/iommu.c | 39 +++++++++++++++-----------
> > 2 files changed, 23 insertions(+), 24 deletions(-)
> >
> > diff --git a/arch/powerpc/include/asm/tce.h b/arch/powerpc/include/asm/tce.h
> > index db5fc2f2262d..0c34d2756d92 100644
> > --- a/arch/powerpc/include/asm/tce.h
> > +++ b/arch/powerpc/include/asm/tce.h
> > @@ -19,15 +19,7 @@
> > #define TCE_VB 0
> > #define TCE_PCI 1
> >
> >
> > -/* TCE page size is 4096 bytes (1 << 12) */
> > -
> > -#define TCE_SHIFT 12
> > -#define TCE_PAGE_SIZE (1 << TCE_SHIFT)
> > -
> > #define TCE_ENTRY_SIZE 8 /* each TCE is 64 bits */
> > -
> > -#define TCE_RPN_MASK 0xfffffffffful /* 40-bit RPN (4K pages) */
> > -#define TCE_RPN_SHIFT 12
> > #define TCE_VALID 0x800 /* TCE valid */
> > #define TCE_ALLIO 0x400 /* TCE valid for all lpars */
> > #define TCE_PCI_WRITE 0x2 /* write from PCI allowed */
> > diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c
> > index 67c9953a6503..796ab356341c 100644
> > --- a/arch/powerpc/platforms/pseries/iommu.c
> > +++ b/arch/powerpc/platforms/pseries/iommu.c
> > @@ -107,6 +107,8 @@ static int tce_build_pSeries(struct iommu_table *tbl, long index,
> > u64 proto_tce;
> > __be64 *tcep;
> > u64 rpn;
> > + const unsigned long tceshift = tbl->it_page_shift;
> > + const unsigned long pagesize = IOMMU_PAGE_SIZE(tbl);
>
> (nit: only used once)
>
> >
> >
> > proto_tce = TCE_PCI_READ; // Read allowed
> >
> >
> > @@ -117,10 +119,10 @@ static int tce_build_pSeries(struct iommu_table *tbl, long index,
>
>
> ... this pseries which is not pseriesLP, i.e. no LPAR == bare metal
> pseries such as ancient power5 or cellbe (I guess) and for those
> TCE_RPN_MASK may actually make sense, keep it.
>
> The rest of the patch looks good. Thanks,
>
>
> >
> >
> > while (npages--) {
> > /* can't move this out since we might cross MEMBLOCK boundary */
> > - rpn = __pa(uaddr) >> TCE_SHIFT;
> > - *tcep = cpu_to_be64(proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT);
> > + rpn = __pa(uaddr) >> tceshift;
> > + *tcep = cpu_to_be64(proto_tce | rpn << tceshift);
> >
> >
> > - uaddr += TCE_PAGE_SIZE;
> > + uaddr += pagesize;
> > tcep++;
> > }
> > return 0;
> > @@ -146,7 +148,7 @@ static unsigned long tce_get_pseries(struct iommu_table *tbl, long index)
> > return be64_to_cpu(*tcep);
> > }
> >
> >
> > -static void tce_free_pSeriesLP(unsigned long liobn, long, long);
> > +static void tce_free_pSeriesLP(unsigned long liobn, long, long, long);
> > static void tce_freemulti_pSeriesLP(struct iommu_table*, long, long);
> >
> >
> > static int tce_build_pSeriesLP(unsigned long liobn, long tcenum, long tceshift,
> > @@ -166,12 +168,12 @@ static int tce_build_pSeriesLP(unsigned long liobn, long tcenum, long tceshift,
> > proto_tce |= TCE_PCI_WRITE;
> >
> >
> > while (npages--) {
> > - tce = proto_tce | (rpn & TCE_RPN_MASK) << tceshift;
> > + tce = proto_tce | rpn << tceshift;
> > rc = plpar_tce_put((u64)liobn, (u64)tcenum << tceshift, tce);
> >
> >
> > if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
> > ret = (int)rc;
> > - tce_free_pSeriesLP(liobn, tcenum_start,
> > + tce_free_pSeriesLP(liobn, tcenum_start, tceshift,
> > (npages_start - (npages + 1)));
> > break;
> > }
> > @@ -205,10 +207,11 @@ static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
> > long tcenum_start = tcenum, npages_start = npages;
> > int ret = 0;
> > unsigned long flags;
> > + const unsigned long tceshift = tbl->it_page_shift;
> >
> >
> > if ((npages == 1) || !firmware_has_feature(FW_FEATURE_PUT_TCE_IND)) {
> > return tce_build_pSeriesLP(tbl->it_index, tcenum,
> > - tbl->it_page_shift, npages, uaddr,
> > + tceshift, npages, uaddr,
> > direction, attrs);
> > }
> >
> >
> > @@ -225,13 +228,13 @@ static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
> > if (!tcep) {
> > local_irq_restore(flags);
> > return tce_build_pSeriesLP(tbl->it_index, tcenum,
> > - tbl->it_page_shift,
> > + tceshift,
> > npages, uaddr, direction, attrs);
> > }
> > __this_cpu_write(tce_page, tcep);
> > }
> >
> >
> > - rpn = __pa(uaddr) >> TCE_SHIFT;
> > + rpn = __pa(uaddr) >> tceshift;
> > proto_tce = TCE_PCI_READ;
> > if (direction != DMA_TO_DEVICE)
> > proto_tce |= TCE_PCI_WRITE;
> > @@ -245,12 +248,12 @@ static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
> > limit = min_t(long, npages, 4096/TCE_ENTRY_SIZE);
> >
> >
> > for (l = 0; l < limit; l++) {
> > - tcep[l] = cpu_to_be64(proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT);
> > + tcep[l] = cpu_to_be64(proto_tce | rpn << tceshift);
> > rpn++;
> > }
> >
> >
> > rc = plpar_tce_put_indirect((u64)tbl->it_index,
> > - (u64)tcenum << 12,
> > + (u64)tcenum << tceshift,
> > (u64)__pa(tcep),
> > limit);
> >
> >
> > @@ -277,12 +280,13 @@ static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
> > return ret;
> > }
> >
> >
> > -static void tce_free_pSeriesLP(unsigned long liobn, long tcenum, long npages)
> > +static void tce_free_pSeriesLP(unsigned long liobn, long tcenum, long tceshift,
> > + long npages)
> > {
> > u64 rc;
> >
> >
> > while (npages--) {
> > - rc = plpar_tce_put((u64)liobn, (u64)tcenum << 12, 0);
> > + rc = plpar_tce_put((u64)liobn, (u64)tcenum << tceshift, 0);
> >
> >
> > if (rc && printk_ratelimit()) {
> > printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
> > @@ -301,9 +305,11 @@ static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long n
> > u64 rc;
> >
> >
> > if (!firmware_has_feature(FW_FEATURE_STUFF_TCE))
> > - return tce_free_pSeriesLP(tbl->it_index, tcenum, npages);
> > + return tce_free_pSeriesLP(tbl->it_index, tcenum,
> > + tbl->it_page_shift, npages);
> >
> >
> > - rc = plpar_tce_stuff((u64)tbl->it_index, (u64)tcenum << 12, 0, npages);
> > + rc = plpar_tce_stuff((u64)tbl->it_index,
> > + (u64)tcenum << tbl->it_page_shift, 0, npages);
> >
> >
> > if (rc && printk_ratelimit()) {
> > printk("tce_freemulti_pSeriesLP: plpar_tce_stuff failed\n");
> > @@ -319,7 +325,8 @@ static unsigned long tce_get_pSeriesLP(struct iommu_table *tbl, long tcenum)
> > u64 rc;
> > unsigned long tce_ret;
> >
> >
> > - rc = plpar_tce_get((u64)tbl->it_index, (u64)tcenum << 12, &tce_ret);
> > + rc = plpar_tce_get((u64)tbl->it_index,
> > + (u64)tcenum << tbl->it_page_shift, &tce_ret);
> >
> >
> > if (rc && printk_ratelimit()) {
> > printk("tce_get_pSeriesLP: plpar_tce_get failed. rc=%lld\n", rc);
> >
>