[PATCH v6 10/18] mtd: rawnand: sunxi: bound DMA batches by the user-data register bank
From: James Hilliard
Date: Mon Sep 14 2026 - 17:17:21 EST
The H6/H616 controller concatenates protected user data in a bank of 32
four-byte registers. The default layout can allocate more than 128 bytes
across a page, even though each ECC step fits its individual length limit.
DMA transfers then access beyond the user-data register bank. For example,
a 16 KiB page with 1280 OOB bytes and BCH40/1024 has 160 user-data bytes,
while a 16 KiB page with 1664 OOB bytes can allocate 512 bytes.
Do not cap that allocation: changing the lengths would move ECC offsets
and make existing pages written through PIO incompatible. Instead, split
DMA transfers while retaining the original logical main-data and OOB
offsets. Fill the first batch up to the register-bank limit. Transfer each
remaining logical step separately using hardware slot and user-data
register zero, repositioning the main column and spare-area base first.
PAGE_OP generates the second and subsequent main-data columns from its
hardware slot index, not relative to the initial column. Read and write
probes on H616 confirm this behavior. A multi-slot batch starting partway
through the page would therefore return to the wrong main-data column.
Single-slot later batches avoid those internally generated column changes.
Pages whose user data already fits retain the existing single-batch path.
Pass the user-data register index directly to the protected-OOB helpers
and program length fields from the batch's logical starting step. Collect
each batch's ECC status and protected OOB before reusing the registers.
Keep ECC accounting and trailing-OOB handling page-wide, remembering
whether any batch found an erased chunk. Discard all partial DMA statistics
before a PIO read retry.
Use one page-read setup or program-begin command for the whole page. Only
issue program-end after every batch and any trailing OOB transfer succeed.
Do not retry a write in PIO after a batch has already been transferred.
The allocator, free-OOB layout and raw-access callbacks are unchanged.
Fixes: 54dcd6aa69db ("mtd: rawnand: sunxi: introduce maximize variable user data length")
Signed-off-by: James Hilliard <james.hilliard1@xxxxxxxxx>
---
drivers/mtd/nand/raw/sunxi_nand.c | 284 ++++++++++++++++++++++----------------
1 file changed, 163 insertions(+), 121 deletions(-)
diff --git a/drivers/mtd/nand/raw/sunxi_nand.c b/drivers/mtd/nand/raw/sunxi_nand.c
index e29638c828ca..18a4d1cc4246 100644
--- a/drivers/mtd/nand/raw/sunxi_nand.c
+++ b/drivers/mtd/nand/raw/sunxi_nand.c
@@ -54,6 +54,7 @@
#define NFC_REG_H6_RDATA_1 0x004C
#define NFC_REG_A10_USER_DATA 0x0050
#define NFC_REG_H6_USER_DATA 0x0080
+#define NFC_H6_USER_DATA_REGS 32
#define NFC_REG_USER_DATA(nfc, x) (nfc->caps->reg_user_data + ((x) * 4))
#define NFC_REG_H6_USER_DATA_LEN 0x0070
/* A USER_DATA_LEN register can hold the length of 8 USER_DATA registers */
@@ -932,41 +933,47 @@ static u8 sunxi_nfc_user_data_sz(struct sunxi_nand_chip *sunxi_nand, int step)
return sunxi_nand->user_data_bytes[step];
}
+/* Keep the on-flash layout, but fit each DMA batch in the user-data bank. */
+static int sunxi_nfc_dma_batch_steps(struct nand_chip *nand,
+ int first_step, int end_step)
+{
+ struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
+ struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
+ unsigned int user_data_sz = 0;
+ int i;
+
+ if (!nfc->caps->reg_user_data_len)
+ return end_step - first_step;
+
+ /*
+ * PAGE_OP generates main-data columns from hardware slot zero, not
+ * from the initial column. Use only slot zero after the first batch,
+ * so the explicit column change selects each remaining logical step.
+ */
+ if (first_step)
+ return 1;
+
+ for (i = first_step; i < end_step; i++) {
+ user_data_sz += sunxi_nfc_user_data_sz(sunxi_nand, i);
+ if (user_data_sz > NFC_H6_USER_DATA_REGS * sizeof(u32))
+ break;
+ }
+
+ return i - first_step;
+}
+
+/* PIO uses register zero; DMA concatenates user data within each batch. */
static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct nand_chip *nand, u8 *oob,
- int step, bool bbm, int page,
+ unsigned int reg_index, bool bbm, int page,
unsigned int user_data_sz)
{
- struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
u32 user_data;
+ unsigned int i;
- if (!nfc->caps->reg_user_data_len) {
- /*
- * For A10, the user data for step n is in the nth
- * REG_USER_DATA
- */
- user_data = readl(nfc->regs + NFC_REG_USER_DATA(nfc, step));
- sunxi_nfc_user_data_to_buf(user_data, oob);
- } else {
- /*
- * For H6 NAND controller, the user data for all steps is
- * contained in 32 user data registers, but not at a specific
- * offset for each step, they are just concatenated.
- */
- unsigned int user_data_off = 0;
- unsigned int reg_off;
- u8 *ptr = oob;
- unsigned int i;
-
- for (i = 0; i < step; i++)
- user_data_off += sunxi_nfc_user_data_sz(sunxi_nand, i);
-
- user_data_off /= 4;
- for (i = 0; i < user_data_sz / 4; i++, ptr += 4) {
- reg_off = NFC_REG_USER_DATA(nfc, user_data_off + i);
- user_data = readl(nfc->regs + reg_off);
- sunxi_nfc_user_data_to_buf(user_data, ptr);
- }
+ for (i = 0; i < user_data_sz / 4; i++) {
+ user_data = readl(nfc->regs + NFC_REG_USER_DATA(nfc, reg_index + i));
+ sunxi_nfc_user_data_to_buf(user_data, oob + i * 4);
}
/* De-randomize the Bad Block Marker. */
@@ -1022,13 +1029,13 @@ static void sunxi_nfc_set_user_data_len(struct sunxi_nfc *nfc,
}
static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand,
- const u8 *oob, int step,
+ const u8 *oob, unsigned int reg_index,
bool bbm, int page,
unsigned int user_data_sz)
{
struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
- struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
u8 user_data[SUNXI_NFC_MAX_USER_DATA_SZ] = {};
+ unsigned int i;
/* Randomize the Bad Block Marker. */
if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
@@ -1037,33 +1044,10 @@ static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand,
oob = user_data;
}
- if (!nfc->caps->reg_user_data_len) {
- /*
- * For A10, the user data for step n is in the nth
- * REG_USER_DATA
- */
- writel(sunxi_nfc_buf_to_user_data(oob),
- nfc->regs + NFC_REG_USER_DATA(nfc, step));
- } else {
- /*
- * For H6 NAND controller, the user data for all steps is
- * contained in 32 user data registers, but not at a specific
- * offset for each step, they are just concatenated.
- */
- unsigned int user_data_off = 0;
- const u8 *ptr = oob;
- unsigned int i;
-
- for (i = 0; i < step; i++)
- user_data_off += sunxi_nfc_user_data_sz(sunxi_nand, i);
-
- user_data_off /= 4;
- for (i = 0; i < user_data_sz / 4; i++, ptr += 4) {
- writel(sunxi_nfc_buf_to_user_data(ptr),
- nfc->regs + NFC_REG_USER_DATA(nfc, user_data_off + i));
- }
+ for (i = 0; i < user_data_sz / 4; i++) {
+ writel(sunxi_nfc_buf_to_user_data(oob + i * 4),
+ nfc->regs + NFC_REG_USER_DATA(nfc, reg_index + i));
}
-
}
static void sunxi_nfc_hw_ecc_update_stats(struct nand_chip *nand,
@@ -1313,27 +1297,35 @@ static int sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand,
return 0;
}
-static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf,
- int oob_required, int page,
- int nchunks)
+static int sunxi_nfc_hw_ecc_read_batch_dma(struct nand_chip *nand, u8 *buf,
+ int oob_required, int page,
+ int first_step, int nchunks,
+ int *raw_mode)
{
bool randomized = nand->options & NAND_NEED_SCRAMBLING;
struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
struct mtd_info *mtd = nand_to_mtd(nand);
struct nand_ecc_ctrl *ecc = &nand->ecc;
- unsigned int corrected = mtd->ecc_stats.corrected;
- unsigned int failed = mtd->ecc_stats.failed;
+ unsigned int reg_index = 0, user_data_sz;
unsigned int max_bitflips = 0;
- int ret, i, raw_mode = 0;
+ int ret, i;
struct scatterlist sg;
u32 status, pattern_found, wait;
+ if (first_step) {
+ ret = sunxi_nfc_read_column(nand, page, first_step * ecc->size,
+ NULL, 0);
+ if (ret)
+ return ret;
+ }
+
ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
if (ret)
return ret;
- ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, nchunks,
+ ret = sunxi_nfc_dma_op_prepare(nfc, buf + first_step * ecc->size,
+ ecc->size, nchunks,
DMA_FROM_DEVICE, &sg);
if (ret)
return ret;
@@ -1341,7 +1333,12 @@ static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf
sunxi_nfc_hw_ecc_enable(nand);
sunxi_nfc_reset_user_data_len(nfc);
for (i = 0; i < nchunks; i++)
- sunxi_nfc_set_user_data_len(nfc, sunxi_nfc_user_data_sz(sunxi_nand, i), i);
+ sunxi_nfc_set_user_data_len(nfc,
+ sunxi_nfc_user_data_sz(sunxi_nand, first_step + i), i);
+ /* exec_op() restores the page's spare base during column changes. */
+ if (first_step)
+ writel(mtd->writesize + sunxi_get_oob_offset(sunxi_nand, ecc, first_step),
+ nfc->regs + NFC_REG_SPARE_AREA(nfc));
sunxi_nfc_randomizer_config(nand, page, false);
sunxi_nfc_randomizer_enable(nand);
@@ -1374,15 +1371,17 @@ static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf
pattern_found = readl(nfc->regs + nfc->caps->reg_pat_found);
pattern_found = field_get(NFC_ECC_PAT_FOUND_MSK(nfc), pattern_found);
- for (i = 0; i < nchunks; i++) {
- int data_off = i * ecc->size;
- unsigned int user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, i);
- int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, i);
+ for (i = 0; i < nchunks; i++, reg_index += user_data_sz / 4) {
+ int logical_step = first_step + i;
+ int data_off = logical_step * ecc->size;
+ int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, logical_step);
u8 *data = buf + data_off;
u8 *oob = nand->oob_poi + oob_off;
bool erased;
int bitflips;
+ user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, logical_step);
+
bitflips = sunxi_nfc_hw_ecc_correct(nand, randomized ? data : NULL,
oob_required ? oob : NULL,
i, status, pattern_found,
@@ -1397,40 +1396,64 @@ static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf
ret = sunxi_nfc_read_column(nand, page, mtd->writesize + oob_off,
oob, ecc->bytes + user_data_sz);
if (ret)
- goto err_stats;
+ return ret;
- sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, i, !i,
+ sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, reg_index, !logical_step,
page, user_data_sz);
}
if (erased)
- raw_mode = 1;
+ *raw_mode = 1;
sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, bitflips);
}
if (status & NFC_ECC_ERR_MSK(nfc)) {
for (i = 0; i < nchunks; i++) {
- int data_off = i * ecc->size;
- unsigned int user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, i);
- int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, i);
+ int logical_step = first_step + i;
+ int data_off = logical_step * ecc->size;
+ int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, logical_step);
u8 *data = buf + data_off;
u8 *oob = nand->oob_poi + oob_off;
if (!(status & NFC_ECC_ERR(i)))
continue;
+ user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, logical_step);
ret = sunxi_nfc_hw_ecc_read_error(nand, data, data_off, oob,
mtd->writesize + oob_off,
user_data_sz,
&max_bitflips, page);
if (ret < 0)
- goto err_stats;
+ return ret;
if (ret)
- raw_mode = 1;
+ *raw_mode = 1;
}
}
+ return max_bitflips;
+}
+
+static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, u8 *buf,
+ int oob_required, int page, int nchunks)
+{
+ struct mtd_info *mtd = nand_to_mtd(nand);
+ unsigned int corrected = mtd->ecc_stats.corrected;
+ unsigned int failed = mtd->ecc_stats.failed;
+ unsigned int max_bitflips = 0;
+ int raw_mode = 0;
+ int first_step, batch_steps, ret;
+
+ for (first_step = 0; first_step < nchunks; first_step += batch_steps) {
+ batch_steps = sunxi_nfc_dma_batch_steps(nand, first_step, nchunks);
+ ret = sunxi_nfc_hw_ecc_read_batch_dma(nand, buf, oob_required, page,
+ first_step, batch_steps,
+ &raw_mode);
+ if (ret < 0)
+ goto err_stats;
+ max_bitflips = max_t(unsigned int, max_bitflips, ret);
+ }
+
if (oob_required) {
ret = sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi,
NULL, !raw_mode, page);
@@ -1761,69 +1784,88 @@ static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *nand,
{
struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
+ struct mtd_info *mtd = nand_to_mtd(nand);
struct nand_ecc_ctrl *ecc = &nand->ecc;
struct scatterlist sg;
u32 wait;
- int ret, i;
+ int first_step, batch_steps, ret, i;
sunxi_nfc_select_chip(nand, nand->cur_cs);
- ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
- if (ret)
- return ret;
+ for (first_step = 0; first_step < ecc->steps; first_step += batch_steps) {
+ unsigned int reg_index = 0;
- ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, ecc->steps,
- DMA_TO_DEVICE, &sg);
- if (ret)
- goto pio_fallback;
+ batch_steps = sunxi_nfc_dma_batch_steps(nand, first_step, ecc->steps);
+ ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
+ if (ret)
+ return ret;
- sunxi_nfc_reset_user_data_len(nfc);
- for (i = 0; i < ecc->steps; i++) {
- unsigned int user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, i);
- int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, i);
- const u8 *oob = nand->oob_poi + oob_off;
+ ret = sunxi_nfc_dma_op_prepare(nfc, buf + first_step * ecc->size,
+ ecc->size, batch_steps, DMA_TO_DEVICE, &sg);
+ if (ret) {
+ /* Only retry before any part of the page has been transferred. */
+ if (first_step)
+ return ret;
+ goto pio_fallback;
+ }
- sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, i, !i, page,
- user_data_sz);
- sunxi_nfc_set_user_data_len(nfc, user_data_sz, i);
- }
+ sunxi_nfc_reset_user_data_len(nfc);
+ for (i = first_step; i < first_step + batch_steps; i++) {
+ unsigned int user_data_sz = sunxi_nfc_user_data_sz(sunxi_nand, i);
+ int oob_off = sunxi_get_oob_offset(sunxi_nand, ecc, i);
+ const u8 *oob = nand->oob_poi + oob_off;
- ret = nand_prog_page_begin_op(nand, page, 0, NULL, 0);
- if (ret) {
- sunxi_nfc_dma_op_abort(nfc);
- sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg);
- return ret;
- }
+ sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, reg_index, !i,
+ page, user_data_sz);
+ sunxi_nfc_set_user_data_len(nfc, user_data_sz, i - first_step);
+ reg_index += user_data_sz / 4;
+ }
- sunxi_nfc_hw_ecc_enable(nand);
- sunxi_nfc_randomizer_config(nand, page, false);
- sunxi_nfc_randomizer_enable(nand);
+ if (first_step)
+ ret = nand_change_write_column_op(nand, first_step * ecc->size,
+ NULL, 0, false);
+ else
+ ret = nand_prog_page_begin_op(nand, page, 0, NULL, 0);
+ if (ret) {
+ sunxi_nfc_dma_op_abort(nfc);
+ sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg);
+ return ret;
+ }
- writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
- nfc->regs + NFC_REG_WCMD_SET);
+ /* exec_op() restores the page's spare base during column changes. */
+ if (first_step)
+ writel(mtd->writesize + sunxi_get_oob_offset(sunxi_nand, ecc, first_step),
+ nfc->regs + NFC_REG_SPARE_AREA(nfc));
+ sunxi_nfc_hw_ecc_enable(nand);
+ sunxi_nfc_randomizer_config(nand, page, false);
+ sunxi_nfc_randomizer_enable(nand);
- wait = NFC_CMD_INT_FLAG;
+ writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
+ nfc->regs + NFC_REG_WCMD_SET);
- if (nfc->use_mdma)
- wait |= NFC_DMA_INT_FLAG;
- else
- dma_async_issue_pending(nfc->dmac);
+ wait = NFC_CMD_INT_FLAG;
- writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
- NFC_DATA_TRANS | NFC_ACCESS_DIR,
- nfc->regs + NFC_REG_CMD);
+ if (nfc->use_mdma)
+ wait |= NFC_DMA_INT_FLAG;
+ else
+ dma_async_issue_pending(nfc->dmac);
- ret = sunxi_nfc_wait_events(nfc, wait, false, 0);
- if (ret)
- sunxi_nfc_dma_op_abort(nfc);
+ writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
+ NFC_DATA_TRANS | NFC_ACCESS_DIR,
+ nfc->regs + NFC_REG_CMD);
- sunxi_nfc_randomizer_disable(nand);
- sunxi_nfc_hw_ecc_disable(nand);
+ ret = sunxi_nfc_wait_events(nfc, wait, false, 0);
+ if (ret)
+ sunxi_nfc_dma_op_abort(nfc);
- sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg);
+ sunxi_nfc_randomizer_disable(nand);
+ sunxi_nfc_hw_ecc_disable(nand);
- if (ret)
- return ret;
+ sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg);
+
+ if (ret)
+ return ret;
+ }
if (oob_required || (nand->options & NAND_NEED_SCRAMBLING)) {
/* TODO: use DMA to transfer extra OOB bytes ? */
--
2.53.0