Hi Roger On Tue, Oct 11, 2022 at 1:50 PM Roger Quadros <rog...@kernel.org> wrote: > > The BCH detection hardware can generate ECC bytes for multiple > sectors in one go. Use that feature. > > correct() only corrects one sector at a time so we need to call it > repeatedly for each sector. >
Reviewed-by: Michael Trimarchi <mich...@amarulasolutions.com> > Signed-off-by: Roger Quadros <rog...@kernel.org> > --- > drivers/mtd/nand/raw/omap_gpmc.c | 325 +++++++++++++++++++++---------- > 1 file changed, 223 insertions(+), 102 deletions(-) > > diff --git a/drivers/mtd/nand/raw/omap_gpmc.c > b/drivers/mtd/nand/raw/omap_gpmc.c > index b36fe762b3..b5ad66ad49 100644 > --- a/drivers/mtd/nand/raw/omap_gpmc.c > +++ b/drivers/mtd/nand/raw/omap_gpmc.c > @@ -27,6 +27,9 @@ > > #define BADBLOCK_MARKER_LENGTH 2 > #define SECTOR_BYTES 512 > +#define ECCSIZE0_SHIFT 12 > +#define ECCSIZE1_SHIFT 22 > +#define ECC1RESULTSIZE 0x1 > #define ECCCLEAR (0x1 << 8) > #define ECCRESULTREG1 (0x1 << 0) > /* 4 bit padding to make byte aligned, 56 = 52 + 4 */ > @@ -187,72 +190,35 @@ static int __maybe_unused omap_correct_data(struct > mtd_info *mtd, uint8_t *dat, > __maybe_unused > static void omap_enable_hwecc(struct mtd_info *mtd, int32_t mode) > { > - struct nand_chip *nand = mtd_to_nand(mtd); > - struct omap_nand_info *info = nand_get_controller_data(nand); > + struct nand_chip *nand = mtd_to_nand(mtd); > + struct omap_nand_info *info = nand_get_controller_data(nand); > unsigned int dev_width = (nand->options & NAND_BUSWIDTH_16) ? 1 : 0; > - unsigned int ecc_algo = 0; > - unsigned int bch_type = 0; > - unsigned int eccsize1 = 0x00, eccsize0 = 0x00, bch_wrapmode = 0x00; > - u32 ecc_size_config_val = 0; > - u32 ecc_config_val = 0; > - int cs = info->cs; > + u32 val; > > - /* configure GPMC for specific ecc-scheme */ > - switch (info->ecc_scheme) { > - case OMAP_ECC_HAM1_CODE_SW: > - return; > - case OMAP_ECC_HAM1_CODE_HW: > - ecc_algo = 0x0; > - bch_type = 0x0; > - bch_wrapmode = 0x00; > - eccsize0 = 0xFF; > - eccsize1 = 0xFF; > + /* Clear ecc and enable bits */ > + writel(ECCCLEAR | ECCRESULTREG1, &gpmc_cfg->ecc_control); > + > + /* program ecc and result sizes */ > + val = ((((nand->ecc.size >> 1) - 1) << ECCSIZE1_SHIFT) | > + ECC1RESULTSIZE); > + writel(val, &gpmc_cfg->ecc_size_config); > + > + switch (mode) { > + case NAND_ECC_READ: > + case NAND_ECC_WRITE: > + writel(ECCCLEAR | ECCRESULTREG1, &gpmc_cfg->ecc_control); > break; > - case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: > - case OMAP_ECC_BCH8_CODE_HW: > - ecc_algo = 0x1; > - bch_type = 0x1; > - if (mode == NAND_ECC_WRITE) { > - bch_wrapmode = 0x01; > - eccsize0 = 0; /* extra bits in nibbles per sector */ > - eccsize1 = 28; /* OOB bits in nibbles per sector */ > - } else { > - bch_wrapmode = 0x01; > - eccsize0 = 26; /* ECC bits in nibbles per sector */ > - eccsize1 = 2; /* non-ECC bits in nibbles per sector > */ > - } > - break; > - case OMAP_ECC_BCH16_CODE_HW: > - ecc_algo = 0x1; > - bch_type = 0x2; > - if (mode == NAND_ECC_WRITE) { > - bch_wrapmode = 0x01; > - eccsize0 = 0; /* extra bits in nibbles per sector */ > - eccsize1 = 52; /* OOB bits in nibbles per sector */ > - } else { > - bch_wrapmode = 0x01; > - eccsize0 = 52; /* ECC bits in nibbles per sector */ > - eccsize1 = 0; /* non-ECC bits in nibbles per sector > */ > - } > + case NAND_ECC_READSYN: > + writel(ECCCLEAR, &gpmc_cfg->ecc_control); > break; > default: > - return; > + printf("%s: error: unrecognized Mode[%d]!\n", __func__, mode); > + break; > } > - /* Clear ecc and enable bits */ > - writel(ECCCLEAR | ECCRESULTREG1, &gpmc_cfg->ecc_control); > - /* Configure ecc size for BCH */ > - ecc_size_config_val = (eccsize1 << 22) | (eccsize0 << 12); > - writel(ecc_size_config_val, &gpmc_cfg->ecc_size_config); > - > - /* Configure device details for BCH engine */ > - ecc_config_val = ((ecc_algo << 16) | /* HAM1 | BCHx */ > - (bch_type << 12) | /* BCH4/BCH8/BCH16 */ > - (bch_wrapmode << 8) | /* wrap mode */ > - (dev_width << 7) | /* bus width */ > - (0x0 << 4) | /* number of sectors */ > - (cs << 1) | /* ECC CS */ > - (0x1)); /* enable ECC */ > - writel(ecc_config_val, &gpmc_cfg->ecc_config); > + > + /* (ECC 16 or 8 bit col) | ( CS ) | ECC Enable */ > + val = (dev_width << 7) | (info->cs << 1) | (0x1); > + writel(val, &gpmc_cfg->ecc_config); > } > > /* > @@ -271,6 +237,124 @@ static void omap_enable_hwecc(struct mtd_info *mtd, > int32_t mode) > */ > static int omap_calculate_ecc(struct mtd_info *mtd, const uint8_t *dat, > uint8_t *ecc_code) > +{ > + u32 val; > + > + val = readl(&gpmc_cfg->ecc1_result); > + ecc_code[0] = val & 0xFF; > + ecc_code[1] = (val >> 16) & 0xFF; > + ecc_code[2] = ((val >> 8) & 0x0F) | ((val >> 20) & 0xF0); > + > + return 0; > +} > + > +/* GPMC ecc engine settings for read */ > +#define BCH_WRAPMODE_1 1 /* BCH wrap mode 1 */ > +#define BCH8R_ECC_SIZE0 0x1a /* ecc_size0 = 26 */ > +#define BCH8R_ECC_SIZE1 0x2 /* ecc_size1 = 2 */ > +#define BCH4R_ECC_SIZE0 0xd /* ecc_size0 = 13 */ > +#define BCH4R_ECC_SIZE1 0x3 /* ecc_size1 = 3 */ > + > +/* GPMC ecc engine settings for write */ > +#define BCH_WRAPMODE_6 6 /* BCH wrap mode 6 */ > +#define BCH_ECC_SIZE0 0x0 /* ecc_size0 = 0, no oob protection > */ > +#define BCH_ECC_SIZE1 0x20 /* ecc_size1 = 32 */ > + > +/** > + * omap_enable_hwecc_bch - Program GPMC to perform BCH ECC calculation > + * @mtd: MTD device structure > + * @mode: Read/Write mode > + * > + * When using BCH with SW correction (i.e. no ELM), sector size is set > + * to 512 bytes and we use BCH_WRAPMODE_6 wrapping mode > + * for both reading and writing with: > + * eccsize0 = 0 (no additional protected byte in spare area) > + * eccsize1 = 32 (skip 32 nibbles = 16 bytes per sector in spare area) > + */ > +static void __maybe_unused omap_enable_hwecc_bch(struct mtd_info *mtd, > + int mode) > +{ > + unsigned int bch_type; > + unsigned int dev_width, nsectors; > + struct nand_chip *chip = mtd_to_nand(mtd); > + struct omap_nand_info *info = nand_get_controller_data(chip); > + u32 val, wr_mode; > + unsigned int ecc_size1, ecc_size0; > + > + /* GPMC configurations for calculating ECC */ > + switch (info->ecc_scheme) { > + case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: > + bch_type = 1; > + nsectors = 1; > + wr_mode = BCH_WRAPMODE_6; > + ecc_size0 = BCH_ECC_SIZE0; > + ecc_size1 = BCH_ECC_SIZE1; > + break; > + case OMAP_ECC_BCH8_CODE_HW: > + bch_type = 1; > + nsectors = chip->ecc.steps; > + if (mode == NAND_ECC_READ) { > + wr_mode = BCH_WRAPMODE_1; > + ecc_size0 = BCH8R_ECC_SIZE0; > + ecc_size1 = BCH8R_ECC_SIZE1; > + } else { > + wr_mode = BCH_WRAPMODE_6; > + ecc_size0 = BCH_ECC_SIZE0; > + ecc_size1 = BCH_ECC_SIZE1; > + } > + break; > + case OMAP_ECC_BCH16_CODE_HW: > + bch_type = 0x2; > + nsectors = chip->ecc.steps; > + if (mode == NAND_ECC_READ) { > + wr_mode = 0x01; > + ecc_size0 = 52; /* ECC bits in nibbles per sector */ > + ecc_size1 = 0; /* non-ECC bits in nibbles per sector > */ > + } else { > + wr_mode = 0x01; > + ecc_size0 = 0; /* extra bits in nibbles per sector */ > + ecc_size1 = 52; /* OOB bits in nibbles per sector */ > + } > + break; > + default: > + return; > + } > + > + writel(ECCRESULTREG1, &gpmc_cfg->ecc_control); > + > + /* Configure ecc size for BCH */ > + val = (ecc_size1 << ECCSIZE1_SHIFT) | (ecc_size0 << ECCSIZE0_SHIFT); > + writel(val, &gpmc_cfg->ecc_size_config); > + > + dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0; > + > + /* BCH configuration */ > + val = ((1 << 16) | /* enable BCH */ > + (bch_type << 12) | /* BCH4/BCH8/BCH16 */ > + (wr_mode << 8) | /* wrap mode */ > + (dev_width << 7) | /* bus width */ > + (((nsectors - 1) & 0x7) << 4) | /* number of sectors */ > + (info->cs << 1) | /* ECC CS */ > + (0x1)); /* enable ECC */ > + > + writel(val, &gpmc_cfg->ecc_config); > + > + /* Clear ecc and enable bits */ > + writel(ECCCLEAR | ECCRESULTREG1, &gpmc_cfg->ecc_control); > +} > + > +/** > + * _omap_calculate_ecc_bch - Generate BCH ECC bytes for one sector > + * @mtd: MTD device structure > + * @dat: The pointer to data on which ecc is computed > + * @ecc_code: The ecc_code buffer > + * @sector: The sector number (for a multi sector page) > + * > + * Support calculating of BCH4/8/16 ECC vectors for one sector > + * within a page. Sector number is in @sector. > + */ > +static int _omap_calculate_ecc_bch(struct mtd_info *mtd, const u8 *dat, > + u8 *ecc_code, int sector) > { > struct nand_chip *chip = mtd_to_nand(mtd); > struct omap_nand_info *info = nand_get_controller_data(chip); > @@ -279,17 +363,11 @@ static int omap_calculate_ecc(struct mtd_info *mtd, > const uint8_t *dat, > int8_t i = 0, j; > > switch (info->ecc_scheme) { > - case OMAP_ECC_HAM1_CODE_HW: > - val = readl(&gpmc_cfg->ecc1_result); > - ecc_code[0] = val & 0xFF; > - ecc_code[1] = (val >> 16) & 0xFF; > - ecc_code[2] = ((val >> 8) & 0x0F) | ((val >> 20) & 0xF0); > - break; > #ifdef CONFIG_BCH > case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: > #endif > case OMAP_ECC_BCH8_CODE_HW: > - ptr = &gpmc_cfg->bch_result_0_3[0].bch_result_x[3]; > + ptr = &gpmc_cfg->bch_result_0_3[sector].bch_result_x[3]; > val = readl(ptr); > ecc_code[i++] = (val >> 0) & 0xFF; > ptr--; > @@ -301,23 +379,24 @@ static int omap_calculate_ecc(struct mtd_info *mtd, > const uint8_t *dat, > ecc_code[i++] = (val >> 0) & 0xFF; > ptr--; > } > + > break; > case OMAP_ECC_BCH16_CODE_HW: > - val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[2]); > + val = > readl(&gpmc_cfg->bch_result_4_6[sector].bch_result_x[2]); > ecc_code[i++] = (val >> 8) & 0xFF; > ecc_code[i++] = (val >> 0) & 0xFF; > - val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[1]); > + val = > readl(&gpmc_cfg->bch_result_4_6[sector].bch_result_x[1]); > ecc_code[i++] = (val >> 24) & 0xFF; > ecc_code[i++] = (val >> 16) & 0xFF; > ecc_code[i++] = (val >> 8) & 0xFF; > ecc_code[i++] = (val >> 0) & 0xFF; > - val = readl(&gpmc_cfg->bch_result_4_6[0].bch_result_x[0]); > + val = > readl(&gpmc_cfg->bch_result_4_6[sector].bch_result_x[0]); > ecc_code[i++] = (val >> 24) & 0xFF; > ecc_code[i++] = (val >> 16) & 0xFF; > ecc_code[i++] = (val >> 8) & 0xFF; > ecc_code[i++] = (val >> 0) & 0xFF; > for (j = 3; j >= 0; j--) { > - val = > readl(&gpmc_cfg->bch_result_0_3[0].bch_result_x[j] > + val = > readl(&gpmc_cfg->bch_result_0_3[sector].bch_result_x[j] > ); > ecc_code[i++] = (val >> 24) & 0xFF; > ecc_code[i++] = (val >> 16) & 0xFF; > @@ -330,18 +409,18 @@ static int omap_calculate_ecc(struct mtd_info *mtd, > const uint8_t *dat, > } > /* ECC scheme specific syndrome customizations */ > switch (info->ecc_scheme) { > - case OMAP_ECC_HAM1_CODE_HW: > - break; > #ifdef CONFIG_BCH > case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW: > - > + /* Add constant polynomial to remainder, so that > + * ECC of blank pages results in 0x0 on reading back > + */ > for (i = 0; i < chip->ecc.bytes; i++) > - *(ecc_code + i) = *(ecc_code + i) ^ > - bch8_polynomial[i]; > + ecc_code[i] ^= bch8_polynomial[i]; > break; > #endif > case OMAP_ECC_BCH8_CODE_HW: > - ecc_code[chip->ecc.bytes - 1] = 0x00; > + /* Set 14th ECC byte as 0x0 for ROM compatibility */ > + ecc_code[chip->ecc.bytes - 1] = 0x0; > break; > case OMAP_ECC_BCH16_CODE_HW: > break; > @@ -351,6 +430,22 @@ static int omap_calculate_ecc(struct mtd_info *mtd, > const uint8_t *dat, > return 0; > } > > +/** > + * omap_calculate_ecc_bch - ECC generator for 1 sector > + * @mtd: MTD device structure > + * @dat: The pointer to data on which ecc is computed > + * @ecc_code: The ecc_code buffer > + * > + * Support calculating of BCH4/8/16 ECC vectors for one sector. This is used > + * when SW based correction is required as ECC is required for one sector > + * at a time. > + */ > +static int omap_calculate_ecc_bch(struct mtd_info *mtd, > + const u_char *dat, u_char *ecc_calc) > +{ > + return _omap_calculate_ecc_bch(mtd, dat, ecc_calc, 0); > +} > + > static inline void omap_nand_read_buf(struct mtd_info *mtd, uint8_t *buf, > int len) > { > struct nand_chip *chip = mtd_to_nand(mtd); > @@ -475,6 +570,35 @@ static void omap_nand_read_prefetch(struct mtd_info > *mtd, uint8_t *buf, int len) > #endif /* CONFIG_NAND_OMAP_GPMC_PREFETCH */ > > #ifdef CONFIG_NAND_OMAP_ELM > + > +/** > + * omap_calculate_ecc_bch_multi - Generate ECC for multiple sectors > + * @mtd: MTD device structure > + * @dat: The pointer to data on which ecc is computed > + * @ecc_code: The ecc_code buffer > + * > + * Support calculating of BCH4/8/16 ecc vectors for the entire page in one > go. > + */ > +static int omap_calculate_ecc_bch_multi(struct mtd_info *mtd, > + const u_char *dat, u_char *ecc_calc) > +{ > + struct nand_chip *chip = mtd_to_nand(mtd); > + int eccbytes = chip->ecc.bytes; > + unsigned long nsectors; > + int i, ret; > + > + nsectors = ((readl(&gpmc_cfg->ecc_config) >> 4) & 0x7) + 1; > + for (i = 0; i < nsectors; i++) { > + ret = _omap_calculate_ecc_bch(mtd, dat, ecc_calc, i); > + if (ret) > + return ret; > + > + ecc_calc += eccbytes; > + } > + > + return 0; > +} > + > /* > * omap_reverse_list - re-orders list elements in reverse order [internal] > * @list: pointer to start of list > @@ -627,52 +751,49 @@ static int omap_read_page_bch(struct mtd_info *mtd, > struct nand_chip *chip, > { > int i, eccsize = chip->ecc.size; > int eccbytes = chip->ecc.bytes; > + int ecctotal = chip->ecc.total; > int eccsteps = chip->ecc.steps; > uint8_t *p = buf; > uint8_t *ecc_calc = chip->buffers->ecccalc; > uint8_t *ecc_code = chip->buffers->ecccode; > uint32_t *eccpos = chip->ecc.layout->eccpos; > uint8_t *oob = chip->oob_poi; > - uint32_t data_pos; > uint32_t oob_pos; > > - data_pos = 0; > /* oob area start */ > oob_pos = (eccsize * eccsteps) + chip->ecc.layout->eccpos[0]; > oob += chip->ecc.layout->eccpos[0]; > > - for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize, > - oob += eccbytes) { > - chip->ecc.hwctl(mtd, NAND_ECC_READ); > - /* read data */ > - chip->cmdfunc(mtd, NAND_CMD_RNDOUT, data_pos, -1); > - chip->read_buf(mtd, p, eccsize); > - > - /* read respective ecc from oob area */ > - chip->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_pos, -1); > - chip->read_buf(mtd, oob, eccbytes); > - /* read syndrome */ > - chip->ecc.calculate(mtd, p, &ecc_calc[i]); > - > - data_pos += eccsize; > - oob_pos += eccbytes; > - } > + /* Enable ECC engine */ > + chip->ecc.hwctl(mtd, NAND_ECC_READ); > + > + /* read entire page */ > + chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1); > + chip->read_buf(mtd, buf, mtd->writesize); > + > + /* read all ecc bytes from oob area */ > + chip->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_pos, -1); > + chip->read_buf(mtd, oob, ecctotal); > + > + /* Calculate ecc bytes */ > + omap_calculate_ecc_bch_multi(mtd, buf, ecc_calc); > > for (i = 0; i < chip->ecc.total; i++) > ecc_code[i] = chip->oob_poi[eccpos[i]]; > > + /* error detect & correct */ > eccsteps = chip->ecc.steps; > p = buf; > > for (i = 0 ; eccsteps; eccsteps--, i += eccbytes, p += eccsize) { > int stat; > - > stat = chip->ecc.correct(mtd, p, &ecc_code[i], &ecc_calc[i]); > if (stat < 0) > mtd->ecc_stats.failed++; > else > mtd->ecc_stats.corrected += stat; > } > + > return 0; > } > #endif /* CONFIG_NAND_OMAP_ELM */ > @@ -820,9 +941,9 @@ static int omap_select_ecc_scheme(struct nand_chip *nand, > nand->ecc.strength = 8; > nand->ecc.size = SECTOR_BYTES; > nand->ecc.bytes = 13; > - nand->ecc.hwctl = omap_enable_hwecc; > + nand->ecc.hwctl = omap_enable_hwecc_bch; > nand->ecc.correct = omap_correct_data_bch_sw; > - nand->ecc.calculate = omap_calculate_ecc; > + nand->ecc.calculate = omap_calculate_ecc_bch; > /* define ecc-layout */ > ecclayout->eccbytes = nand->ecc.bytes * eccsteps; > ecclayout->eccpos[0] = BADBLOCK_MARKER_LENGTH; > @@ -861,9 +982,9 @@ static int omap_select_ecc_scheme(struct nand_chip *nand, > nand->ecc.strength = 8; > nand->ecc.size = SECTOR_BYTES; > nand->ecc.bytes = 14; > - nand->ecc.hwctl = omap_enable_hwecc; > + nand->ecc.hwctl = omap_enable_hwecc_bch; > nand->ecc.correct = omap_correct_data_bch; > - nand->ecc.calculate = omap_calculate_ecc; > + nand->ecc.calculate = omap_calculate_ecc_bch; > nand->ecc.read_page = omap_read_page_bch; > /* define ecc-layout */ > ecclayout->eccbytes = nand->ecc.bytes * eccsteps; > @@ -894,9 +1015,9 @@ static int omap_select_ecc_scheme(struct nand_chip *nand, > nand->ecc.size = SECTOR_BYTES; > nand->ecc.bytes = 26; > nand->ecc.strength = 16; > - nand->ecc.hwctl = omap_enable_hwecc; > + nand->ecc.hwctl = omap_enable_hwecc_bch; > nand->ecc.correct = omap_correct_data_bch; > - nand->ecc.calculate = omap_calculate_ecc; > + nand->ecc.calculate = omap_calculate_ecc_bch; > nand->ecc.read_page = omap_read_page_bch; > /* define ecc-layout */ > ecclayout->eccbytes = nand->ecc.bytes * eccsteps; > -- > 2.17.1 > -- Michael Nazzareno Trimarchi Co-Founder & Chief Executive Officer M. +39 347 913 2170 mich...@amarulasolutions.com __________________________________ Amarula Solutions BV Joop Geesinkweg 125, 1114 AB, Amsterdam, NL T. +31 (0)85 111 9172 i...@amarulasolutions.com www.amarulasolutions.com