Merge tag 'mtd/for-4.17' of git://git.infradead.org/linux-mtd
[muen/linux.git] / drivers / mtd / nand / raw / atmel / pmecc.c
1 /*
2  * Copyright 2017 ATMEL
3  * Copyright 2017 Free Electrons
4  *
5  * Author: Boris Brezillon <boris.brezillon@free-electrons.com>
6  *
7  * Derived from the atmel_nand.c driver which contained the following
8  * copyrights:
9  *
10  *   Copyright 2003 Rick Bronson
11  *
12  *   Derived from drivers/mtd/nand/autcpu12.c (removed in v3.8)
13  *      Copyright 2001 Thomas Gleixner (gleixner@autronix.de)
14  *
15  *   Derived from drivers/mtd/spia.c (removed in v3.8)
16  *      Copyright 2000 Steven J. Hill (sjhill@cotw.com)
17  *
18  *   Add Hardware ECC support for AT91SAM9260 / AT91SAM9263
19  *      Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright 2007
20  *
21  *   Derived from Das U-Boot source code
22  *      (u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c)
23  *      Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas
24  *
25  *   Add Programmable Multibit ECC support for various AT91 SoC
26  *      Copyright 2012 ATMEL, Hong Xu
27  *
28  *   Add Nand Flash Controller support for SAMA5 SoC
29  *      Copyright 2013 ATMEL, Josh Wu (josh.wu@atmel.com)
30  *
31  * This program is free software; you can redistribute it and/or modify
32  * it under the terms of the GNU General Public License version 2 as
33  * published by the Free Software Foundation.
34  *
35  * The PMECC is an hardware assisted BCH engine, which means part of the
36  * ECC algorithm is left to the software. The hardware/software repartition
37  * is explained in the "PMECC Controller Functional Description" chapter in
38  * Atmel datasheets, and some of the functions in this file are directly
39  * implementing the algorithms described in the "Software Implementation"
40  * sub-section.
41  *
42  * TODO: it seems that the software BCH implementation in lib/bch.c is already
43  * providing some of the logic we are implementing here. It would be smart
44  * to expose the needed lib/bch.c helpers/functions and re-use them here.
45  */
46
47 #include <linux/genalloc.h>
48 #include <linux/iopoll.h>
49 #include <linux/module.h>
50 #include <linux/mtd/rawnand.h>
51 #include <linux/of_irq.h>
52 #include <linux/of_platform.h>
53 #include <linux/platform_device.h>
54 #include <linux/slab.h>
55
56 #include "pmecc.h"
57
58 /* Galois field dimension */
59 #define PMECC_GF_DIMENSION_13                   13
60 #define PMECC_GF_DIMENSION_14                   14
61
62 /* Primitive Polynomial used by PMECC */
63 #define PMECC_GF_13_PRIMITIVE_POLY              0x201b
64 #define PMECC_GF_14_PRIMITIVE_POLY              0x4443
65
66 #define PMECC_LOOKUP_TABLE_SIZE_512             0x2000
67 #define PMECC_LOOKUP_TABLE_SIZE_1024            0x4000
68
69 /* Time out value for reading PMECC status register */
70 #define PMECC_MAX_TIMEOUT_MS                    100
71
72 /* PMECC Register Definitions */
73 #define ATMEL_PMECC_CFG                         0x0
74 #define PMECC_CFG_BCH_STRENGTH(x)               (x)
75 #define PMECC_CFG_BCH_STRENGTH_MASK             GENMASK(2, 0)
76 #define PMECC_CFG_SECTOR512                     (0 << 4)
77 #define PMECC_CFG_SECTOR1024                    (1 << 4)
78 #define PMECC_CFG_NSECTORS(x)                   ((fls(x) - 1) << 8)
79 #define PMECC_CFG_READ_OP                       (0 << 12)
80 #define PMECC_CFG_WRITE_OP                      (1 << 12)
81 #define PMECC_CFG_SPARE_ENABLE                  BIT(16)
82 #define PMECC_CFG_AUTO_ENABLE                   BIT(20)
83
84 #define ATMEL_PMECC_SAREA                       0x4
85 #define ATMEL_PMECC_SADDR                       0x8
86 #define ATMEL_PMECC_EADDR                       0xc
87
88 #define ATMEL_PMECC_CLK                         0x10
89 #define PMECC_CLK_133MHZ                        (2 << 0)
90
91 #define ATMEL_PMECC_CTRL                        0x14
92 #define PMECC_CTRL_RST                          BIT(0)
93 #define PMECC_CTRL_DATA                         BIT(1)
94 #define PMECC_CTRL_USER                         BIT(2)
95 #define PMECC_CTRL_ENABLE                       BIT(4)
96 #define PMECC_CTRL_DISABLE                      BIT(5)
97
98 #define ATMEL_PMECC_SR                          0x18
99 #define PMECC_SR_BUSY                           BIT(0)
100 #define PMECC_SR_ENABLE                         BIT(4)
101
102 #define ATMEL_PMECC_IER                         0x1c
103 #define ATMEL_PMECC_IDR                         0x20
104 #define ATMEL_PMECC_IMR                         0x24
105 #define ATMEL_PMECC_ISR                         0x28
106 #define PMECC_ERROR_INT                         BIT(0)
107
108 #define ATMEL_PMECC_ECC(sector, n)              \
109         ((((sector) + 1) * 0x40) + (n))
110
111 #define ATMEL_PMECC_REM(sector, n)              \
112         ((((sector) + 1) * 0x40) + ((n) * 4) + 0x200)
113
114 /* PMERRLOC Register Definitions */
115 #define ATMEL_PMERRLOC_ELCFG                    0x0
116 #define PMERRLOC_ELCFG_SECTOR_512               (0 << 0)
117 #define PMERRLOC_ELCFG_SECTOR_1024              (1 << 0)
118 #define PMERRLOC_ELCFG_NUM_ERRORS(n)            ((n) << 16)
119
120 #define ATMEL_PMERRLOC_ELPRIM                   0x4
121 #define ATMEL_PMERRLOC_ELEN                     0x8
122 #define ATMEL_PMERRLOC_ELDIS                    0xc
123 #define PMERRLOC_DISABLE                        BIT(0)
124
125 #define ATMEL_PMERRLOC_ELSR                     0x10
126 #define PMERRLOC_ELSR_BUSY                      BIT(0)
127
128 #define ATMEL_PMERRLOC_ELIER                    0x14
129 #define ATMEL_PMERRLOC_ELIDR                    0x18
130 #define ATMEL_PMERRLOC_ELIMR                    0x1c
131 #define ATMEL_PMERRLOC_ELISR                    0x20
132 #define PMERRLOC_ERR_NUM_MASK                   GENMASK(12, 8)
133 #define PMERRLOC_CALC_DONE                      BIT(0)
134
135 #define ATMEL_PMERRLOC_SIGMA(x)                 (((x) * 0x4) + 0x28)
136
137 #define ATMEL_PMERRLOC_EL(offs, x)              (((x) * 0x4) + (offs))
138
139 struct atmel_pmecc_gf_tables {
140         u16 *alpha_to;
141         u16 *index_of;
142 };
143
144 struct atmel_pmecc_caps {
145         const int *strengths;
146         int nstrengths;
147         int el_offset;
148         bool correct_erased_chunks;
149 };
150
151 struct atmel_pmecc {
152         struct device *dev;
153         const struct atmel_pmecc_caps *caps;
154
155         struct {
156                 void __iomem *base;
157                 void __iomem *errloc;
158         } regs;
159
160         struct mutex lock;
161 };
162
163 struct atmel_pmecc_user_conf_cache {
164         u32 cfg;
165         u32 sarea;
166         u32 saddr;
167         u32 eaddr;
168 };
169
170 struct atmel_pmecc_user {
171         struct atmel_pmecc_user_conf_cache cache;
172         struct atmel_pmecc *pmecc;
173         const struct atmel_pmecc_gf_tables *gf_tables;
174         int eccbytes;
175         s16 *partial_syn;
176         s16 *si;
177         s16 *lmu;
178         s16 *smu;
179         s32 *mu;
180         s32 *dmu;
181         s32 *delta;
182         u32 isr;
183 };
184
185 static DEFINE_MUTEX(pmecc_gf_tables_lock);
186 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_512;
187 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_1024;
188
189 static inline int deg(unsigned int poly)
190 {
191         /* polynomial degree is the most-significant bit index */
192         return fls(poly) - 1;
193 }
194
195 static int atmel_pmecc_build_gf_tables(int mm, unsigned int poly,
196                                        struct atmel_pmecc_gf_tables *gf_tables)
197 {
198         unsigned int i, x = 1;
199         const unsigned int k = BIT(deg(poly));
200         unsigned int nn = BIT(mm) - 1;
201
202         /* primitive polynomial must be of degree m */
203         if (k != (1u << mm))
204                 return -EINVAL;
205
206         for (i = 0; i < nn; i++) {
207                 gf_tables->alpha_to[i] = x;
208                 gf_tables->index_of[x] = i;
209                 if (i && (x == 1))
210                         /* polynomial is not primitive (a^i=1 with 0<i<2^m-1) */
211                         return -EINVAL;
212                 x <<= 1;
213                 if (x & k)
214                         x ^= poly;
215         }
216         gf_tables->alpha_to[nn] = 1;
217         gf_tables->index_of[0] = 0;
218
219         return 0;
220 }
221
222 static const struct atmel_pmecc_gf_tables *
223 atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req *req)
224 {
225         struct atmel_pmecc_gf_tables *gf_tables;
226         unsigned int poly, degree, table_size;
227         int ret;
228
229         if (req->ecc.sectorsize == 512) {
230                 degree = PMECC_GF_DIMENSION_13;
231                 poly = PMECC_GF_13_PRIMITIVE_POLY;
232                 table_size = PMECC_LOOKUP_TABLE_SIZE_512;
233         } else {
234                 degree = PMECC_GF_DIMENSION_14;
235                 poly = PMECC_GF_14_PRIMITIVE_POLY;
236                 table_size = PMECC_LOOKUP_TABLE_SIZE_1024;
237         }
238
239         gf_tables = kzalloc(sizeof(*gf_tables) +
240                             (2 * table_size * sizeof(u16)),
241                             GFP_KERNEL);
242         if (!gf_tables)
243                 return ERR_PTR(-ENOMEM);
244
245         gf_tables->alpha_to = (void *)(gf_tables + 1);
246         gf_tables->index_of = gf_tables->alpha_to + table_size;
247
248         ret = atmel_pmecc_build_gf_tables(degree, poly, gf_tables);
249         if (ret) {
250                 kfree(gf_tables);
251                 return ERR_PTR(ret);
252         }
253
254         return gf_tables;
255 }
256
257 static const struct atmel_pmecc_gf_tables *
258 atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req *req)
259 {
260         const struct atmel_pmecc_gf_tables **gf_tables, *ret;
261
262         mutex_lock(&pmecc_gf_tables_lock);
263         if (req->ecc.sectorsize == 512)
264                 gf_tables = &pmecc_gf_tables_512;
265         else
266                 gf_tables = &pmecc_gf_tables_1024;
267
268         ret = *gf_tables;
269
270         if (!ret) {
271                 ret = atmel_pmecc_create_gf_tables(req);
272                 if (!IS_ERR(ret))
273                         *gf_tables = ret;
274         }
275         mutex_unlock(&pmecc_gf_tables_lock);
276
277         return ret;
278 }
279
280 static int atmel_pmecc_prepare_user_req(struct atmel_pmecc *pmecc,
281                                         struct atmel_pmecc_user_req *req)
282 {
283         int i, max_eccbytes, eccbytes = 0, eccstrength = 0;
284
285         if (req->pagesize <= 0 || req->oobsize <= 0 || req->ecc.bytes <= 0)
286                 return -EINVAL;
287
288         if (req->ecc.ooboffset >= 0 &&
289             req->ecc.ooboffset + req->ecc.bytes > req->oobsize)
290                 return -EINVAL;
291
292         if (req->ecc.sectorsize == ATMEL_PMECC_SECTOR_SIZE_AUTO) {
293                 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH)
294                         return -EINVAL;
295
296                 if (req->pagesize > 512)
297                         req->ecc.sectorsize = 1024;
298                 else
299                         req->ecc.sectorsize = 512;
300         }
301
302         if (req->ecc.sectorsize != 512 && req->ecc.sectorsize != 1024)
303                 return -EINVAL;
304
305         if (req->pagesize % req->ecc.sectorsize)
306                 return -EINVAL;
307
308         req->ecc.nsectors = req->pagesize / req->ecc.sectorsize;
309
310         max_eccbytes = req->ecc.bytes;
311
312         for (i = 0; i < pmecc->caps->nstrengths; i++) {
313                 int nbytes, strength = pmecc->caps->strengths[i];
314
315                 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH &&
316                     strength < req->ecc.strength)
317                         continue;
318
319                 nbytes = DIV_ROUND_UP(strength * fls(8 * req->ecc.sectorsize),
320                                       8);
321                 nbytes *= req->ecc.nsectors;
322
323                 if (nbytes > max_eccbytes)
324                         break;
325
326                 eccstrength = strength;
327                 eccbytes = nbytes;
328
329                 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH)
330                         break;
331         }
332
333         if (!eccstrength)
334                 return -EINVAL;
335
336         req->ecc.bytes = eccbytes;
337         req->ecc.strength = eccstrength;
338
339         if (req->ecc.ooboffset < 0)
340                 req->ecc.ooboffset = req->oobsize - eccbytes;
341
342         return 0;
343 }
344
345 struct atmel_pmecc_user *
346 atmel_pmecc_create_user(struct atmel_pmecc *pmecc,
347                         struct atmel_pmecc_user_req *req)
348 {
349         struct atmel_pmecc_user *user;
350         const struct atmel_pmecc_gf_tables *gf_tables;
351         int strength, size, ret;
352
353         ret = atmel_pmecc_prepare_user_req(pmecc, req);
354         if (ret)
355                 return ERR_PTR(ret);
356
357         size = sizeof(*user);
358         size = ALIGN(size, sizeof(u16));
359         /* Reserve space for partial_syn, si and smu */
360         size += ((2 * req->ecc.strength) + 1) * sizeof(u16) *
361                 (2 + req->ecc.strength + 2);
362         /* Reserve space for lmu. */
363         size += (req->ecc.strength + 1) * sizeof(u16);
364         /* Reserve space for mu, dmu and delta. */
365         size = ALIGN(size, sizeof(s32));
366         size += (req->ecc.strength + 1) * sizeof(s32) * 3;
367
368         user = kzalloc(size, GFP_KERNEL);
369         if (!user)
370                 return ERR_PTR(-ENOMEM);
371
372         user->pmecc = pmecc;
373
374         user->partial_syn = (s16 *)PTR_ALIGN(user + 1, sizeof(u16));
375         user->si = user->partial_syn + ((2 * req->ecc.strength) + 1);
376         user->lmu = user->si + ((2 * req->ecc.strength) + 1);
377         user->smu = user->lmu + (req->ecc.strength + 1);
378         user->mu = (s32 *)PTR_ALIGN(user->smu +
379                                     (((2 * req->ecc.strength) + 1) *
380                                      (req->ecc.strength + 2)),
381                                     sizeof(s32));
382         user->dmu = user->mu + req->ecc.strength + 1;
383         user->delta = user->dmu + req->ecc.strength + 1;
384
385         gf_tables = atmel_pmecc_get_gf_tables(req);
386         if (IS_ERR(gf_tables)) {
387                 kfree(user);
388                 return ERR_CAST(gf_tables);
389         }
390
391         user->gf_tables = gf_tables;
392
393         user->eccbytes = req->ecc.bytes / req->ecc.nsectors;
394
395         for (strength = 0; strength < pmecc->caps->nstrengths; strength++) {
396                 if (pmecc->caps->strengths[strength] == req->ecc.strength)
397                         break;
398         }
399
400         user->cache.cfg = PMECC_CFG_BCH_STRENGTH(strength) |
401                           PMECC_CFG_NSECTORS(req->ecc.nsectors);
402
403         if (req->ecc.sectorsize == 1024)
404                 user->cache.cfg |= PMECC_CFG_SECTOR1024;
405
406         user->cache.sarea = req->oobsize - 1;
407         user->cache.saddr = req->ecc.ooboffset;
408         user->cache.eaddr = req->ecc.ooboffset + req->ecc.bytes - 1;
409
410         return user;
411 }
412 EXPORT_SYMBOL_GPL(atmel_pmecc_create_user);
413
414 void atmel_pmecc_destroy_user(struct atmel_pmecc_user *user)
415 {
416         kfree(user);
417 }
418 EXPORT_SYMBOL_GPL(atmel_pmecc_destroy_user);
419
420 static int get_strength(struct atmel_pmecc_user *user)
421 {
422         const int *strengths = user->pmecc->caps->strengths;
423
424         return strengths[user->cache.cfg & PMECC_CFG_BCH_STRENGTH_MASK];
425 }
426
427 static int get_sectorsize(struct atmel_pmecc_user *user)
428 {
429         return user->cache.cfg & PMECC_CFG_SECTOR1024 ? 1024 : 512;
430 }
431
432 static void atmel_pmecc_gen_syndrome(struct atmel_pmecc_user *user, int sector)
433 {
434         int strength = get_strength(user);
435         u32 value;
436         int i;
437
438         /* Fill odd syndromes */
439         for (i = 0; i < strength; i++) {
440                 value = readl_relaxed(user->pmecc->regs.base +
441                                       ATMEL_PMECC_REM(sector, i / 2));
442                 if (i & 1)
443                         value >>= 16;
444
445                 user->partial_syn[(2 * i) + 1] = value;
446         }
447 }
448
449 static void atmel_pmecc_substitute(struct atmel_pmecc_user *user)
450 {
451         int degree = get_sectorsize(user) == 512 ? 13 : 14;
452         int cw_len = BIT(degree) - 1;
453         int strength = get_strength(user);
454         s16 *alpha_to = user->gf_tables->alpha_to;
455         s16 *index_of = user->gf_tables->index_of;
456         s16 *partial_syn = user->partial_syn;
457         s16 *si;
458         int i, j;
459
460         /*
461          * si[] is a table that holds the current syndrome value,
462          * an element of that table belongs to the field
463          */
464         si = user->si;
465
466         memset(&si[1], 0, sizeof(s16) * ((2 * strength) - 1));
467
468         /* Computation 2t syndromes based on S(x) */
469         /* Odd syndromes */
470         for (i = 1; i < 2 * strength; i += 2) {
471                 for (j = 0; j < degree; j++) {
472                         if (partial_syn[i] & BIT(j))
473                                 si[i] = alpha_to[i * j] ^ si[i];
474                 }
475         }
476         /* Even syndrome = (Odd syndrome) ** 2 */
477         for (i = 2, j = 1; j <= strength; i = ++j << 1) {
478                 if (si[j] == 0) {
479                         si[i] = 0;
480                 } else {
481                         s16 tmp;
482
483                         tmp = index_of[si[j]];
484                         tmp = (tmp * 2) % cw_len;
485                         si[i] = alpha_to[tmp];
486                 }
487         }
488 }
489
490 static void atmel_pmecc_get_sigma(struct atmel_pmecc_user *user)
491 {
492         s16 *lmu = user->lmu;
493         s16 *si = user->si;
494         s32 *mu = user->mu;
495         s32 *dmu = user->dmu;
496         s32 *delta = user->delta;
497         int degree = get_sectorsize(user) == 512 ? 13 : 14;
498         int cw_len = BIT(degree) - 1;
499         int strength = get_strength(user);
500         int num = 2 * strength + 1;
501         s16 *index_of = user->gf_tables->index_of;
502         s16 *alpha_to = user->gf_tables->alpha_to;
503         int i, j, k;
504         u32 dmu_0_count, tmp;
505         s16 *smu = user->smu;
506
507         /* index of largest delta */
508         int ro;
509         int largest;
510         int diff;
511
512         dmu_0_count = 0;
513
514         /* First Row */
515
516         /* Mu */
517         mu[0] = -1;
518
519         memset(smu, 0, sizeof(s16) * num);
520         smu[0] = 1;
521
522         /* discrepancy set to 1 */
523         dmu[0] = 1;
524         /* polynom order set to 0 */
525         lmu[0] = 0;
526         delta[0] = (mu[0] * 2 - lmu[0]) >> 1;
527
528         /* Second Row */
529
530         /* Mu */
531         mu[1] = 0;
532         /* Sigma(x) set to 1 */
533         memset(&smu[num], 0, sizeof(s16) * num);
534         smu[num] = 1;
535
536         /* discrepancy set to S1 */
537         dmu[1] = si[1];
538
539         /* polynom order set to 0 */
540         lmu[1] = 0;
541
542         delta[1] = (mu[1] * 2 - lmu[1]) >> 1;
543
544         /* Init the Sigma(x) last row */
545         memset(&smu[(strength + 1) * num], 0, sizeof(s16) * num);
546
547         for (i = 1; i <= strength; i++) {
548                 mu[i + 1] = i << 1;
549                 /* Begin Computing Sigma (Mu+1) and L(mu) */
550                 /* check if discrepancy is set to 0 */
551                 if (dmu[i] == 0) {
552                         dmu_0_count++;
553
554                         tmp = ((strength - (lmu[i] >> 1) - 1) / 2);
555                         if ((strength - (lmu[i] >> 1) - 1) & 0x1)
556                                 tmp += 2;
557                         else
558                                 tmp += 1;
559
560                         if (dmu_0_count == tmp) {
561                                 for (j = 0; j <= (lmu[i] >> 1) + 1; j++)
562                                         smu[(strength + 1) * num + j] =
563                                                         smu[i * num + j];
564
565                                 lmu[strength + 1] = lmu[i];
566                                 return;
567                         }
568
569                         /* copy polynom */
570                         for (j = 0; j <= lmu[i] >> 1; j++)
571                                 smu[(i + 1) * num + j] = smu[i * num + j];
572
573                         /* copy previous polynom order to the next */
574                         lmu[i + 1] = lmu[i];
575                 } else {
576                         ro = 0;
577                         largest = -1;
578                         /* find largest delta with dmu != 0 */
579                         for (j = 0; j < i; j++) {
580                                 if ((dmu[j]) && (delta[j] > largest)) {
581                                         largest = delta[j];
582                                         ro = j;
583                                 }
584                         }
585
586                         /* compute difference */
587                         diff = (mu[i] - mu[ro]);
588
589                         /* Compute degree of the new smu polynomial */
590                         if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff))
591                                 lmu[i + 1] = lmu[i];
592                         else
593                                 lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2;
594
595                         /* Init smu[i+1] with 0 */
596                         for (k = 0; k < num; k++)
597                                 smu[(i + 1) * num + k] = 0;
598
599                         /* Compute smu[i+1] */
600                         for (k = 0; k <= lmu[ro] >> 1; k++) {
601                                 s16 a, b, c;
602
603                                 if (!(smu[ro * num + k] && dmu[i]))
604                                         continue;
605
606                                 a = index_of[dmu[i]];
607                                 b = index_of[dmu[ro]];
608                                 c = index_of[smu[ro * num + k]];
609                                 tmp = a + (cw_len - b) + c;
610                                 a = alpha_to[tmp % cw_len];
611                                 smu[(i + 1) * num + (k + diff)] = a;
612                         }
613
614                         for (k = 0; k <= lmu[i] >> 1; k++)
615                                 smu[(i + 1) * num + k] ^= smu[i * num + k];
616                 }
617
618                 /* End Computing Sigma (Mu+1) and L(mu) */
619                 /* In either case compute delta */
620                 delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1;
621
622                 /* Do not compute discrepancy for the last iteration */
623                 if (i >= strength)
624                         continue;
625
626                 for (k = 0; k <= (lmu[i + 1] >> 1); k++) {
627                         tmp = 2 * (i - 1);
628                         if (k == 0) {
629                                 dmu[i + 1] = si[tmp + 3];
630                         } else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) {
631                                 s16 a, b, c;
632
633                                 a = index_of[smu[(i + 1) * num + k]];
634                                 b = si[2 * (i - 1) + 3 - k];
635                                 c = index_of[b];
636                                 tmp = a + c;
637                                 tmp %= cw_len;
638                                 dmu[i + 1] = alpha_to[tmp] ^ dmu[i + 1];
639                         }
640                 }
641         }
642 }
643
644 static int atmel_pmecc_err_location(struct atmel_pmecc_user *user)
645 {
646         int sector_size = get_sectorsize(user);
647         int degree = sector_size == 512 ? 13 : 14;
648         struct atmel_pmecc *pmecc = user->pmecc;
649         int strength = get_strength(user);
650         int ret, roots_nbr, i, err_nbr = 0;
651         int num = (2 * strength) + 1;
652         s16 *smu = user->smu;
653         u32 val;
654
655         writel(PMERRLOC_DISABLE, pmecc->regs.errloc + ATMEL_PMERRLOC_ELDIS);
656
657         for (i = 0; i <= user->lmu[strength + 1] >> 1; i++) {
658                 writel_relaxed(smu[(strength + 1) * num + i],
659                                pmecc->regs.errloc + ATMEL_PMERRLOC_SIGMA(i));
660                 err_nbr++;
661         }
662
663         val = (err_nbr - 1) << 16;
664         if (sector_size == 1024)
665                 val |= 1;
666
667         writel(val, pmecc->regs.errloc + ATMEL_PMERRLOC_ELCFG);
668         writel((sector_size * 8) + (degree * strength),
669                pmecc->regs.errloc + ATMEL_PMERRLOC_ELEN);
670
671         ret = readl_relaxed_poll_timeout(pmecc->regs.errloc +
672                                          ATMEL_PMERRLOC_ELISR,
673                                          val, val & PMERRLOC_CALC_DONE, 0,
674                                          PMECC_MAX_TIMEOUT_MS * 1000);
675         if (ret) {
676                 dev_err(pmecc->dev,
677                         "PMECC: Timeout to calculate error location.\n");
678                 return ret;
679         }
680
681         roots_nbr = (val & PMERRLOC_ERR_NUM_MASK) >> 8;
682         /* Number of roots == degree of smu hence <= cap */
683         if (roots_nbr == user->lmu[strength + 1] >> 1)
684                 return err_nbr - 1;
685
686         /*
687          * Number of roots does not match the degree of smu
688          * unable to correct error.
689          */
690         return -EBADMSG;
691 }
692
693 int atmel_pmecc_correct_sector(struct atmel_pmecc_user *user, int sector,
694                                void *data, void *ecc)
695 {
696         struct atmel_pmecc *pmecc = user->pmecc;
697         int sectorsize = get_sectorsize(user);
698         int eccbytes = user->eccbytes;
699         int i, nerrors;
700
701         if (!(user->isr & BIT(sector)))
702                 return 0;
703
704         atmel_pmecc_gen_syndrome(user, sector);
705         atmel_pmecc_substitute(user);
706         atmel_pmecc_get_sigma(user);
707
708         nerrors = atmel_pmecc_err_location(user);
709         if (nerrors < 0)
710                 return nerrors;
711
712         for (i = 0; i < nerrors; i++) {
713                 const char *area;
714                 int byte, bit;
715                 u32 errpos;
716                 u8 *ptr;
717
718                 errpos = readl_relaxed(pmecc->regs.errloc +
719                                 ATMEL_PMERRLOC_EL(pmecc->caps->el_offset, i));
720                 errpos--;
721
722                 byte = errpos / 8;
723                 bit = errpos % 8;
724
725                 if (byte < sectorsize) {
726                         ptr = data + byte;
727                         area = "data";
728                 } else if (byte < sectorsize + eccbytes) {
729                         ptr = ecc + byte - sectorsize;
730                         area = "ECC";
731                 } else {
732                         dev_dbg(pmecc->dev,
733                                 "Invalid errpos value (%d, max is %d)\n",
734                                 errpos, (sectorsize + eccbytes) * 8);
735                         return -EINVAL;
736                 }
737
738                 dev_dbg(pmecc->dev,
739                         "Bit flip in %s area, byte %d: 0x%02x -> 0x%02x\n",
740                         area, byte, *ptr, (unsigned int)(*ptr ^ BIT(bit)));
741
742                 *ptr ^= BIT(bit);
743         }
744
745         return nerrors;
746 }
747 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_sector);
748
749 bool atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user *user)
750 {
751         return user->pmecc->caps->correct_erased_chunks;
752 }
753 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_erased_chunks);
754
755 void atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user *user,
756                                         int sector, void *ecc)
757 {
758         struct atmel_pmecc *pmecc = user->pmecc;
759         u8 *ptr = ecc;
760         int i;
761
762         for (i = 0; i < user->eccbytes; i++)
763                 ptr[i] = readb_relaxed(pmecc->regs.base +
764                                        ATMEL_PMECC_ECC(sector, i));
765 }
766 EXPORT_SYMBOL_GPL(atmel_pmecc_get_generated_eccbytes);
767
768 void atmel_pmecc_reset(struct atmel_pmecc *pmecc)
769 {
770         writel(PMECC_CTRL_RST, pmecc->regs.base + ATMEL_PMECC_CTRL);
771         writel(PMECC_CTRL_DISABLE, pmecc->regs.base + ATMEL_PMECC_CTRL);
772 }
773 EXPORT_SYMBOL_GPL(atmel_pmecc_reset);
774
775 int atmel_pmecc_enable(struct atmel_pmecc_user *user, int op)
776 {
777         struct atmel_pmecc *pmecc = user->pmecc;
778         u32 cfg;
779
780         if (op != NAND_ECC_READ && op != NAND_ECC_WRITE) {
781                 dev_err(pmecc->dev, "Bad ECC operation!");
782                 return -EINVAL;
783         }
784
785         mutex_lock(&user->pmecc->lock);
786
787         cfg = user->cache.cfg;
788         if (op == NAND_ECC_WRITE)
789                 cfg |= PMECC_CFG_WRITE_OP;
790         else
791                 cfg |= PMECC_CFG_AUTO_ENABLE;
792
793         writel(cfg, pmecc->regs.base + ATMEL_PMECC_CFG);
794         writel(user->cache.sarea, pmecc->regs.base + ATMEL_PMECC_SAREA);
795         writel(user->cache.saddr, pmecc->regs.base + ATMEL_PMECC_SADDR);
796         writel(user->cache.eaddr, pmecc->regs.base + ATMEL_PMECC_EADDR);
797
798         writel(PMECC_CTRL_ENABLE, pmecc->regs.base + ATMEL_PMECC_CTRL);
799         writel(PMECC_CTRL_DATA, pmecc->regs.base + ATMEL_PMECC_CTRL);
800
801         return 0;
802 }
803 EXPORT_SYMBOL_GPL(atmel_pmecc_enable);
804
805 void atmel_pmecc_disable(struct atmel_pmecc_user *user)
806 {
807         atmel_pmecc_reset(user->pmecc);
808         mutex_unlock(&user->pmecc->lock);
809 }
810 EXPORT_SYMBOL_GPL(atmel_pmecc_disable);
811
812 int atmel_pmecc_wait_rdy(struct atmel_pmecc_user *user)
813 {
814         struct atmel_pmecc *pmecc = user->pmecc;
815         u32 status;
816         int ret;
817
818         ret = readl_relaxed_poll_timeout(pmecc->regs.base +
819                                          ATMEL_PMECC_SR,
820                                          status, !(status & PMECC_SR_BUSY), 0,
821                                          PMECC_MAX_TIMEOUT_MS * 1000);
822         if (ret) {
823                 dev_err(pmecc->dev,
824                         "Timeout while waiting for PMECC ready.\n");
825                 return ret;
826         }
827
828         user->isr = readl_relaxed(pmecc->regs.base + ATMEL_PMECC_ISR);
829
830         return 0;
831 }
832 EXPORT_SYMBOL_GPL(atmel_pmecc_wait_rdy);
833
834 static struct atmel_pmecc *atmel_pmecc_create(struct platform_device *pdev,
835                                         const struct atmel_pmecc_caps *caps,
836                                         int pmecc_res_idx, int errloc_res_idx)
837 {
838         struct device *dev = &pdev->dev;
839         struct atmel_pmecc *pmecc;
840         struct resource *res;
841
842         pmecc = devm_kzalloc(dev, sizeof(*pmecc), GFP_KERNEL);
843         if (!pmecc)
844                 return ERR_PTR(-ENOMEM);
845
846         pmecc->caps = caps;
847         pmecc->dev = dev;
848         mutex_init(&pmecc->lock);
849
850         res = platform_get_resource(pdev, IORESOURCE_MEM, pmecc_res_idx);
851         pmecc->regs.base = devm_ioremap_resource(dev, res);
852         if (IS_ERR(pmecc->regs.base))
853                 return ERR_CAST(pmecc->regs.base);
854
855         res = platform_get_resource(pdev, IORESOURCE_MEM, errloc_res_idx);
856         pmecc->regs.errloc = devm_ioremap_resource(dev, res);
857         if (IS_ERR(pmecc->regs.errloc))
858                 return ERR_CAST(pmecc->regs.errloc);
859
860         /* Disable all interrupts before registering the PMECC handler. */
861         writel(0xffffffff, pmecc->regs.base + ATMEL_PMECC_IDR);
862         atmel_pmecc_reset(pmecc);
863
864         return pmecc;
865 }
866
867 static void devm_atmel_pmecc_put(struct device *dev, void *res)
868 {
869         struct atmel_pmecc **pmecc = res;
870
871         put_device((*pmecc)->dev);
872 }
873
874 static struct atmel_pmecc *atmel_pmecc_get_by_node(struct device *userdev,
875                                                    struct device_node *np)
876 {
877         struct platform_device *pdev;
878         struct atmel_pmecc *pmecc, **ptr;
879
880         pdev = of_find_device_by_node(np);
881         if (!pdev || !platform_get_drvdata(pdev))
882                 return ERR_PTR(-EPROBE_DEFER);
883
884         ptr = devres_alloc(devm_atmel_pmecc_put, sizeof(*ptr), GFP_KERNEL);
885         if (!ptr)
886                 return ERR_PTR(-ENOMEM);
887
888         get_device(&pdev->dev);
889         pmecc = platform_get_drvdata(pdev);
890
891         *ptr = pmecc;
892
893         devres_add(userdev, ptr);
894
895         return pmecc;
896 }
897
898 static const int atmel_pmecc_strengths[] = { 2, 4, 8, 12, 24, 32 };
899
900 static struct atmel_pmecc_caps at91sam9g45_caps = {
901         .strengths = atmel_pmecc_strengths,
902         .nstrengths = 5,
903         .el_offset = 0x8c,
904 };
905
906 static struct atmel_pmecc_caps sama5d4_caps = {
907         .strengths = atmel_pmecc_strengths,
908         .nstrengths = 5,
909         .el_offset = 0x8c,
910         .correct_erased_chunks = true,
911 };
912
913 static struct atmel_pmecc_caps sama5d2_caps = {
914         .strengths = atmel_pmecc_strengths,
915         .nstrengths = 6,
916         .el_offset = 0xac,
917         .correct_erased_chunks = true,
918 };
919
920 static const struct of_device_id atmel_pmecc_legacy_match[] = {
921         { .compatible = "atmel,sama5d4-nand", &sama5d4_caps },
922         { .compatible = "atmel,sama5d2-nand", &sama5d2_caps },
923         { /* sentinel */ }
924 };
925
926 struct atmel_pmecc *devm_atmel_pmecc_get(struct device *userdev)
927 {
928         struct atmel_pmecc *pmecc;
929         struct device_node *np;
930
931         if (!userdev)
932                 return ERR_PTR(-EINVAL);
933
934         if (!userdev->of_node)
935                 return NULL;
936
937         np = of_parse_phandle(userdev->of_node, "ecc-engine", 0);
938         if (np) {
939                 pmecc = atmel_pmecc_get_by_node(userdev, np);
940                 of_node_put(np);
941         } else {
942                 /*
943                  * Support old DT bindings: in this case the PMECC iomem
944                  * resources are directly defined in the user pdev at position
945                  * 1 and 2. Extract all relevant information from there.
946                  */
947                 struct platform_device *pdev = to_platform_device(userdev);
948                 const struct atmel_pmecc_caps *caps;
949                 const struct of_device_id *match;
950
951                 /* No PMECC engine available. */
952                 if (!of_property_read_bool(userdev->of_node,
953                                            "atmel,has-pmecc"))
954                         return NULL;
955
956                 caps = &at91sam9g45_caps;
957
958                 /* Find the caps associated to the NAND dev node. */
959                 match = of_match_node(atmel_pmecc_legacy_match,
960                                       userdev->of_node);
961                 if (match && match->data)
962                         caps = match->data;
963
964                 pmecc = atmel_pmecc_create(pdev, caps, 1, 2);
965         }
966
967         return pmecc;
968 }
969 EXPORT_SYMBOL(devm_atmel_pmecc_get);
970
971 static const struct of_device_id atmel_pmecc_match[] = {
972         { .compatible = "atmel,at91sam9g45-pmecc", &at91sam9g45_caps },
973         { .compatible = "atmel,sama5d4-pmecc", &sama5d4_caps },
974         { .compatible = "atmel,sama5d2-pmecc", &sama5d2_caps },
975         { /* sentinel */ }
976 };
977 MODULE_DEVICE_TABLE(of, atmel_pmecc_match);
978
979 static int atmel_pmecc_probe(struct platform_device *pdev)
980 {
981         struct device *dev = &pdev->dev;
982         const struct atmel_pmecc_caps *caps;
983         struct atmel_pmecc *pmecc;
984
985         caps = of_device_get_match_data(&pdev->dev);
986         if (!caps) {
987                 dev_err(dev, "Invalid caps\n");
988                 return -EINVAL;
989         }
990
991         pmecc = atmel_pmecc_create(pdev, caps, 0, 1);
992         if (IS_ERR(pmecc))
993                 return PTR_ERR(pmecc);
994
995         platform_set_drvdata(pdev, pmecc);
996
997         return 0;
998 }
999
1000 static struct platform_driver atmel_pmecc_driver = {
1001         .driver = {
1002                 .name = "atmel-pmecc",
1003                 .of_match_table = of_match_ptr(atmel_pmecc_match),
1004         },
1005         .probe = atmel_pmecc_probe,
1006 };
1007 module_platform_driver(atmel_pmecc_driver);
1008
1009 MODULE_LICENSE("GPL");
1010 MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com>");
1011 MODULE_DESCRIPTION("PMECC engine driver");
1012 MODULE_ALIAS("platform:atmel_pmecc");