source: src/linux/universal/linux-4.9/drivers/md/dm-verity-fec.c @ 31859

Last change on this file since 31859 was 31859, checked in by brainslayer, 11 days ago

kernel update

File size: 20.7 KB
Line 
1/*
2 * Copyright (C) 2015 Google, Inc.
3 *
4 * Author: Sami Tolvanen <samitolvanen@google.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the Free
8 * Software Foundation; either version 2 of the License, or (at your option)
9 * any later version.
10 */
11
12#include "dm-verity-fec.h"
13#include <linux/math64.h>
14
15#define DM_MSG_PREFIX   "verity-fec"
16
17/*
18 * If error correction has been configured, returns true.
19 */
20bool verity_fec_is_enabled(struct dm_verity *v)
21{
22        return v->fec && v->fec->dev;
23}
24
25/*
26 * Return a pointer to dm_verity_fec_io after dm_verity_io and its variable
27 * length fields.
28 */
29static inline struct dm_verity_fec_io *fec_io(struct dm_verity_io *io)
30{
31        return (struct dm_verity_fec_io *) verity_io_digest_end(io->v, io);
32}
33
34/*
35 * Return an interleaved offset for a byte in RS block.
36 */
37static inline u64 fec_interleave(struct dm_verity *v, u64 offset)
38{
39        u32 mod;
40
41        mod = do_div(offset, v->fec->rsn);
42        return offset + mod * (v->fec->rounds << v->data_dev_block_bits);
43}
44
45/*
46 * Decode an RS block using Reed-Solomon.
47 */
48static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio,
49                          u8 *data, u8 *fec, int neras)
50{
51        int i;
52        uint16_t par[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN];
53
54        for (i = 0; i < v->fec->roots; i++)
55                par[i] = fec[i];
56
57        return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras,
58                          fio->erasures, 0, NULL);
59}
60
61/*
62 * Read error-correcting codes for the requested RS block. Returns a pointer
63 * to the data block. Caller is responsible for releasing buf.
64 */
65static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index,
66                           unsigned *offset, struct dm_buffer **buf)
67{
68        u64 position, block;
69        u8 *res;
70
71        position = (index + rsb) * v->fec->roots;
72        block = position >> v->data_dev_block_bits;
73        *offset = (unsigned)(position - (block << v->data_dev_block_bits));
74
75        res = dm_bufio_read(v->fec->bufio, v->fec->start + block, buf);
76        if (unlikely(IS_ERR(res))) {
77                DMERR("%s: FEC %llu: parity read failed (block %llu): %ld",
78                      v->data_dev->name, (unsigned long long)rsb,
79                      (unsigned long long)(v->fec->start + block),
80                      PTR_ERR(res));
81                *buf = NULL;
82        }
83
84        return res;
85}
86
87/* Loop over each preallocated buffer slot. */
88#define fec_for_each_prealloc_buffer(__i) \
89        for (__i = 0; __i < DM_VERITY_FEC_BUF_PREALLOC; __i++)
90
91/* Loop over each extra buffer slot. */
92#define fec_for_each_extra_buffer(io, __i) \
93        for (__i = DM_VERITY_FEC_BUF_PREALLOC; __i < DM_VERITY_FEC_BUF_MAX; __i++)
94
95/* Loop over each allocated buffer. */
96#define fec_for_each_buffer(io, __i) \
97        for (__i = 0; __i < (io)->nbufs; __i++)
98
99/* Loop over each RS block in each allocated buffer. */
100#define fec_for_each_buffer_rs_block(io, __i, __j) \
101        fec_for_each_buffer(io, __i) \
102                for (__j = 0; __j < 1 << DM_VERITY_FEC_BUF_RS_BITS; __j++)
103
104/*
105 * Return a pointer to the current RS block when called inside
106 * fec_for_each_buffer_rs_block.
107 */
108static inline u8 *fec_buffer_rs_block(struct dm_verity *v,
109                                      struct dm_verity_fec_io *fio,
110                                      unsigned i, unsigned j)
111{
112        return &fio->bufs[i][j * v->fec->rsn];
113}
114
115/*
116 * Return an index to the current RS block when called inside
117 * fec_for_each_buffer_rs_block.
118 */
119static inline unsigned fec_buffer_rs_index(unsigned i, unsigned j)
120{
121        return (i << DM_VERITY_FEC_BUF_RS_BITS) + j;
122}
123
124/*
125 * Decode all RS blocks from buffers and copy corrected bytes into fio->output
126 * starting from block_offset.
127 */
128static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio,
129                           u64 rsb, int byte_index, unsigned block_offset,
130                           int neras)
131{
132        int r, corrected = 0, res;
133        struct dm_buffer *buf;
134        unsigned n, i, offset;
135        u8 *par, *block;
136
137        par = fec_read_parity(v, rsb, block_offset, &offset, &buf);
138        if (IS_ERR(par))
139                return PTR_ERR(par);
140
141        /*
142         * Decode the RS blocks we have in bufs. Each RS block results in
143         * one corrected target byte and consumes fec->roots parity bytes.
144         */
145        fec_for_each_buffer_rs_block(fio, n, i) {
146                block = fec_buffer_rs_block(v, fio, n, i);
147                res = fec_decode_rs8(v, fio, block, &par[offset], neras);
148                if (res < 0) {
149                        r = res;
150                        goto error;
151                }
152
153                corrected += res;
154                fio->output[block_offset] = block[byte_index];
155
156                block_offset++;
157                if (block_offset >= 1 << v->data_dev_block_bits)
158                        goto done;
159
160                /* read the next block when we run out of parity bytes */
161                offset += v->fec->roots;
162                if (offset >= 1 << v->data_dev_block_bits) {
163                        dm_bufio_release(buf);
164
165                        par = fec_read_parity(v, rsb, block_offset, &offset, &buf);
166                        if (unlikely(IS_ERR(par)))
167                                return PTR_ERR(par);
168                }
169        }
170done:
171        r = corrected;
172error:
173        dm_bufio_release(buf);
174
175        if (r < 0 && neras)
176                DMERR_LIMIT("%s: FEC %llu: failed to correct: %d",
177                            v->data_dev->name, (unsigned long long)rsb, r);
178        else if (r > 0)
179                DMWARN_LIMIT("%s: FEC %llu: corrected %d errors",
180                             v->data_dev->name, (unsigned long long)rsb, r);
181
182        return r;
183}
184
185/*
186 * Locate data block erasures using verity hashes.
187 */
188static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io,
189                          u8 *want_digest, u8 *data)
190{
191        if (unlikely(verity_hash(v, verity_io_hash_desc(v, io),
192                                 data, 1 << v->data_dev_block_bits,
193                                 verity_io_real_digest(v, io))))
194                return 0;
195
196        return memcmp(verity_io_real_digest(v, io), want_digest,
197                      v->digest_size) != 0;
198}
199
200/*
201 * Read data blocks that are part of the RS block and deinterleave as much as
202 * fits into buffers. Check for erasure locations if @neras is non-NULL.
203 */
204static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io,
205                         u64 rsb, u64 target, unsigned block_offset,
206                         int *neras)
207{
208        bool is_zero;
209        int i, j, target_index = -1;
210        struct dm_buffer *buf;
211        struct dm_bufio_client *bufio;
212        struct dm_verity_fec_io *fio = fec_io(io);
213        u64 block, ileaved;
214        u8 *bbuf, *rs_block;
215        u8 want_digest[v->digest_size];
216        unsigned n, k;
217
218        if (neras)
219                *neras = 0;
220
221        /*
222         * read each of the rsn data blocks that are part of the RS block, and
223         * interleave contents to available bufs
224         */
225        for (i = 0; i < v->fec->rsn; i++) {
226                ileaved = fec_interleave(v, rsb * v->fec->rsn + i);
227
228                /*
229                 * target is the data block we want to correct, target_index is
230                 * the index of this block within the rsn RS blocks
231                 */
232                if (ileaved == target)
233                        target_index = i;
234
235                block = ileaved >> v->data_dev_block_bits;
236                bufio = v->fec->data_bufio;
237
238                if (block >= v->data_blocks) {
239                        block -= v->data_blocks;
240
241                        /*
242                         * blocks outside the area were assumed to contain
243                         * zeros when encoding data was generated
244                         */
245                        if (unlikely(block >= v->fec->hash_blocks))
246                                continue;
247
248                        block += v->hash_start;
249                        bufio = v->bufio;
250                }
251
252                bbuf = dm_bufio_read(bufio, block, &buf);
253                if (unlikely(IS_ERR(bbuf))) {
254                        DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld",
255                                     v->data_dev->name,
256                                     (unsigned long long)rsb,
257                                     (unsigned long long)block, PTR_ERR(bbuf));
258
259                        /* assume the block is corrupted */
260                        if (neras && *neras <= v->fec->roots)
261                                fio->erasures[(*neras)++] = i;
262
263                        continue;
264                }
265
266                /* locate erasures if the block is on the data device */
267                if (bufio == v->fec->data_bufio &&
268                    verity_hash_for_block(v, io, block, want_digest,
269                                          &is_zero) == 0) {
270                        /* skip known zero blocks entirely */
271                        if (is_zero)
272                                goto done;
273
274                        /*
275                         * skip if we have already found the theoretical
276                         * maximum number (i.e. fec->roots) of erasures
277                         */
278                        if (neras && *neras <= v->fec->roots &&
279                            fec_is_erasure(v, io, want_digest, bbuf))
280                                fio->erasures[(*neras)++] = i;
281                }
282
283                /*
284                 * deinterleave and copy the bytes that fit into bufs,
285                 * starting from block_offset
286                 */
287                fec_for_each_buffer_rs_block(fio, n, j) {
288                        k = fec_buffer_rs_index(n, j) + block_offset;
289
290                        if (k >= 1 << v->data_dev_block_bits)
291                                goto done;
292
293                        rs_block = fec_buffer_rs_block(v, fio, n, j);
294                        rs_block[i] = bbuf[k];
295                }
296done:
297                dm_bufio_release(buf);
298        }
299
300        return target_index;
301}
302
303/*
304 * Allocate RS control structure and FEC buffers from preallocated mempools,
305 * and attempt to allocate as many extra buffers as available.
306 */
307static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)
308{
309        unsigned n;
310
311        if (!fio->rs) {
312                fio->rs = mempool_alloc(v->fec->rs_pool, 0);
313                if (unlikely(!fio->rs)) {
314                        DMERR("failed to allocate RS");
315                        return -ENOMEM;
316                }
317        }
318
319        fec_for_each_prealloc_buffer(n) {
320                if (fio->bufs[n])
321                        continue;
322
323                fio->bufs[n] = mempool_alloc(v->fec->prealloc_pool, GFP_NOIO);
324                if (unlikely(!fio->bufs[n])) {
325                        DMERR("failed to allocate FEC buffer");
326                        return -ENOMEM;
327                }
328        }
329
330        /* try to allocate the maximum number of buffers */
331        fec_for_each_extra_buffer(fio, n) {
332                if (fio->bufs[n])
333                        continue;
334
335                fio->bufs[n] = mempool_alloc(v->fec->extra_pool, GFP_NOIO);
336                /* we can manage with even one buffer if necessary */
337                if (unlikely(!fio->bufs[n]))
338                        break;
339        }
340        fio->nbufs = n;
341
342        if (!fio->output) {
343                fio->output = mempool_alloc(v->fec->output_pool, GFP_NOIO);
344
345                if (!fio->output) {
346                        DMERR("failed to allocate FEC page");
347                        return -ENOMEM;
348                }
349        }
350
351        return 0;
352}
353
354/*
355 * Initialize buffers and clear erasures. fec_read_bufs() assumes buffers are
356 * zeroed before deinterleaving.
357 */
358static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)
359{
360        unsigned n;
361
362        fec_for_each_buffer(fio, n)
363                memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS);
364
365        memset(fio->erasures, 0, sizeof(fio->erasures));
366}
367
368/*
369 * Decode all RS blocks in a single data block and return the target block
370 * (indicated by @offset) in fio->output. If @use_erasures is non-zero, uses
371 * hashes to locate erasures.
372 */
373static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io,
374                          struct dm_verity_fec_io *fio, u64 rsb, u64 offset,
375                          bool use_erasures)
376{
377        int r, neras = 0;
378        unsigned pos;
379
380        r = fec_alloc_bufs(v, fio);
381        if (unlikely(r < 0))
382                return r;
383
384        for (pos = 0; pos < 1 << v->data_dev_block_bits; ) {
385                fec_init_bufs(v, fio);
386
387                r = fec_read_bufs(v, io, rsb, offset, pos,
388                                  use_erasures ? &neras : NULL);
389                if (unlikely(r < 0))
390                        return r;
391
392                r = fec_decode_bufs(v, fio, rsb, r, pos, neras);
393                if (r < 0)
394                        return r;
395
396                pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS;
397        }
398
399        /* Always re-validate the corrected block against the expected hash */
400        r = verity_hash(v, verity_io_hash_desc(v, io), fio->output,
401                        1 << v->data_dev_block_bits,
402                        verity_io_real_digest(v, io));
403        if (unlikely(r < 0))
404                return r;
405
406        if (memcmp(verity_io_real_digest(v, io), verity_io_want_digest(v, io),
407                   v->digest_size)) {
408                DMERR_LIMIT("%s: FEC %llu: failed to correct (%d erasures)",
409                            v->data_dev->name, (unsigned long long)rsb, neras);
410                return -EILSEQ;
411        }
412
413        return 0;
414}
415
416static int fec_bv_copy(struct dm_verity *v, struct dm_verity_io *io, u8 *data,
417                       size_t len)
418{
419        struct dm_verity_fec_io *fio = fec_io(io);
420
421        memcpy(data, &fio->output[fio->output_pos], len);
422        fio->output_pos += len;
423
424        return 0;
425}
426
427/*
428 * Correct errors in a block. Copies corrected block to dest if non-NULL,
429 * otherwise to a bio_vec starting from iter.
430 */
431int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io,
432                      enum verity_block_type type, sector_t block, u8 *dest,
433                      struct bvec_iter *iter)
434{
435        int r;
436        struct dm_verity_fec_io *fio = fec_io(io);
437        u64 offset, res, rsb;
438
439        if (!verity_fec_is_enabled(v))
440                return -EOPNOTSUPP;
441
442        if (fio->level >= DM_VERITY_FEC_MAX_RECURSION) {
443                DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name);
444                return -EIO;
445        }
446
447        fio->level++;
448
449        if (type == DM_VERITY_BLOCK_TYPE_METADATA)
450                block += v->data_blocks;
451
452        /*
453         * For RS(M, N), the continuous FEC data is divided into blocks of N
454         * bytes. Since block size may not be divisible by N, the last block
455         * is zero padded when decoding.
456         *
457         * Each byte of the block is covered by a different RS(M, N) code,
458         * and each code is interleaved over N blocks to make it less likely
459         * that bursty corruption will leave us in unrecoverable state.
460         */
461
462        offset = block << v->data_dev_block_bits;
463        res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits);
464
465        /*
466         * The base RS block we can feed to the interleaver to find out all
467         * blocks required for decoding.
468         */
469        rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits);
470
471        /*
472         * Locating erasures is slow, so attempt to recover the block without
473         * them first. Do a second attempt with erasures if the corruption is
474         * bad enough.
475         */
476        r = fec_decode_rsb(v, io, fio, rsb, offset, false);
477        if (r < 0) {
478                r = fec_decode_rsb(v, io, fio, rsb, offset, true);
479                if (r < 0)
480                        goto done;
481        }
482
483        if (dest)
484                memcpy(dest, fio->output, 1 << v->data_dev_block_bits);
485        else if (iter) {
486                fio->output_pos = 0;
487                r = verity_for_bv_block(v, io, iter, fec_bv_copy);
488        }
489
490done:
491        fio->level--;
492        return r;
493}
494
495/*
496 * Clean up per-bio data.
497 */
498void verity_fec_finish_io(struct dm_verity_io *io)
499{
500        unsigned n;
501        struct dm_verity_fec *f = io->v->fec;
502        struct dm_verity_fec_io *fio = fec_io(io);
503
504        if (!verity_fec_is_enabled(io->v))
505                return;
506
507        mempool_free(fio->rs, f->rs_pool);
508
509        fec_for_each_prealloc_buffer(n)
510                mempool_free(fio->bufs[n], f->prealloc_pool);
511
512        fec_for_each_extra_buffer(fio, n)
513                mempool_free(fio->bufs[n], f->extra_pool);
514
515        mempool_free(fio->output, f->output_pool);
516}
517
518/*
519 * Initialize per-bio data.
520 */
521void verity_fec_init_io(struct dm_verity_io *io)
522{
523        struct dm_verity_fec_io *fio = fec_io(io);
524
525        if (!verity_fec_is_enabled(io->v))
526                return;
527
528        fio->rs = NULL;
529        memset(fio->bufs, 0, sizeof(fio->bufs));
530        fio->nbufs = 0;
531        fio->output = NULL;
532        fio->level = 0;
533}
534
535/*
536 * Append feature arguments and values to the status table.
537 */
538unsigned verity_fec_status_table(struct dm_verity *v, unsigned sz,
539                                 char *result, unsigned maxlen)
540{
541        if (!verity_fec_is_enabled(v))
542                return sz;
543
544        DMEMIT(" " DM_VERITY_OPT_FEC_DEV " %s "
545               DM_VERITY_OPT_FEC_BLOCKS " %llu "
546               DM_VERITY_OPT_FEC_START " %llu "
547               DM_VERITY_OPT_FEC_ROOTS " %d",
548               v->fec->dev->name,
549               (unsigned long long)v->fec->blocks,
550               (unsigned long long)v->fec->start,
551               v->fec->roots);
552
553        return sz;
554}
555
556void verity_fec_dtr(struct dm_verity *v)
557{
558        struct dm_verity_fec *f = v->fec;
559
560        if (!verity_fec_is_enabled(v))
561                goto out;
562
563        mempool_destroy(f->rs_pool);
564        mempool_destroy(f->prealloc_pool);
565        mempool_destroy(f->extra_pool);
566        kmem_cache_destroy(f->cache);
567
568        if (f->data_bufio)
569                dm_bufio_client_destroy(f->data_bufio);
570        if (f->bufio)
571                dm_bufio_client_destroy(f->bufio);
572
573        if (f->dev)
574                dm_put_device(v->ti, f->dev);
575out:
576        kfree(f);
577        v->fec = NULL;
578}
579
580static void *fec_rs_alloc(gfp_t gfp_mask, void *pool_data)
581{
582        struct dm_verity *v = (struct dm_verity *)pool_data;
583
584        return init_rs(8, 0x11d, 0, 1, v->fec->roots);
585}
586
587static void fec_rs_free(void *element, void *pool_data)
588{
589        struct rs_control *rs = (struct rs_control *)element;
590
591        if (rs)
592                free_rs(rs);
593}
594
595bool verity_is_fec_opt_arg(const char *arg_name)
596{
597        return (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV) ||
598                !strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS) ||
599                !strcasecmp(arg_name, DM_VERITY_OPT_FEC_START) ||
600                !strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS));
601}
602
603int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v,
604                              unsigned *argc, const char *arg_name)
605{
606        int r;
607        struct dm_target *ti = v->ti;
608        const char *arg_value;
609        unsigned long long num_ll;
610        unsigned char num_c;
611        char dummy;
612
613        if (!*argc) {
614                ti->error = "FEC feature arguments require a value";
615                return -EINVAL;
616        }
617
618        arg_value = dm_shift_arg(as);
619        (*argc)--;
620
621        if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV)) {
622                r = dm_get_device(ti, arg_value, FMODE_READ, &v->fec->dev);
623                if (r) {
624                        ti->error = "FEC device lookup failed";
625                        return r;
626                }
627
628        } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS)) {
629                if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 ||
630                    ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT))
631                     >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) {
632                        ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS;
633                        return -EINVAL;
634                }
635                v->fec->blocks = num_ll;
636
637        } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_START)) {
638                if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 ||
639                    ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) >>
640                     (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) {
641                        ti->error = "Invalid " DM_VERITY_OPT_FEC_START;
642                        return -EINVAL;
643                }
644                v->fec->start = num_ll;
645
646        } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)) {
647                if (sscanf(arg_value, "%hhu%c", &num_c, &dummy) != 1 || !num_c ||
648                    num_c < (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MAX_RSN) ||
649                    num_c > (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN)) {
650                        ti->error = "Invalid " DM_VERITY_OPT_FEC_ROOTS;
651                        return -EINVAL;
652                }
653                v->fec->roots = num_c;
654
655        } else {
656                ti->error = "Unrecognized verity FEC feature request";
657                return -EINVAL;
658        }
659
660        return 0;
661}
662
663/*
664 * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr.
665 */
666int verity_fec_ctr_alloc(struct dm_verity *v)
667{
668        struct dm_verity_fec *f;
669
670        f = kzalloc(sizeof(struct dm_verity_fec), GFP_KERNEL);
671        if (!f) {
672                v->ti->error = "Cannot allocate FEC structure";
673                return -ENOMEM;
674        }
675        v->fec = f;
676
677        return 0;
678}
679
680/*
681 * Validate arguments and preallocate memory. Must be called after arguments
682 * have been parsed using verity_fec_parse_opt_args.
683 */
684int verity_fec_ctr(struct dm_verity *v)
685{
686        struct dm_verity_fec *f = v->fec;
687        struct dm_target *ti = v->ti;
688        u64 hash_blocks;
689
690        if (!verity_fec_is_enabled(v)) {
691                verity_fec_dtr(v);
692                return 0;
693        }
694
695        /*
696         * FEC is computed over data blocks, possible metadata, and
697         * hash blocks. In other words, FEC covers total of fec_blocks
698         * blocks consisting of the following:
699         *
700         *  data blocks | hash blocks | metadata (optional)
701         *
702         * We allow metadata after hash blocks to support a use case
703         * where all data is stored on the same device and FEC covers
704         * the entire area.
705         *
706         * If metadata is included, we require it to be available on the
707         * hash device after the hash blocks.
708         */
709
710        hash_blocks = v->hash_blocks - v->hash_start;
711
712        /*
713         * Require matching block sizes for data and hash devices for
714         * simplicity.
715         */
716        if (v->data_dev_block_bits != v->hash_dev_block_bits) {
717                ti->error = "Block sizes must match to use FEC";
718                return -EINVAL;
719        }
720
721        if (!f->roots) {
722                ti->error = "Missing " DM_VERITY_OPT_FEC_ROOTS;
723                return -EINVAL;
724        }
725        f->rsn = DM_VERITY_FEC_RSM - f->roots;
726
727        if (!f->blocks) {
728                ti->error = "Missing " DM_VERITY_OPT_FEC_BLOCKS;
729                return -EINVAL;
730        }
731
732        f->rounds = f->blocks;
733        if (sector_div(f->rounds, f->rsn))
734                f->rounds++;
735
736        /*
737         * Due to optional metadata, f->blocks can be larger than
738         * data_blocks and hash_blocks combined.
739         */
740        if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) {
741                ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS;
742                return -EINVAL;
743        }
744
745        /*
746         * Metadata is accessed through the hash device, so we require
747         * it to be large enough.
748         */
749        f->hash_blocks = f->blocks - v->data_blocks;
750        if (dm_bufio_get_device_size(v->bufio) < f->hash_blocks) {
751                ti->error = "Hash device is too small for "
752                        DM_VERITY_OPT_FEC_BLOCKS;
753                return -E2BIG;
754        }
755
756        f->bufio = dm_bufio_client_create(f->dev->bdev,
757                                          1 << v->data_dev_block_bits,
758                                          1, 0, NULL, NULL);
759        if (IS_ERR(f->bufio)) {
760                ti->error = "Cannot initialize FEC bufio client";
761                return PTR_ERR(f->bufio);
762        }
763
764        if (dm_bufio_get_device_size(f->bufio) <
765            ((f->start + f->rounds * f->roots) >> v->data_dev_block_bits)) {
766                ti->error = "FEC device is too small";
767                return -E2BIG;
768        }
769
770        f->data_bufio = dm_bufio_client_create(v->data_dev->bdev,
771                                               1 << v->data_dev_block_bits,
772                                               1, 0, NULL, NULL);
773        if (IS_ERR(f->data_bufio)) {
774                ti->error = "Cannot initialize FEC data bufio client";
775                return PTR_ERR(f->data_bufio);
776        }
777
778        if (dm_bufio_get_device_size(f->data_bufio) < v->data_blocks) {
779                ti->error = "Data device is too small";
780                return -E2BIG;
781        }
782
783        /* Preallocate an rs_control structure for each worker thread */
784        f->rs_pool = mempool_create(num_online_cpus(), fec_rs_alloc,
785                                    fec_rs_free, (void *) v);
786        if (!f->rs_pool) {
787                ti->error = "Cannot allocate RS pool";
788                return -ENOMEM;
789        }
790
791        f->cache = kmem_cache_create("dm_verity_fec_buffers",
792                                     f->rsn << DM_VERITY_FEC_BUF_RS_BITS,
793                                     0, 0, NULL);
794        if (!f->cache) {
795                ti->error = "Cannot create FEC buffer cache";
796                return -ENOMEM;
797        }
798
799        /* Preallocate DM_VERITY_FEC_BUF_PREALLOC buffers for each thread */
800        f->prealloc_pool = mempool_create_slab_pool(num_online_cpus() *
801                                                    DM_VERITY_FEC_BUF_PREALLOC,
802                                                    f->cache);
803        if (!f->prealloc_pool) {
804                ti->error = "Cannot allocate FEC buffer prealloc pool";
805                return -ENOMEM;
806        }
807
808        f->extra_pool = mempool_create_slab_pool(0, f->cache);
809        if (!f->extra_pool) {
810                ti->error = "Cannot allocate FEC buffer extra pool";
811                return -ENOMEM;
812        }
813
814        /* Preallocate an output buffer for each thread */
815        f->output_pool = mempool_create_kmalloc_pool(num_online_cpus(),
816                                                     1 << v->data_dev_block_bits);
817        if (!f->output_pool) {
818                ti->error = "Cannot allocate FEC output pool";
819                return -ENOMEM;
820        }
821
822        /* Reserve space for our per-bio data */
823        ti->per_io_data_size += sizeof(struct dm_verity_fec_io);
824
825        return 0;
826}
Note: See TracBrowser for help on using the repository browser.