go分布式存储，rs纠删码_纠删码可用容量计算

丢失的数据不能小于校验快的数据，否则没法复原数据

可用容量看，4+2纠删码的利用率是66%（4/(4+2)），但3副本只有33%(4/(4*3))，两者差了2倍，8+2纠删码的利用率更可以做到80%，由此可见纠删码的可用容量对比多副本要高很多

rs.txt 内容

AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDSDG
1

分片和解码

package main

// rs 纠删码
import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"

	"github.com/klauspost/reedsolomon"
)

const (
	dataShards = 2 // 数据分片数
	parShards  = 1 // 校验分片数
)

func main() {
	//encoder()
	decode()
}

func encoder() {

	fname := "rs.txt"
	// Create encoding matrix.
	enc, err := reedsolomon.New(dataShards, parShards)
	checkErr(err)

	fmt.Println("Opening", fname)
	b, err := ioutil.ReadFile(fname)
	checkErr(err)

	// Split the file into equally sized shards.
	// 此时分成了3等片
	shards, err := enc.Split(b)
	checkErr(err)
	fmt.Printf("File split into %d data+parity shards with %d bytes/shard.\n", len(shards), len(shards[0]))

	// Encode parity
	// 分片 encode 下
	err = enc.Encode(shards)
	checkErr(err)

	// Write out the resulting files.
	_, file := filepath.Split(fname)
	dir := "./tmp/encoder/"
	for i, shard := range shards {
		outfn := fmt.Sprintf("%s.%d", file, i)

		fmt.Println("Writing to", outfn)
		// 分割的分片写入各个文件中
		err = ioutil.WriteFile(filepath.Join(dir, outfn), shard, os.ModePerm)
		checkErr(err)
	}
}

func checkErr(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %s", err.Error())
		os.Exit(2)
	}
}

// -------------------------  decode  -----------------------

func decode() {
	// Create matrix
	enc, err := reedsolomon.New(dataShards, parShards)
	checkErr(err)

	fname := "./tmp/encoder/rs.txt"
	// Create shards and load the data.
	shards := make([][]byte, dataShards+parShards)
	var iNum []int
	for i := range shards {
		infn := fmt.Sprintf("%s.%d", fname, i)
		fmt.Println("Opening", infn)
		shards[i], err = ioutil.ReadFile(infn)
		if err != nil {
			fmt.Println("Error reading file", err)
			shards[i] = nil
			iNum = append(iNum, i)
		}
	}

	// Verify the shards
	ok, err := enc.Verify(shards)
	if ok {
		fmt.Println("No reconstruction needed")
	} else {
		fmt.Println("Verification failed. Reconstructing data")
		// 恢复切片
		err = enc.Reconstruct(shards)
		if err != nil {
			fmt.Println("Reconstruct failed -", err)
			os.Exit(1)
		}
		// 再次验证分片
		ok, err = enc.Verify(shards)
		if !ok {
			fmt.Println("Verification failed after reconstruction, data likely corrupted.")
			os.Exit(1)
		}
		checkErr(err)
		// 丢失的分片数据恢复
		for _, i2 := range iNum {
			outfn := fname + "." + strconv.Itoa(i2)
			err = ioutil.WriteFile(outfn, shards[i2], os.ModePerm)

		}
	}

	outfn := "./tmp/decoder/rs.txt"
	fmt.Println("Writing data to", outfn)
	f, err := os.Create(outfn)
	checkErr(err)

	// We don't know the exact filesize.
	// 还原数据
	err = enc.Join(f, shards, len(shards[0])*dataShards)
	checkErr(err)
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125