@cryptotaxi247 / kubo / commits / 7910c6e4b

Remove snappy from Godeps

It is in go-datastore License: MIT Signed-off-by: Jakub Sztandera <kubuxu@protonmail.ch>

Jakub Sztandera committed Jun 11, 2016 at 16:23 UTC 7910c6e4b74b6bde11b5b604d04394d4fce576a0
4 files changed -982
Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/decode.go deleted
-292
@@ -1,292 +0,0 @@
1 -// Copyright 2011 The Snappy-Go Authors. All rights reserved.
2 -// Use of this source code is governed by a BSD-style
3 -// license that can be found in the LICENSE file.
4 -
5 -package snappy
6 -
7 -import (
8 - "encoding/binary"
9 - "errors"
10 - "io"
11 -)
12 -
13 -var (
14 - // ErrCorrupt reports that the input is invalid.
15 - ErrCorrupt = errors.New("snappy: corrupt input")
16 - // ErrUnsupported reports that the input isn't supported.
17 - ErrUnsupported = errors.New("snappy: unsupported input")
18 -)
19 -
20 -// DecodedLen returns the length of the decoded block.
21 -func DecodedLen(src []byte) (int, error) {
22 - v, _, err := decodedLen(src)
23 - return v, err
24 -}
25 -
26 -// decodedLen returns the length of the decoded block and the number of bytes
27 -// that the length header occupied.
28 -func decodedLen(src []byte) (blockLen, headerLen int, err error) {
29 - v, n := binary.Uvarint(src)
30 - if n == 0 {
31 - return 0, 0, ErrCorrupt
32 - }
33 - if uint64(int(v)) != v {
34 - return 0, 0, errors.New("snappy: decoded block is too large")
35 - }
36 - return int(v), n, nil
37 -}
38 -
39 -// Decode returns the decoded form of src. The returned slice may be a sub-
40 -// slice of dst if dst was large enough to hold the entire decoded block.
41 -// Otherwise, a newly allocated slice will be returned.
42 -// It is valid to pass a nil dst.
43 -func Decode(dst, src []byte) ([]byte, error) {
44 - dLen, s, err := decodedLen(src)
45 - if err != nil {
46 - return nil, err
47 - }
48 - if len(dst) < dLen {
49 - dst = make([]byte, dLen)
50 - }
51 -
52 - var d, offset, length int
53 - for s < len(src) {
54 - switch src[s] & 0x03 {
55 - case tagLiteral:
56 - x := uint(src[s] >> 2)
57 - switch {
58 - case x < 60:
59 - s += 1
60 - case x == 60:
61 - s += 2
62 - if s > len(src) {
63 - return nil, ErrCorrupt
64 - }
65 - x = uint(src[s-1])
66 - case x == 61:
67 - s += 3
68 - if s > len(src) {
69 - return nil, ErrCorrupt
70 - }
71 - x = uint(src[s-2]) | uint(src[s-1])<<8
72 - case x == 62:
73 - s += 4
74 - if s > len(src) {
75 - return nil, ErrCorrupt
76 - }
77 - x = uint(src[s-3]) | uint(src[s-2])<<8 | uint(src[s-1])<<16
78 - case x == 63:
79 - s += 5
80 - if s > len(src) {
81 - return nil, ErrCorrupt
82 - }
83 - x = uint(src[s-4]) | uint(src[s-3])<<8 | uint(src[s-2])<<16 | uint(src[s-1])<<24
84 - }
85 - length = int(x + 1)
86 - if length <= 0 {
87 - return nil, errors.New("snappy: unsupported literal length")
88 - }
89 - if length > len(dst)-d || length > len(src)-s {
90 - return nil, ErrCorrupt
91 - }
92 - copy(dst[d:], src[s:s+length])
93 - d += length
94 - s += length
95 - continue
96 -
97 - case tagCopy1:
98 - s += 2
99 - if s > len(src) {
100 - return nil, ErrCorrupt
101 - }
102 - length = 4 + int(src[s-2])>>2&0x7
103 - offset = int(src[s-2])&0xe0<<3 | int(src[s-1])
104 -
105 - case tagCopy2:
106 - s += 3
107 - if s > len(src) {
108 - return nil, ErrCorrupt
109 - }
110 - length = 1 + int(src[s-3])>>2
111 - offset = int(src[s-2]) | int(src[s-1])<<8
112 -
113 - case tagCopy4:
114 - return nil, errors.New("snappy: unsupported COPY_4 tag")
115 - }
116 -
117 - end := d + length
118 - if offset > d || end > len(dst) {
119 - return nil, ErrCorrupt
120 - }
121 - for ; d < end; d++ {
122 - dst[d] = dst[d-offset]
123 - }
124 - }
125 - if d != dLen {
126 - return nil, ErrCorrupt
127 - }
128 - return dst[:d], nil
129 -}
130 -
131 -// NewReader returns a new Reader that decompresses from r, using the framing
132 -// format described at
133 -// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
134 -func NewReader(r io.Reader) *Reader {
135 - return &Reader{
136 - r: r,
137 - decoded: make([]byte, maxUncompressedChunkLen),
138 - buf: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)+checksumSize),
139 - }
140 -}
141 -
142 -// Reader is an io.Reader than can read Snappy-compressed bytes.
143 -type Reader struct {
144 - r io.Reader
145 - err error
146 - decoded []byte
147 - buf []byte
148 - // decoded[i:j] contains decoded bytes that have not yet been passed on.
149 - i, j int
150 - readHeader bool
151 -}
152 -
153 -// Reset discards any buffered data, resets all state, and switches the Snappy
154 -// reader to read from r. This permits reusing a Reader rather than allocating
155 -// a new one.
156 -func (r *Reader) Reset(reader io.Reader) {
157 - r.r = reader
158 - r.err = nil
159 - r.i = 0
160 - r.j = 0
161 - r.readHeader = false
162 -}
163 -
164 -func (r *Reader) readFull(p []byte) (ok bool) {
165 - if _, r.err = io.ReadFull(r.r, p); r.err != nil {
166 - if r.err == io.ErrUnexpectedEOF {
167 - r.err = ErrCorrupt
168 - }
169 - return false
170 - }
171 - return true
172 -}
173 -
174 -// Read satisfies the io.Reader interface.
175 -func (r *Reader) Read(p []byte) (int, error) {
176 - if r.err != nil {
177 - return 0, r.err
178 - }
179 - for {
180 - if r.i < r.j {
181 - n := copy(p, r.decoded[r.i:r.j])
182 - r.i += n
183 - return n, nil
184 - }
185 - if !r.readFull(r.buf[:4]) {
186 - return 0, r.err
187 - }
188 - chunkType := r.buf[0]
189 - if !r.readHeader {
190 - if chunkType != chunkTypeStreamIdentifier {
191 - r.err = ErrCorrupt
192 - return 0, r.err
193 - }
194 - r.readHeader = true
195 - }
196 - chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16
197 - if chunkLen > len(r.buf) {
198 - r.err = ErrUnsupported
199 - return 0, r.err
200 - }
201 -
202 - // The chunk types are specified at
203 - // https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
204 - switch chunkType {
205 - case chunkTypeCompressedData:
206 - // Section 4.2. Compressed data (chunk type 0x00).
207 - if chunkLen < checksumSize {
208 - r.err = ErrCorrupt
209 - return 0, r.err
210 - }
211 - buf := r.buf[:chunkLen]
212 - if !r.readFull(buf) {
213 - return 0, r.err
214 - }
215 - checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
216 - buf = buf[checksumSize:]
217 -
218 - n, err := DecodedLen(buf)
219 - if err != nil {
220 - r.err = err
221 - return 0, r.err
222 - }
223 - if n > len(r.decoded) {
224 - r.err = ErrCorrupt
225 - return 0, r.err
226 - }
227 - if _, err := Decode(r.decoded, buf); err != nil {
228 - r.err = err
229 - return 0, r.err
230 - }
231 - if crc(r.decoded[:n]) != checksum {
232 - r.err = ErrCorrupt
233 - return 0, r.err
234 - }
235 - r.i, r.j = 0, n
236 - continue
237 -
238 - case chunkTypeUncompressedData:
239 - // Section 4.3. Uncompressed data (chunk type 0x01).
240 - if chunkLen < checksumSize {
241 - r.err = ErrCorrupt
242 - return 0, r.err
243 - }
244 - buf := r.buf[:checksumSize]
245 - if !r.readFull(buf) {
246 - return 0, r.err
247 - }
248 - checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
249 - // Read directly into r.decoded instead of via r.buf.
250 - n := chunkLen - checksumSize
251 - if !r.readFull(r.decoded[:n]) {
252 - return 0, r.err
253 - }
254 - if crc(r.decoded[:n]) != checksum {
255 - r.err = ErrCorrupt
256 - return 0, r.err
257 - }
258 - r.i, r.j = 0, n
259 - continue
260 -
261 - case chunkTypeStreamIdentifier:
262 - // Section 4.1. Stream identifier (chunk type 0xff).
263 - if chunkLen != len(magicBody) {
264 - r.err = ErrCorrupt
265 - return 0, r.err
266 - }
267 - if !r.readFull(r.buf[:len(magicBody)]) {
268 - return 0, r.err
269 - }
270 - for i := 0; i < len(magicBody); i++ {
271 - if r.buf[i] != magicBody[i] {
272 - r.err = ErrCorrupt
273 - return 0, r.err
274 - }
275 - }
276 - continue
277 - }
278 -
279 - if chunkType <= 0x7f {
280 - // Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
281 - r.err = ErrUnsupported
282 - return 0, r.err
283 -
284 - } else {
285 - // Section 4.4 Padding (chunk type 0xfe).
286 - // Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
287 - if !r.readFull(r.buf[:chunkLen]) {
288 - return 0, r.err
289 - }
290 - }
291 - }
292 -}
Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/encode.go deleted
-258
@@ -1,258 +0,0 @@
1 -// Copyright 2011 The Snappy-Go Authors. All rights reserved.
2 -// Use of this source code is governed by a BSD-style
3 -// license that can be found in the LICENSE file.
4 -
5 -package snappy
6 -
7 -import (
8 - "encoding/binary"
9 - "io"
10 -)
11 -
12 -// We limit how far copy back-references can go, the same as the C++ code.
13 -const maxOffset = 1 << 15
14 -
15 -// emitLiteral writes a literal chunk and returns the number of bytes written.
16 -func emitLiteral(dst, lit []byte) int {
17 - i, n := 0, uint(len(lit)-1)
18 - switch {
19 - case n < 60:
20 - dst[0] = uint8(n)<<2 | tagLiteral
21 - i = 1
22 - case n < 1<<8:
23 - dst[0] = 60<<2 | tagLiteral
24 - dst[1] = uint8(n)
25 - i = 2
26 - case n < 1<<16:
27 - dst[0] = 61<<2 | tagLiteral
28 - dst[1] = uint8(n)
29 - dst[2] = uint8(n >> 8)
30 - i = 3
31 - case n < 1<<24:
32 - dst[0] = 62<<2 | tagLiteral
33 - dst[1] = uint8(n)
34 - dst[2] = uint8(n >> 8)
35 - dst[3] = uint8(n >> 16)
36 - i = 4
37 - case int64(n) < 1<<32:
38 - dst[0] = 63<<2 | tagLiteral
39 - dst[1] = uint8(n)
40 - dst[2] = uint8(n >> 8)
41 - dst[3] = uint8(n >> 16)
42 - dst[4] = uint8(n >> 24)
43 - i = 5
44 - default:
45 - panic("snappy: source buffer is too long")
46 - }
47 - if copy(dst[i:], lit) != len(lit) {
48 - panic("snappy: destination buffer is too short")
49 - }
50 - return i + len(lit)
51 -}
52 -
53 -// emitCopy writes a copy chunk and returns the number of bytes written.
54 -func emitCopy(dst []byte, offset, length int) int {
55 - i := 0
56 - for length > 0 {
57 - x := length - 4
58 - if 0 <= x && x < 1<<3 && offset < 1<<11 {
59 - dst[i+0] = uint8(offset>>8)&0x07<<5 | uint8(x)<<2 | tagCopy1
60 - dst[i+1] = uint8(offset)
61 - i += 2
62 - break
63 - }
64 -
65 - x = length
66 - if x > 1<<6 {
67 - x = 1 << 6
68 - }
69 - dst[i+0] = uint8(x-1)<<2 | tagCopy2
70 - dst[i+1] = uint8(offset)
71 - dst[i+2] = uint8(offset >> 8)
72 - i += 3
73 - length -= x
74 - }
75 - return i
76 -}
77 -
78 -// Encode returns the encoded form of src. The returned slice may be a sub-
79 -// slice of dst if dst was large enough to hold the entire encoded block.
80 -// Otherwise, a newly allocated slice will be returned.
81 -// It is valid to pass a nil dst.
82 -func Encode(dst, src []byte) ([]byte, error) {
83 - if n := MaxEncodedLen(len(src)); len(dst) < n {
84 - dst = make([]byte, n)
85 - }
86 -
87 - // The block starts with the varint-encoded length of the decompressed bytes.
88 - d := binary.PutUvarint(dst, uint64(len(src)))
89 -
90 - // Return early if src is short.
91 - if len(src) <= 4 {
92 - if len(src) != 0 {
93 - d += emitLiteral(dst[d:], src)
94 - }
95 - return dst[:d], nil
96 - }
97 -
98 - // Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive.
99 - const maxTableSize = 1 << 14
100 - shift, tableSize := uint(32-8), 1<<8
101 - for tableSize < maxTableSize && tableSize < len(src) {
102 - shift--
103 - tableSize *= 2
104 - }
105 - var table [maxTableSize]int
106 -
107 - // Iterate over the source bytes.
108 - var (
109 - s int // The iterator position.
110 - t int // The last position with the same hash as s.
111 - lit int // The start position of any pending literal bytes.
112 - )
113 - for s+3 < len(src) {
114 - // Update the hash table.
115 - b0, b1, b2, b3 := src[s], src[s+1], src[s+2], src[s+3]
116 - h := uint32(b0) | uint32(b1)<<8 | uint32(b2)<<16 | uint32(b3)<<24
117 - p := &table[(h*0x1e35a7bd)>>shift]
118 - // We need to to store values in [-1, inf) in table. To save
119 - // some initialization time, (re)use the table's zero value
120 - // and shift the values against this zero: add 1 on writes,
121 - // subtract 1 on reads.
122 - t, *p = *p-1, s+1
123 - // If t is invalid or src[s:s+4] differs from src[t:t+4], accumulate a literal byte.
124 - if t < 0 || s-t >= maxOffset || b0 != src[t] || b1 != src[t+1] || b2 != src[t+2] || b3 != src[t+3] {
125 - s++
126 - continue
127 - }
128 - // Otherwise, we have a match. First, emit any pending literal bytes.
129 - if lit != s {
130 - d += emitLiteral(dst[d:], src[lit:s])
131 - }
132 - // Extend the match to be as long as possible.
133 - s0 := s
134 - s, t = s+4, t+4
135 - for s < len(src) && src[s] == src[t] {
136 - s++
137 - t++
138 - }
139 - // Emit the copied bytes.
140 - d += emitCopy(dst[d:], s-t, s-s0)
141 - lit = s
142 - }
143 -
144 - // Emit any final pending literal bytes and return.
145 - if lit != len(src) {
146 - d += emitLiteral(dst[d:], src[lit:])
147 - }
148 - return dst[:d], nil
149 -}
150 -
151 -// MaxEncodedLen returns the maximum length of a snappy block, given its
152 -// uncompressed length.
153 -func MaxEncodedLen(srcLen int) int {
154 - // Compressed data can be defined as:
155 - // compressed := item* literal*
156 - // item := literal* copy
157 - //
158 - // The trailing literal sequence has a space blowup of at most 62/60
159 - // since a literal of length 60 needs one tag byte + one extra byte
160 - // for length information.
161 - //
162 - // Item blowup is trickier to measure. Suppose the "copy" op copies
163 - // 4 bytes of data. Because of a special check in the encoding code,
164 - // we produce a 4-byte copy only if the offset is < 65536. Therefore
165 - // the copy op takes 3 bytes to encode, and this type of item leads
166 - // to at most the 62/60 blowup for representing literals.
167 - //
168 - // Suppose the "copy" op copies 5 bytes of data. If the offset is big
169 - // enough, it will take 5 bytes to encode the copy op. Therefore the
170 - // worst case here is a one-byte literal followed by a five-byte copy.
171 - // That is, 6 bytes of input turn into 7 bytes of "compressed" data.
172 - //
173 - // This last factor dominates the blowup, so the final estimate is:
174 - return 32 + srcLen + srcLen/6
175 -}
176 -
177 -// NewWriter returns a new Writer that compresses to w, using the framing
178 -// format described at
179 -// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
180 -func NewWriter(w io.Writer) *Writer {
181 - return &Writer{
182 - w: w,
183 - enc: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)),
184 - }
185 -}
186 -
187 -// Writer is an io.Writer than can write Snappy-compressed bytes.
188 -type Writer struct {
189 - w io.Writer
190 - err error
191 - enc []byte
192 - buf [checksumSize + chunkHeaderSize]byte
193 - wroteHeader bool
194 -}
195 -
196 -// Reset discards the writer's state and switches the Snappy writer to write to
197 -// w. This permits reusing a Writer rather than allocating a new one.
198 -func (w *Writer) Reset(writer io.Writer) {
199 - w.w = writer
200 - w.err = nil
201 - w.wroteHeader = false
202 -}
203 -
204 -// Write satisfies the io.Writer interface.
205 -func (w *Writer) Write(p []byte) (n int, errRet error) {
206 - if w.err != nil {
207 - return 0, w.err
208 - }
209 - if !w.wroteHeader {
210 - copy(w.enc, magicChunk)
211 - if _, err := w.w.Write(w.enc[:len(magicChunk)]); err != nil {
212 - w.err = err
213 - return n, err
214 - }
215 - w.wroteHeader = true
216 - }
217 - for len(p) > 0 {
218 - var uncompressed []byte
219 - if len(p) > maxUncompressedChunkLen {
220 - uncompressed, p = p[:maxUncompressedChunkLen], p[maxUncompressedChunkLen:]
221 - } else {
222 - uncompressed, p = p, nil
223 - }
224 - checksum := crc(uncompressed)
225 -
226 - // Compress the buffer, discarding the result if the improvement
227 - // isn't at least 12.5%.
228 - chunkType := uint8(chunkTypeCompressedData)
229 - chunkBody, err := Encode(w.enc, uncompressed)
230 - if err != nil {
231 - w.err = err
232 - return n, err
233 - }
234 - if len(chunkBody) >= len(uncompressed)-len(uncompressed)/8 {
235 - chunkType, chunkBody = chunkTypeUncompressedData, uncompressed
236 - }
237 -
238 - chunkLen := 4 + len(chunkBody)
239 - w.buf[0] = chunkType
240 - w.buf[1] = uint8(chunkLen >> 0)
241 - w.buf[2] = uint8(chunkLen >> 8)
242 - w.buf[3] = uint8(chunkLen >> 16)
243 - w.buf[4] = uint8(checksum >> 0)
244 - w.buf[5] = uint8(checksum >> 8)
245 - w.buf[6] = uint8(checksum >> 16)
246 - w.buf[7] = uint8(checksum >> 24)
247 - if _, err = w.w.Write(w.buf[:]); err != nil {
248 - w.err = err
249 - return n, err
250 - }
251 - if _, err = w.w.Write(chunkBody); err != nil {
252 - w.err = err
253 - return n, err
254 - }
255 - n += len(uncompressed)
256 - }
257 - return n, nil
258 -}
Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy.go deleted
-68
@@ -1,68 +0,0 @@
1 -// Copyright 2011 The Snappy-Go Authors. All rights reserved.
2 -// Use of this source code is governed by a BSD-style
3 -// license that can be found in the LICENSE file.
4 -
5 -// Package snappy implements the snappy block-based compression format.
6 -// It aims for very high speeds and reasonable compression.
7 -//
8 -// The C++ snappy implementation is at http://code.google.com/p/snappy/
9 -package snappy
10 -
11 -import (
12 - "hash/crc32"
13 -)
14 -
15 -/*
16 -Each encoded block begins with the varint-encoded length of the decoded data,
17 -followed by a sequence of chunks. Chunks begin and end on byte boundaries. The
18 -first byte of each chunk is broken into its 2 least and 6 most significant bits
19 -called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag.
20 -Zero means a literal tag. All other values mean a copy tag.
21 -
22 -For literal tags:
23 - - If m < 60, the next 1 + m bytes are literal bytes.
24 - - Otherwise, let n be the little-endian unsigned integer denoted by the next
25 - m - 59 bytes. The next 1 + n bytes after that are literal bytes.
26 -
27 -For copy tags, length bytes are copied from offset bytes ago, in the style of
28 -Lempel-Ziv compression algorithms. In particular:
29 - - For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12).
30 - The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10
31 - of the offset. The next byte is bits 0-7 of the offset.
32 - - For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65).
33 - The length is 1 + m. The offset is the little-endian unsigned integer
34 - denoted by the next 2 bytes.
35 - - For l == 3, this tag is a legacy format that is no longer supported.
36 -*/
37 -const (
38 - tagLiteral = 0x00
39 - tagCopy1 = 0x01
40 - tagCopy2 = 0x02
41 - tagCopy4 = 0x03
42 -)
43 -
44 -const (
45 - checksumSize = 4
46 - chunkHeaderSize = 4
47 - magicChunk = "\xff\x06\x00\x00" + magicBody
48 - magicBody = "sNaPpY"
49 - // https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt says
50 - // that "the uncompressed data in a chunk must be no longer than 65536 bytes".
51 - maxUncompressedChunkLen = 65536
52 -)
53 -
54 -const (
55 - chunkTypeCompressedData = 0x00
56 - chunkTypeUncompressedData = 0x01
57 - chunkTypePadding = 0xfe
58 - chunkTypeStreamIdentifier = 0xff
59 -)
60 -
61 -var crcTable = crc32.MakeTable(crc32.Castagnoli)
62 -
63 -// crc implements the checksum specified in section 3 of
64 -// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
65 -func crc(b []byte) uint32 {
66 - c := crc32.Update(0, crcTable, b)
67 - return uint32(c>>15|c<<17) + 0xa282ead8
68 -}
Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy_test.go deleted
-364
@@ -1,364 +0,0 @@
1 -// Copyright 2011 The Snappy-Go Authors. All rights reserved.
2 -// Use of this source code is governed by a BSD-style
3 -// license that can be found in the LICENSE file.
4 -
5 -package snappy
6 -
7 -import (
8 - "bytes"
9 - "flag"
10 - "fmt"
11 - "io"
12 - "io/ioutil"
13 - "math/rand"
14 - "net/http"
15 - "os"
16 - "path/filepath"
17 - "strings"
18 - "testing"
19 -)
20 -
21 -var (
22 - download = flag.Bool("download", false, "If true, download any missing files before running benchmarks")
23 - testdata = flag.String("testdata", "testdata", "Directory containing the test data")
24 -)
25 -
26 -func roundtrip(b, ebuf, dbuf []byte) error {
27 - e, err := Encode(ebuf, b)
28 - if err != nil {
29 - return fmt.Errorf("encoding error: %v", err)
30 - }
31 - d, err := Decode(dbuf, e)
32 - if err != nil {
33 - return fmt.Errorf("decoding error: %v", err)
34 - }
35 - if !bytes.Equal(b, d) {
36 - return fmt.Errorf("roundtrip mismatch:\n\twant %v\n\tgot %v", b, d)
37 - }
38 - return nil
39 -}
40 -
41 -func TestEmpty(t *testing.T) {
42 - if err := roundtrip(nil, nil, nil); err != nil {
43 - t.Fatal(err)
44 - }
45 -}
46 -
47 -func TestSmallCopy(t *testing.T) {
48 - for _, ebuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} {
49 - for _, dbuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} {
50 - for i := 0; i < 32; i++ {
51 - s := "aaaa" + strings.Repeat("b", i) + "aaaabbbb"
52 - if err := roundtrip([]byte(s), ebuf, dbuf); err != nil {
53 - t.Errorf("len(ebuf)=%d, len(dbuf)=%d, i=%d: %v", len(ebuf), len(dbuf), i, err)
54 - }
55 - }
56 - }
57 - }
58 -}
59 -
60 -func TestSmallRand(t *testing.T) {
61 - rng := rand.New(rand.NewSource(27354294))
62 - for n := 1; n < 20000; n += 23 {
63 - b := make([]byte, n)
64 - for i := range b {
65 - b[i] = uint8(rng.Uint32())
66 - }
67 - if err := roundtrip(b, nil, nil); err != nil {
68 - t.Fatal(err)
69 - }
70 - }
71 -}
72 -
73 -func TestSmallRegular(t *testing.T) {
74 - for n := 1; n < 20000; n += 23 {
75 - b := make([]byte, n)
76 - for i := range b {
77 - b[i] = uint8(i%10 + 'a')
78 - }
79 - if err := roundtrip(b, nil, nil); err != nil {
80 - t.Fatal(err)
81 - }
82 - }
83 -}
84 -
85 -func cmp(a, b []byte) error {
86 - if len(a) != len(b) {
87 - return fmt.Errorf("got %d bytes, want %d", len(a), len(b))
88 - }
89 - for i := range a {
90 - if a[i] != b[i] {
91 - return fmt.Errorf("byte #%d: got 0x%02x, want 0x%02x", i, a[i], b[i])
92 - }
93 - }
94 - return nil
95 -}
96 -
97 -func TestFramingFormat(t *testing.T) {
98 - // src is comprised of alternating 1e5-sized sequences of random
99 - // (incompressible) bytes and repeated (compressible) bytes. 1e5 was chosen
100 - // because it is larger than maxUncompressedChunkLen (64k).
101 - src := make([]byte, 1e6)
102 - rng := rand.New(rand.NewSource(1))
103 - for i := 0; i < 10; i++ {
104 - if i%2 == 0 {
105 - for j := 0; j < 1e5; j++ {
106 - src[1e5*i+j] = uint8(rng.Intn(256))
107 - }
108 - } else {
109 - for j := 0; j < 1e5; j++ {
110 - src[1e5*i+j] = uint8(i)
111 - }
112 - }
113 - }
114 -
115 - buf := new(bytes.Buffer)
116 - if _, err := NewWriter(buf).Write(src); err != nil {
117 - t.Fatalf("Write: encoding: %v", err)
118 - }
119 - dst, err := ioutil.ReadAll(NewReader(buf))
120 - if err != nil {
121 - t.Fatalf("ReadAll: decoding: %v", err)
122 - }
123 - if err := cmp(dst, src); err != nil {
124 - t.Fatal(err)
125 - }
126 -}
127 -
128 -func TestReaderReset(t *testing.T) {
129 - gold := bytes.Repeat([]byte("All that is gold does not glitter,\n"), 10000)
130 - buf := new(bytes.Buffer)
131 - if _, err := NewWriter(buf).Write(gold); err != nil {
132 - t.Fatalf("Write: %v", err)
133 - }
134 - encoded, invalid, partial := buf.String(), "invalid", "partial"
135 - r := NewReader(nil)
136 - for i, s := range []string{encoded, invalid, partial, encoded, partial, invalid, encoded, encoded} {
137 - if s == partial {
138 - r.Reset(strings.NewReader(encoded))
139 - if _, err := r.Read(make([]byte, 101)); err != nil {
140 - t.Errorf("#%d: %v", i, err)
141 - continue
142 - }
143 - continue
144 - }
145 - r.Reset(strings.NewReader(s))
146 - got, err := ioutil.ReadAll(r)
147 - switch s {
148 - case encoded:
149 - if err != nil {
150 - t.Errorf("#%d: %v", i, err)
151 - continue
152 - }
153 - if err := cmp(got, gold); err != nil {
154 - t.Errorf("#%d: %v", i, err)
155 - continue
156 - }
157 - case invalid:
158 - if err == nil {
159 - t.Errorf("#%d: got nil error, want non-nil", i)
160 - continue
161 - }
162 - }
163 - }
164 -}
165 -
166 -func TestWriterReset(t *testing.T) {
167 - gold := bytes.Repeat([]byte("Not all those who wander are lost;\n"), 10000)
168 - var gots, wants [][]byte
169 - const n = 20
170 - w, failed := NewWriter(nil), false
171 - for i := 0; i <= n; i++ {
172 - buf := new(bytes.Buffer)
173 - w.Reset(buf)
174 - want := gold[:len(gold)*i/n]
175 - if _, err := w.Write(want); err != nil {
176 - t.Errorf("#%d: Write: %v", i, err)
177 - failed = true
178 - continue
179 - }
180 - got, err := ioutil.ReadAll(NewReader(buf))
181 - if err != nil {
182 - t.Errorf("#%d: ReadAll: %v", i, err)
183 - failed = true
184 - continue
185 - }
186 - gots = append(gots, got)
187 - wants = append(wants, want)
188 - }
189 - if failed {
190 - return
191 - }
192 - for i := range gots {
193 - if err := cmp(gots[i], wants[i]); err != nil {
194 - t.Errorf("#%d: %v", i, err)
195 - }
196 - }
197 -}
198 -
199 -func benchDecode(b *testing.B, src []byte) {
200 - encoded, err := Encode(nil, src)
201 - if err != nil {
202 - b.Fatal(err)
203 - }
204 - // Bandwidth is in amount of uncompressed data.
205 - b.SetBytes(int64(len(src)))
206 - b.ResetTimer()
207 - for i := 0; i < b.N; i++ {
208 - Decode(src, encoded)
209 - }
210 -}
211 -
212 -func benchEncode(b *testing.B, src []byte) {
213 - // Bandwidth is in amount of uncompressed data.
214 - b.SetBytes(int64(len(src)))
215 - dst := make([]byte, MaxEncodedLen(len(src)))
216 - b.ResetTimer()
217 - for i := 0; i < b.N; i++ {
218 - Encode(dst, src)
219 - }
220 -}
221 -
222 -func readFile(b testing.TB, filename string) []byte {
223 - src, err := ioutil.ReadFile(filename)
224 - if err != nil {
225 - b.Fatalf("failed reading %s: %s", filename, err)
226 - }
227 - if len(src) == 0 {
228 - b.Fatalf("%s has zero length", filename)
229 - }
230 - return src
231 -}
232 -
233 -// expand returns a slice of length n containing repeated copies of src.
234 -func expand(src []byte, n int) []byte {
235 - dst := make([]byte, n)
236 - for x := dst; len(x) > 0; {
237 - i := copy(x, src)
238 - x = x[i:]
239 - }
240 - return dst
241 -}
242 -
243 -func benchWords(b *testing.B, n int, decode bool) {
244 - // NOTE: The file is OS-language dependent so the resulting values are not
245 - // directly comparable for non-US-English OS installations.
246 - data := expand(readFile(b, "/usr/share/dict/words"), n)
247 - if decode {
248 - benchDecode(b, data)
249 - } else {
250 - benchEncode(b, data)
251 - }
252 -}
253 -
254 -func BenchmarkWordsDecode1e3(b *testing.B) { benchWords(b, 1e3, true) }
255 -func BenchmarkWordsDecode1e4(b *testing.B) { benchWords(b, 1e4, true) }
256 -func BenchmarkWordsDecode1e5(b *testing.B) { benchWords(b, 1e5, true) }
257 -func BenchmarkWordsDecode1e6(b *testing.B) { benchWords(b, 1e6, true) }
258 -func BenchmarkWordsEncode1e3(b *testing.B) { benchWords(b, 1e3, false) }
259 -func BenchmarkWordsEncode1e4(b *testing.B) { benchWords(b, 1e4, false) }
260 -func BenchmarkWordsEncode1e5(b *testing.B) { benchWords(b, 1e5, false) }
261 -func BenchmarkWordsEncode1e6(b *testing.B) { benchWords(b, 1e6, false) }
262 -
263 -// testFiles' values are copied directly from
264 -// https://raw.githubusercontent.com/google/snappy/master/snappy_unittest.cc
265 -// The label field is unused in snappy-go.
266 -var testFiles = []struct {
267 - label string
268 - filename string
269 -}{
270 - {"html", "html"},
271 - {"urls", "urls.10K"},
272 - {"jpg", "fireworks.jpeg"},
273 - {"jpg_200", "fireworks.jpeg"},
274 - {"pdf", "paper-100k.pdf"},
275 - {"html4", "html_x_4"},
276 - {"txt1", "alice29.txt"},
277 - {"txt2", "asyoulik.txt"},
278 - {"txt3", "lcet10.txt"},
279 - {"txt4", "plrabn12.txt"},
280 - {"pb", "geo.protodata"},
281 - {"gaviota", "kppkn.gtb"},
282 -}
283 -
284 -// The test data files are present at this canonical URL.
285 -const baseURL = "https://raw.githubusercontent.com/google/snappy/master/testdata/"
286 -
287 -func downloadTestdata(basename string) (errRet error) {
288 - filename := filepath.Join(*testdata, basename)
289 - if stat, err := os.Stat(filename); err == nil && stat.Size() != 0 {
290 - return nil
291 - }
292 -
293 - if !*download {
294 - return fmt.Errorf("test data not found; skipping benchmark without the -download flag")
295 - }
296 - // Download the official snappy C++ implementation reference test data
297 - // files for benchmarking.
298 - if err := os.Mkdir(*testdata, 0777); err != nil && !os.IsExist(err) {
299 - return fmt.Errorf("failed to create testdata: %s", err)
300 - }
301 -
302 - f, err := os.Create(filename)
303 - if err != nil {
304 - return fmt.Errorf("failed to create %s: %s", filename, err)
305 - }
306 - defer f.Close()
307 - defer func() {
308 - if errRet != nil {
309 - os.Remove(filename)
310 - }
311 - }()
312 - url := baseURL + basename
313 - resp, err := http.Get(url)
314 - if err != nil {
315 - return fmt.Errorf("failed to download %s: %s", url, err)
316 - }
317 - defer resp.Body.Close()
318 - if s := resp.StatusCode; s != http.StatusOK {
319 - return fmt.Errorf("downloading %s: HTTP status code %d (%s)", url, s, http.StatusText(s))
320 - }
321 - _, err = io.Copy(f, resp.Body)
322 - if err != nil {
323 - return fmt.Errorf("failed to download %s to %s: %s", url, filename, err)
324 - }
325 - return nil
326 -}
327 -
328 -func benchFile(b *testing.B, n int, decode bool) {
329 - if err := downloadTestdata(testFiles[n].filename); err != nil {
330 - b.Fatalf("failed to download testdata: %s", err)
331 - }
332 - data := readFile(b, filepath.Join(*testdata, testFiles[n].filename))
333 - if decode {
334 - benchDecode(b, data)
335 - } else {
336 - benchEncode(b, data)
337 - }
338 -}
339 -
340 -// Naming convention is kept similar to what snappy's C++ implementation uses.
341 -func Benchmark_UFlat0(b *testing.B) { benchFile(b, 0, true) }
342 -func Benchmark_UFlat1(b *testing.B) { benchFile(b, 1, true) }
343 -func Benchmark_UFlat2(b *testing.B) { benchFile(b, 2, true) }
344 -func Benchmark_UFlat3(b *testing.B) { benchFile(b, 3, true) }
345 -func Benchmark_UFlat4(b *testing.B) { benchFile(b, 4, true) }
346 -func Benchmark_UFlat5(b *testing.B) { benchFile(b, 5, true) }
347 -func Benchmark_UFlat6(b *testing.B) { benchFile(b, 6, true) }
348 -func Benchmark_UFlat7(b *testing.B) { benchFile(b, 7, true) }
349 -func Benchmark_UFlat8(b *testing.B) { benchFile(b, 8, true) }
350 -func Benchmark_UFlat9(b *testing.B) { benchFile(b, 9, true) }
351 -func Benchmark_UFlat10(b *testing.B) { benchFile(b, 10, true) }
352 -func Benchmark_UFlat11(b *testing.B) { benchFile(b, 11, true) }
353 -func Benchmark_ZFlat0(b *testing.B) { benchFile(b, 0, false) }
354 -func Benchmark_ZFlat1(b *testing.B) { benchFile(b, 1, false) }
355 -func Benchmark_ZFlat2(b *testing.B) { benchFile(b, 2, false) }
356 -func Benchmark_ZFlat3(b *testing.B) { benchFile(b, 3, false) }
357 -func Benchmark_ZFlat4(b *testing.B) { benchFile(b, 4, false) }
358 -func Benchmark_ZFlat5(b *testing.B) { benchFile(b, 5, false) }
359 -func Benchmark_ZFlat6(b *testing.B) { benchFile(b, 6, false) }
360 -func Benchmark_ZFlat7(b *testing.B) { benchFile(b, 7, false) }
361 -func Benchmark_ZFlat8(b *testing.B) { benchFile(b, 8, false) }
362 -func Benchmark_ZFlat9(b *testing.B) { benchFile(b, 9, false) }
363 -func Benchmark_ZFlat10(b *testing.B) { benchFile(b, 10, false) }
364 -func Benchmark_ZFlat11(b *testing.B) { benchFile(b, 11, false) }