vendor aws, s3
vendor: goamz/aws and goamz/s3
Brian Tiger Chow committed
Feb 4, 2015 at 15:32 UTC
bcfa539272c3ed9850a4c705e9a862564c3eb314
25 files changed
+7569
Godeps/Godeps.json
+8
@@ -80,6 +80,14 @@
80
"ImportPath": "github.com/coreos/go-semver/semver",
81
"Rev": "6fe83ccda8fb9b7549c9ab4ba47f47858bc950aa"
82
},
83
+ {
84
+ "ImportPath": "github.com/crowdmob/goamz/aws",
85
+ "Rev": "82345796204222aa56be89cf930c316b1297f906"
86
+ },
87
+ {
88
+ "ImportPath": "github.com/crowdmob/goamz/s3",
89
+ "Rev": "82345796204222aa56be89cf930c316b1297f906"
90
+ },
91
{
92
"ImportPath": "github.com/dustin/go-humanize",
93
"Rev": "b198514c204f20799b91c93b6ffd8b26be04c2c9"
Godeps/_workspace/src/github.com/crowdmob/goamz/aws/attempt.go
new
+74
@@ -0,0 +1,74 @@
1
+package aws
2
+
3
+import (
4
+ "time"
5
+)
6
+
7
+// AttemptStrategy represents a strategy for waiting for an action
8
+// to complete successfully. This is an internal type used by the
9
+// implementation of other goamz packages.
10
+type AttemptStrategy struct {
11
+ Total time.Duration // total duration of attempt.
12
+ Delay time.Duration // interval between each try in the burst.
13
+ Min int // minimum number of retries; overrides Total
14
+}
15
+
16
+type Attempt struct {
17
+ strategy AttemptStrategy
18
+ last time.Time
19
+ end time.Time
20
+ force bool
21
+ count int
22
+}
23
+
24
+// Start begins a new sequence of attempts for the given strategy.
25
+func (s AttemptStrategy) Start() *Attempt {
26
+ now := time.Now()
27
+ return &Attempt{
28
+ strategy: s,
29
+ last: now,
30
+ end: now.Add(s.Total),
31
+ force: true,
32
+ }
33
+}
34
+
35
+// Next waits until it is time to perform the next attempt or returns
36
+// false if it is time to stop trying.
37
+func (a *Attempt) Next() bool {
38
+ now := time.Now()
39
+ sleep := a.nextSleep(now)
40
+ if !a.force && !now.Add(sleep).Before(a.end) && a.strategy.Min <= a.count {
41
+ return false
42
+ }
43
+ a.force = false
44
+ if sleep > 0 && a.count > 0 {
45
+ time.Sleep(sleep)
46
+ now = time.Now()
47
+ }
48
+ a.count++
49
+ a.last = now
50
+ return true
51
+}
52
+
53
+func (a *Attempt) nextSleep(now time.Time) time.Duration {
54
+ sleep := a.strategy.Delay - now.Sub(a.last)
55
+ if sleep < 0 {
56
+ return 0
57
+ }
58
+ return sleep
59
+}
60
+
61
+// HasNext returns whether another attempt will be made if the current
62
+// one fails. If it returns true, the following call to Next is
63
+// guaranteed to return true.
64
+func (a *Attempt) HasNext() bool {
65
+ if a.force || a.strategy.Min > a.count {
66
+ return true
67
+ }
68
+ now := time.Now()
69
+ if now.Add(a.nextSleep(now)).Before(a.end) {
70
+ a.force = true
71
+ return true
72
+ }
73
+ return false
74
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/aws/attempt_test.go
new
+57
@@ -0,0 +1,57 @@
1
+package aws_test
2
+
3
+import (
4
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/aws"
5
+ "gopkg.in/check.v1"
6
+ "time"
7
+)
8
+
9
+func (S) TestAttemptTiming(c *check.C) {
10
+ testAttempt := aws.AttemptStrategy{
11
+ Total: 0.25e9,
12
+ Delay: 0.1e9,
13
+ }
14
+ want := []time.Duration{0, 0.1e9, 0.2e9, 0.2e9}
15
+ got := make([]time.Duration, 0, len(want)) // avoid allocation when testing timing
16
+ t0 := time.Now()
17
+ for a := testAttempt.Start(); a.Next(); {
18
+ got = append(got, time.Now().Sub(t0))
19
+ }
20
+ got = append(got, time.Now().Sub(t0))
21
+ c.Assert(got, check.HasLen, len(want))
22
+ const margin = 0.01e9
23
+ for i, got := range want {
24
+ lo := want[i] - margin
25
+ hi := want[i] + margin
26
+ if got < lo || got > hi {
27
+ c.Errorf("attempt %d want %g got %g", i, want[i].Seconds(), got.Seconds())
28
+ }
29
+ }
30
+}
31
+
32
+func (S) TestAttemptNextHasNext(c *check.C) {
33
+ a := aws.AttemptStrategy{}.Start()
34
+ c.Assert(a.Next(), check.Equals, true)
35
+ c.Assert(a.Next(), check.Equals, false)
36
+
37
+ a = aws.AttemptStrategy{}.Start()
38
+ c.Assert(a.Next(), check.Equals, true)
39
+ c.Assert(a.HasNext(), check.Equals, false)
40
+ c.Assert(a.Next(), check.Equals, false)
41
+
42
+ a = aws.AttemptStrategy{Total: 2e8}.Start()
43
+ c.Assert(a.Next(), check.Equals, true)
44
+ c.Assert(a.HasNext(), check.Equals, true)
45
+ time.Sleep(2e8)
46
+ c.Assert(a.HasNext(), check.Equals, true)
47
+ c.Assert(a.Next(), check.Equals, true)
48
+ c.Assert(a.Next(), check.Equals, false)
49
+
50
+ a = aws.AttemptStrategy{Total: 1e8, Min: 2}.Start()
51
+ time.Sleep(1e8)
52
+ c.Assert(a.Next(), check.Equals, true)
53
+ c.Assert(a.HasNext(), check.Equals, true)
54
+ c.Assert(a.Next(), check.Equals, true)
55
+ c.Assert(a.HasNext(), check.Equals, false)
56
+ c.Assert(a.Next(), check.Equals, false)
57
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/aws/aws.go
new
+624
@@ -0,0 +1,624 @@
1
+//
2
+// goamz - Go packages to interact with the Amazon Web Services.
3
+//
4
+// https://wiki.ubuntu.com/goamz
5
+//
6
+// Copyright (c) 2011 Canonical Ltd.
7
+//
8
+// Written by Gustavo Niemeyer <gustavo.niemeyer@canonical.com>
9
+//
10
+package aws
11
+
12
+import (
13
+ "encoding/json"
14
+ "encoding/xml"
15
+ "errors"
16
+ "fmt"
17
+ "io/ioutil"
18
+ "net"
19
+ "net/http"
20
+ "net/url"
21
+ "os"
22
+ "os/user"
23
+ "path"
24
+ "regexp"
25
+ "strings"
26
+ "time"
27
+)
28
+
29
+// Regular expressions for INI files
30
+var (
31
+ iniSectionRegexp = regexp.MustCompile(`^\s*\[([^\[\]]+)\]\s*$`)
32
+ iniSettingRegexp = regexp.MustCompile(`^\s*(.+?)\s*=\s*(.*\S)\s*$`)
33
+)
34
+
35
+// Defines the valid signers
36
+const (
37
+ V2Signature = iota
38
+ V4Signature = iota
39
+ Route53Signature = iota
40
+)
41
+
42
+// Defines the service endpoint and correct Signer implementation to use
43
+// to sign requests for this endpoint
44
+type ServiceInfo struct {
45
+ Endpoint string
46
+ Signer uint
47
+}
48
+
49
+// Region defines the URLs where AWS services may be accessed.
50
+//
51
+// See http://goo.gl/d8BP1 for more details.
52
+type Region struct {
53
+ Name string // the canonical name of this region.
54
+ EC2Endpoint string
55
+ S3Endpoint string
56
+ S3BucketEndpoint string // Not needed by AWS S3. Use ${bucket} for bucket name.
57
+ S3LocationConstraint bool // true if this region requires a LocationConstraint declaration.
58
+ S3LowercaseBucket bool // true if the region requires bucket names to be lower case.
59
+ SDBEndpoint string
60
+ SNSEndpoint string
61
+ SQSEndpoint string
62
+ SESEndpoint string
63
+ IAMEndpoint string
64
+ ELBEndpoint string
65
+ DynamoDBEndpoint string
66
+ CloudWatchServicepoint ServiceInfo
67
+ AutoScalingEndpoint string
68
+ RDSEndpoint ServiceInfo
69
+ KinesisEndpoint string
70
+ STSEndpoint string
71
+ CloudFormationEndpoint string
72
+ ElastiCacheEndpoint string
73
+}
74
+
75
+var Regions = map[string]Region{
76
+ APNortheast.Name: APNortheast,
77
+ APSoutheast.Name: APSoutheast,
78
+ APSoutheast2.Name: APSoutheast2,
79
+ EUCentral.Name: EUCentral,
80
+ EUWest.Name: EUWest,
81
+ USEast.Name: USEast,
82
+ USWest.Name: USWest,
83
+ USWest2.Name: USWest2,
84
+ USGovWest.Name: USGovWest,
85
+ SAEast.Name: SAEast,
86
+}
87
+
88
+// Designates a signer interface suitable for signing AWS requests, params
89
+// should be appropriately encoded for the request before signing.
90
+//
91
+// A signer should be initialized with Auth and the appropriate endpoint.
92
+type Signer interface {
93
+ Sign(method, path string, params map[string]string)
94
+}
95
+
96
+// An AWS Service interface with the API to query the AWS service
97
+//
98
+// Supplied as an easy way to mock out service calls during testing.
99
+type AWSService interface {
100
+ // Queries the AWS service at a given method/path with the params and
101
+ // returns an http.Response and error
102
+ Query(method, path string, params map[string]string) (*http.Response, error)
103
+ // Builds an error given an XML payload in the http.Response, can be used
104
+ // to process an error if the status code is not 200 for example.
105
+ BuildError(r *http.Response) error
106
+}
107
+
108
+// Implements a Server Query/Post API to easily query AWS services and build
109
+// errors when desired
110
+type Service struct {
111
+ service ServiceInfo
112
+ signer Signer
113
+}
114
+
115
+// Create a base set of params for an action
116
+func MakeParams(action string) map[string]string {
117
+ params := make(map[string]string)
118
+ params["Action"] = action
119
+ return params
120
+}
121
+
122
+// Create a new AWS server to handle making requests
123
+func NewService(auth Auth, service ServiceInfo) (s *Service, err error) {
124
+ var signer Signer
125
+ switch service.Signer {
126
+ case V2Signature:
127
+ signer, err = NewV2Signer(auth, service)
128
+ // case V4Signature:
129
+ // signer, err = NewV4Signer(auth, service, Regions["eu-west-1"])
130
+ default:
131
+ err = fmt.Errorf("Unsupported signer for service")
132
+ }
133
+ if err != nil {
134
+ return
135
+ }
136
+ s = &Service{service: service, signer: signer}
137
+ return
138
+}
139
+
140
+func (s *Service) Query(method, path string, params map[string]string) (resp *http.Response, err error) {
141
+ params["Timestamp"] = time.Now().UTC().Format(time.RFC3339)
142
+ u, err := url.Parse(s.service.Endpoint)
143
+ if err != nil {
144
+ return nil, err
145
+ }
146
+ u.Path = path
147
+
148
+ s.signer.Sign(method, path, params)
149
+ if method == "GET" {
150
+ u.RawQuery = multimap(params).Encode()
151
+ resp, err = http.Get(u.String())
152
+ } else if method == "POST" {
153
+ resp, err = http.PostForm(u.String(), multimap(params))
154
+ }
155
+
156
+ return
157
+}
158
+
159
+func (s *Service) BuildError(r *http.Response) error {
160
+ errors := ErrorResponse{}
161
+ xml.NewDecoder(r.Body).Decode(&errors)
162
+ var err Error
163
+ err = errors.Errors
164
+ err.RequestId = errors.RequestId
165
+ err.StatusCode = r.StatusCode
166
+ if err.Message == "" {
167
+ err.Message = r.Status
168
+ }
169
+ return &err
170
+}
171
+
172
+type ServiceError interface {
173
+ error
174
+ ErrorCode() string
175
+}
176
+
177
+type ErrorResponse struct {
178
+ Errors Error `xml:"Error"`
179
+ RequestId string // A unique ID for tracking the request
180
+}
181
+
182
+type Error struct {
183
+ StatusCode int
184
+ Type string
185
+ Code string
186
+ Message string
187
+ RequestId string
188
+}
189
+
190
+func (err *Error) Error() string {
191
+ return fmt.Sprintf("Type: %s, Code: %s, Message: %s",
192
+ err.Type, err.Code, err.Message,
193
+ )
194
+}
195
+
196
+func (err *Error) ErrorCode() string {
197
+ return err.Code
198
+}
199
+
200
+type Auth struct {
201
+ AccessKey, SecretKey string
202
+ token string
203
+ expiration time.Time
204
+}
205
+
206
+func (a *Auth) Token() string {
207
+ if a.token == "" {
208
+ return ""
209
+ }
210
+ if time.Since(a.expiration) >= -30*time.Second { //in an ideal world this should be zero assuming the instance is synching it's clock
211
+ *a, _ = GetAuth("", "", "", time.Time{})
212
+ }
213
+ return a.token
214
+}
215
+
216
+func (a *Auth) Expiration() time.Time {
217
+ return a.expiration
218
+}
219
+
220
+// To be used with other APIs that return auth credentials such as STS
221
+func NewAuth(accessKey, secretKey, token string, expiration time.Time) *Auth {
222
+ return &Auth{
223
+ AccessKey: accessKey,
224
+ SecretKey: secretKey,
225
+ token: token,
226
+ expiration: expiration,
227
+ }
228
+}
229
+
230
+// ResponseMetadata
231
+type ResponseMetadata struct {
232
+ RequestId string // A unique ID for tracking the request
233
+}
234
+
235
+type BaseResponse struct {
236
+ ResponseMetadata ResponseMetadata
237
+}
238
+
239
+var unreserved = make([]bool, 128)
240
+var hex = "0123456789ABCDEF"
241
+
242
+func init() {
243
+ // RFC3986
244
+ u := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01234567890-_.~"
245
+ for _, c := range u {
246
+ unreserved[c] = true
247
+ }
248
+}
249
+
250
+func multimap(p map[string]string) url.Values {
251
+ q := make(url.Values, len(p))
252
+ for k, v := range p {
253
+ q[k] = []string{v}
254
+ }
255
+ return q
256
+}
257
+
258
+type credentials struct {
259
+ Code string
260
+ LastUpdated string
261
+ Type string
262
+ AccessKeyId string
263
+ SecretAccessKey string
264
+ Token string
265
+ Expiration string
266
+}
267
+
268
+// GetMetaData retrieves instance metadata about the current machine.
269
+//
270
+// See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AESDG-chapter-instancedata.html for more details.
271
+func GetMetaData(path string) (contents []byte, err error) {
272
+ c := http.Client{
273
+ Transport: &http.Transport{
274
+ Dial: func(netw, addr string) (net.Conn, error) {
275
+ deadline := time.Now().Add(5 * time.Second)
276
+ c, err := net.DialTimeout(netw, addr, time.Second*2)
277
+ if err != nil {
278
+ return nil, err
279
+ }
280
+ c.SetDeadline(deadline)
281
+ return c, nil
282
+ },
283
+ },
284
+ }
285
+
286
+ url := "http://169.254.169.254/latest/meta-data/" + path
287
+
288
+ resp, err := c.Get(url)
289
+ if err != nil {
290
+ return
291
+ }
292
+ defer resp.Body.Close()
293
+
294
+ if resp.StatusCode != 200 {
295
+ err = fmt.Errorf("Code %d returned for url %s", resp.StatusCode, url)
296
+ return
297
+ }
298
+
299
+ body, err := ioutil.ReadAll(resp.Body)
300
+ if err != nil {
301
+ return
302
+ }
303
+ return []byte(body), err
304
+}
305
+
306
+func GetRegion(regionName string) (region Region) {
307
+ region = Regions[regionName]
308
+ return
309
+}
310
+
311
+// GetInstanceCredentials creates an Auth based on the instance's role credentials.
312
+// If the running instance is not in EC2 or does not have a valid IAM role, an error will be returned.
313
+// For more info about setting up IAM roles, see http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
314
+func GetInstanceCredentials() (cred credentials, err error) {
315
+ credentialPath := "iam/security-credentials/"
316
+
317
+ // Get the instance role
318
+ role, err := GetMetaData(credentialPath)
319
+ if err != nil {
320
+ return
321
+ }
322
+
323
+ // Get the instance role credentials
324
+ credentialJSON, err := GetMetaData(credentialPath + string(role))
325
+ if err != nil {
326
+ return
327
+ }
328
+
329
+ err = json.Unmarshal([]byte(credentialJSON), &cred)
330
+ return
331
+}
332
+
333
+// GetAuth creates an Auth based on either passed in credentials,
334
+// environment information or instance based role credentials.
335
+func GetAuth(accessKey string, secretKey, token string, expiration time.Time) (auth Auth, err error) {
336
+ // First try passed in credentials
337
+ if accessKey != "" && secretKey != "" {
338
+ return Auth{accessKey, secretKey, token, expiration}, nil
339
+ }
340
+
341
+ // Next try to get auth from the environment
342
+ auth, err = EnvAuth()
343
+ if err == nil {
344
+ // Found auth, return
345
+ return
346
+ }
347
+
348
+ // Next try getting auth from the instance role
349
+ cred, err := GetInstanceCredentials()
350
+ if err == nil {
351
+ // Found auth, return
352
+ auth.AccessKey = cred.AccessKeyId
353
+ auth.SecretKey = cred.SecretAccessKey
354
+ auth.token = cred.Token
355
+ exptdate, err := time.Parse("2006-01-02T15:04:05Z", cred.Expiration)
356
+ if err != nil {
357
+ err = fmt.Errorf("Error Parsing expiration date: cred.Expiration :%s , error: %s \n", cred.Expiration, err)
358
+ }
359
+ auth.expiration = exptdate
360
+ return auth, err
361
+ }
362
+
363
+ // Next try getting auth from the credentials file
364
+ auth, err = CredentialFileAuth("", "", time.Minute*5)
365
+ if err == nil {
366
+ return
367
+ }
368
+
369
+ //err = errors.New("No valid AWS authentication found")
370
+ err = fmt.Errorf("No valid AWS authentication found: %s", err)
371
+ return auth, err
372
+}
373
+
374
+// EnvAuth creates an Auth based on environment information.
375
+// The AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment
376
+// variables are used.
377
+func EnvAuth() (auth Auth, err error) {
378
+ auth.AccessKey = os.Getenv("AWS_ACCESS_KEY_ID")
379
+ if auth.AccessKey == "" {
380
+ auth.AccessKey = os.Getenv("AWS_ACCESS_KEY")
381
+ }
382
+
383
+ auth.SecretKey = os.Getenv("AWS_SECRET_ACCESS_KEY")
384
+ if auth.SecretKey == "" {
385
+ auth.SecretKey = os.Getenv("AWS_SECRET_KEY")
386
+ }
387
+ if auth.AccessKey == "" {
388
+ err = errors.New("AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY not found in environment")
389
+ }
390
+ if auth.SecretKey == "" {
391
+ err = errors.New("AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY not found in environment")
392
+ }
393
+ return
394
+}
395
+
396
+// CredentialFileAuth creates and Auth based on a credentials file. The file
397
+// contains various authentication profiles for use with AWS.
398
+//
399
+// The credentials file, which is used by other AWS SDKs, is documented at
400
+// http://blogs.aws.amazon.com/security/post/Tx3D6U6WSFGOK2H/A-New-and-Standardized-Way-to-Manage-Credentials-in-the-AWS-SDKs
401
+func CredentialFileAuth(filePath string, profile string, expiration time.Duration) (auth Auth, err error) {
402
+ if profile == "" {
403
+ profile = "default"
404
+ }
405
+
406
+ if filePath == "" {
407
+ u, err := user.Current()
408
+ if err != nil {
409
+ return auth, err
410
+ }
411
+
412
+ filePath = path.Join(u.HomeDir, ".aws", "credentials")
413
+ }
414
+
415
+ // read the file, then parse the INI
416
+ contents, err := ioutil.ReadFile(filePath)
417
+ if err != nil {
418
+ return
419
+ }
420
+
421
+ profiles := parseINI(string(contents))
422
+ profileData, ok := profiles[profile]
423
+
424
+ if !ok {
425
+ err = errors.New("The credentials file did not contain the profile")
426
+ return
427
+ }
428
+
429
+ keyId, ok := profileData["aws_access_key_id"]
430
+ if !ok {
431
+ err = errors.New("The credentials file did not contain required attribute aws_access_key_id")
432
+ return
433
+ }
434
+
435
+ secretKey, ok := profileData["aws_secret_access_key"]
436
+ if !ok {
437
+ err = errors.New("The credentials file did not contain required attribute aws_secret_access_key")
438
+ return
439
+ }
440
+
441
+ auth.AccessKey = keyId
442
+ auth.SecretKey = secretKey
443
+
444
+ if token, ok := profileData["aws_session_token"]; ok {
445
+ auth.token = token
446
+ }
447
+
448
+ auth.expiration = time.Now().Add(expiration)
449
+
450
+ return
451
+}
452
+
453
+// parseINI takes the contents of a credentials file and returns a map, whose keys
454
+// are the various profiles, and whose values are maps of the settings for the
455
+// profiles
456
+func parseINI(fileContents string) map[string]map[string]string {
457
+ profiles := make(map[string]map[string]string)
458
+
459
+ lines := strings.Split(fileContents, "\n")
460
+
461
+ var currentSection map[string]string
462
+ for _, line := range lines {
463
+ // remove comments, which start with a semi-colon
464
+ if split := strings.Split(line, ";"); len(split) > 1 {
465
+ line = split[0]
466
+ }
467
+
468
+ // check if the line is the start of a profile.
469
+ //
470
+ // for example:
471
+ // [default]
472
+ //
473
+ // otherwise, check for the proper setting
474
+ // property=value
475
+ if sectMatch := iniSectionRegexp.FindStringSubmatch(line); len(sectMatch) == 2 {
476
+ currentSection = make(map[string]string)
477
+ profiles[sectMatch[1]] = currentSection
478
+ } else if setMatch := iniSettingRegexp.FindStringSubmatch(line); len(setMatch) == 3 && currentSection != nil {
479
+ currentSection[setMatch[1]] = setMatch[2]
480
+ }
481
+ }
482
+
483
+ return profiles
484
+}
485
+
486
+// Encode takes a string and URI-encodes it in a way suitable
487
+// to be used in AWS signatures.
488
+func Encode(s string) string {
489
+ encode := false
490
+ for i := 0; i != len(s); i++ {
491
+ c := s[i]
492
+ if c > 127 || !unreserved[c] {
493
+ encode = true
494
+ break
495
+ }
496
+ }
497
+ if !encode {
498
+ return s
499
+ }
500
+ e := make([]byte, len(s)*3)
501
+ ei := 0
502
+ for i := 0; i != len(s); i++ {
503
+ c := s[i]
504
+ if c > 127 || !unreserved[c] {
505
+ e[ei] = '%'
506
+ e[ei+1] = hex[c>>4]
507
+ e[ei+2] = hex[c&0xF]
508
+ ei += 3
509
+ } else {
510
+ e[ei] = c
511
+ ei += 1
512
+ }
513
+ }
514
+ return string(e[:ei])
515
+}
516
+
517
+func dialTimeout(network, addr string) (net.Conn, error) {
518
+ return net.DialTimeout(network, addr, time.Duration(2*time.Second))
519
+}
520
+
521
+func AvailabilityZone() string {
522
+ transport := http.Transport{Dial: dialTimeout}
523
+ client := http.Client{
524
+ Transport: &transport,
525
+ }
526
+ resp, err := client.Get("http://169.254.169.254/latest/meta-data/placement/availability-zone")
527
+ if err != nil {
528
+ return "unknown"
529
+ } else {
530
+ defer resp.Body.Close()
531
+ body, err := ioutil.ReadAll(resp.Body)
532
+ if err != nil {
533
+ return "unknown"
534
+ } else {
535
+ return string(body)
536
+ }
537
+ }
538
+}
539
+
540
+func InstanceRegion() string {
541
+ az := AvailabilityZone()
542
+ if az == "unknown" {
543
+ return az
544
+ } else {
545
+ region := az[:len(az)-1]
546
+ return region
547
+ }
548
+}
549
+
550
+func InstanceId() string {
551
+ transport := http.Transport{Dial: dialTimeout}
552
+ client := http.Client{
553
+ Transport: &transport,
554
+ }
555
+ resp, err := client.Get("http://169.254.169.254/latest/meta-data/instance-id")
556
+ if err != nil {
557
+ return "unknown"
558
+ } else {
559
+ defer resp.Body.Close()
560
+ body, err := ioutil.ReadAll(resp.Body)
561
+ if err != nil {
562
+ return "unknown"
563
+ } else {
564
+ return string(body)
565
+ }
566
+ }
567
+}
568
+
569
+func InstanceType() string {
570
+ transport := http.Transport{Dial: dialTimeout}
571
+ client := http.Client{
572
+ Transport: &transport,
573
+ }
574
+ resp, err := client.Get("http://169.254.169.254/latest/meta-data/instance-type")
575
+ if err != nil {
576
+ return "unknown"
577
+ } else {
578
+ defer resp.Body.Close()
579
+ body, err := ioutil.ReadAll(resp.Body)
580
+ if err != nil {
581
+ return "unknown"
582
+ } else {
583
+ return string(body)
584
+ }
585
+ }
586
+}
587
+
588
+func ServerLocalIp() string {
589
+ transport := http.Transport{Dial: dialTimeout}
590
+ client := http.Client{
591
+ Transport: &transport,
592
+ }
593
+ resp, err := client.Get("http://169.254.169.254/latest/meta-data/local-ipv4")
594
+ if err != nil {
595
+ return "127.0.0.1"
596
+ } else {
597
+ defer resp.Body.Close()
598
+ body, err := ioutil.ReadAll(resp.Body)
599
+ if err != nil {
600
+ return "127.0.0.1"
601
+ } else {
602
+ return string(body)
603
+ }
604
+ }
605
+}
606
+
607
+func ServerPublicIp() string {
608
+ transport := http.Transport{Dial: dialTimeout}
609
+ client := http.Client{
610
+ Transport: &transport,
611
+ }
612
+ resp, err := client.Get("http://169.254.169.254/latest/meta-data/public-ipv4")
613
+ if err != nil {
614
+ return "127.0.0.1"
615
+ } else {
616
+ defer resp.Body.Close()
617
+ body, err := ioutil.ReadAll(resp.Body)
618
+ if err != nil {
619
+ return "127.0.0.1"
620
+ } else {
621
+ return string(body)
622
+ }
623
+ }
624
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/aws/aws_test.go
new
+140
@@ -0,0 +1,140 @@
1
+package aws_test
2
+
3
+import (
4
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/aws"
5
+ "gopkg.in/check.v1"
6
+ "io/ioutil"
7
+ "os"
8
+ "strings"
9
+ "testing"
10
+ "time"
11
+)
12
+
13
+func Test(t *testing.T) {
14
+ check.TestingT(t)
15
+}
16
+
17
+var _ = check.Suite(&S{})
18
+
19
+type S struct {
20
+ environ []string
21
+}
22
+
23
+func (s *S) SetUpSuite(c *check.C) {
24
+ s.environ = os.Environ()
25
+}
26
+
27
+func (s *S) TearDownTest(c *check.C) {
28
+ os.Clearenv()
29
+ for _, kv := range s.environ {
30
+ l := strings.SplitN(kv, "=", 2)
31
+ os.Setenv(l[0], l[1])
32
+ }
33
+}
34
+
35
+func (s *S) TestEnvAuthNoSecret(c *check.C) {
36
+ os.Clearenv()
37
+ _, err := aws.EnvAuth()
38
+ c.Assert(err, check.ErrorMatches, "AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY not found in environment")
39
+}
40
+
41
+func (s *S) TestEnvAuthNoAccess(c *check.C) {
42
+ os.Clearenv()
43
+ os.Setenv("AWS_SECRET_ACCESS_KEY", "foo")
44
+ _, err := aws.EnvAuth()
45
+ c.Assert(err, check.ErrorMatches, "AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY not found in environment")
46
+}
47
+
48
+func (s *S) TestEnvAuth(c *check.C) {
49
+ os.Clearenv()
50
+ os.Setenv("AWS_SECRET_ACCESS_KEY", "secret")
51
+ os.Setenv("AWS_ACCESS_KEY_ID", "access")
52
+ auth, err := aws.EnvAuth()
53
+ c.Assert(err, check.IsNil)
54
+ c.Assert(auth, check.Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"})
55
+}
56
+
57
+func (s *S) TestEnvAuthAlt(c *check.C) {
58
+ os.Clearenv()
59
+ os.Setenv("AWS_SECRET_KEY", "secret")
60
+ os.Setenv("AWS_ACCESS_KEY", "access")
61
+ auth, err := aws.EnvAuth()
62
+ c.Assert(err, check.IsNil)
63
+ c.Assert(auth, check.Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"})
64
+}
65
+
66
+func (s *S) TestGetAuthStatic(c *check.C) {
67
+ exptdate := time.Now().Add(time.Hour)
68
+ auth, err := aws.GetAuth("access", "secret", "token", exptdate)
69
+ c.Assert(err, check.IsNil)
70
+ c.Assert(auth.AccessKey, check.Equals, "access")
71
+ c.Assert(auth.SecretKey, check.Equals, "secret")
72
+ c.Assert(auth.Token(), check.Equals, "token")
73
+ c.Assert(auth.Expiration(), check.Equals, exptdate)
74
+}
75
+
76
+func (s *S) TestGetAuthEnv(c *check.C) {
77
+ os.Clearenv()
78
+ os.Setenv("AWS_SECRET_ACCESS_KEY", "secret")
79
+ os.Setenv("AWS_ACCESS_KEY_ID", "access")
80
+ auth, err := aws.GetAuth("", "", "", time.Time{})
81
+ c.Assert(err, check.IsNil)
82
+ c.Assert(auth, check.Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"})
83
+}
84
+
85
+func (s *S) TestEncode(c *check.C) {
86
+ c.Assert(aws.Encode("foo"), check.Equals, "foo")
87
+ c.Assert(aws.Encode("/"), check.Equals, "%2F")
88
+}
89
+
90
+func (s *S) TestRegionsAreNamed(c *check.C) {
91
+ for n, r := range aws.Regions {
92
+ c.Assert(n, check.Equals, r.Name)
93
+ }
94
+}
95
+
96
+func (s *S) TestCredentialsFileAuth(c *check.C) {
97
+ file, err := ioutil.TempFile("", "creds")
98
+
99
+ if err != nil {
100
+ c.Fatal(err)
101
+ }
102
+
103
+ iniFile := `
104
+
105
+[default] ; comment 123
106
+aws_access_key_id = keyid1 ;comment
107
+aws_secret_access_key=key1
108
+
109
+ [profile2]
110
+ aws_access_key_id = keyid2 ;comment
111
+ aws_secret_access_key=key2
112
+ aws_session_token=token1
113
+
114
+`
115
+ _, err = file.WriteString(iniFile)
116
+ if err != nil {
117
+ c.Fatal(err)
118
+ }
119
+
120
+ err = file.Close()
121
+ if err != nil {
122
+ c.Fatal(err)
123
+ }
124
+
125
+ // check non-existant profile
126
+ _, err = aws.CredentialFileAuth(file.Name(), "no profile", 30*time.Minute)
127
+ c.Assert(err, check.Not(check.Equals), nil)
128
+
129
+ defaultProfile, err := aws.CredentialFileAuth(file.Name(), "default", 30*time.Minute)
130
+ c.Assert(err, check.Equals, nil)
131
+ c.Assert(defaultProfile.AccessKey, check.Equals, "keyid1")
132
+ c.Assert(defaultProfile.SecretKey, check.Equals, "key1")
133
+ c.Assert(defaultProfile.Token(), check.Equals, "")
134
+
135
+ profile2, err := aws.CredentialFileAuth(file.Name(), "profile2", 30*time.Minute)
136
+ c.Assert(err, check.Equals, nil)
137
+ c.Assert(profile2.AccessKey, check.Equals, "keyid2")
138
+ c.Assert(profile2.SecretKey, check.Equals, "key2")
139
+ c.Assert(profile2.Token(), check.Equals, "token1")
140
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/aws/client.go
new
+124
@@ -0,0 +1,124 @@
1
+package aws
2
+
3
+import (
4
+ "math"
5
+ "net"
6
+ "net/http"
7
+ "time"
8
+)
9
+
10
+type RetryableFunc func(*http.Request, *http.Response, error) bool
11
+type WaitFunc func(try int)
12
+type DeadlineFunc func() time.Time
13
+
14
+type ResilientTransport struct {
15
+ // Timeout is the maximum amount of time a dial will wait for
16
+ // a connect to complete.
17
+ //
18
+ // The default is no timeout.
19
+ //
20
+ // With or without a timeout, the operating system may impose
21
+ // its own earlier timeout. For instance, TCP timeouts are
22
+ // often around 3 minutes.
23
+ DialTimeout time.Duration
24
+
25
+ // MaxTries, if non-zero, specifies the number of times we will retry on
26
+ // failure. Retries are only attempted for temporary network errors or known
27
+ // safe failures.
28
+ MaxTries int
29
+ Deadline DeadlineFunc
30
+ ShouldRetry RetryableFunc
31
+ Wait WaitFunc
32
+ transport *http.Transport
33
+}
34
+
35
+// Convenience method for creating an http client
36
+func NewClient(rt *ResilientTransport) *http.Client {
37
+ rt.transport = &http.Transport{
38
+ Dial: func(netw, addr string) (net.Conn, error) {
39
+ c, err := net.DialTimeout(netw, addr, rt.DialTimeout)
40
+ if err != nil {
41
+ return nil, err
42
+ }
43
+ c.SetDeadline(rt.Deadline())
44
+ return c, nil
45
+ },
46
+ Proxy: http.ProxyFromEnvironment,
47
+ }
48
+ // TODO: Would be nice is ResilientTransport allowed clients to initialize
49
+ // with http.Transport attributes.
50
+ return &http.Client{
51
+ Transport: rt,
52
+ }
53
+}
54
+
55
+var retryingTransport = &ResilientTransport{
56
+ Deadline: func() time.Time {
57
+ return time.Now().Add(5 * time.Second)
58
+ },
59
+ DialTimeout: 10 * time.Second,
60
+ MaxTries: 3,
61
+ ShouldRetry: awsRetry,
62
+ Wait: ExpBackoff,
63
+}
64
+
65
+// Exported default client
66
+var RetryingClient = NewClient(retryingTransport)
67
+
68
+func (t *ResilientTransport) RoundTrip(req *http.Request) (*http.Response, error) {
69
+ return t.tries(req)
70
+}
71
+
72
+// Retry a request a maximum of t.MaxTries times.
73
+// We'll only retry if the proper criteria are met.
74
+// If a wait function is specified, wait that amount of time
75
+// In between requests.
76
+func (t *ResilientTransport) tries(req *http.Request) (res *http.Response, err error) {
77
+ for try := 0; try < t.MaxTries; try += 1 {
78
+ res, err = t.transport.RoundTrip(req)
79
+
80
+ if !t.ShouldRetry(req, res, err) {
81
+ break
82
+ }
83
+ if res != nil {
84
+ res.Body.Close()
85
+ }
86
+ if t.Wait != nil {
87
+ t.Wait(try)
88
+ }
89
+ }
90
+
91
+ return
92
+}
93
+
94
+func ExpBackoff(try int) {
95
+ time.Sleep(100 * time.Millisecond *
96
+ time.Duration(math.Exp2(float64(try))))
97
+}
98
+
99
+func LinearBackoff(try int) {
100
+ time.Sleep(time.Duration(try*100) * time.Millisecond)
101
+}
102
+
103
+// Decide if we should retry a request.
104
+// In general, the criteria for retrying a request is described here
105
+// http://docs.aws.amazon.com/general/latest/gr/api-retries.html
106
+func awsRetry(req *http.Request, res *http.Response, err error) bool {
107
+ retry := false
108
+
109
+ // Retry if there's a temporary network error.
110
+ if neterr, ok := err.(net.Error); ok {
111
+ if neterr.Temporary() {
112
+ retry = true
113
+ }
114
+ }
115
+
116
+ // Retry if we get a 5xx series error.
117
+ if res != nil {
118
+ if res.StatusCode >= 500 && res.StatusCode < 600 {
119
+ retry = true
120
+ }
121
+ }
122
+
123
+ return retry
124
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/aws/export_test.go
new
+29
@@ -0,0 +1,29 @@
1
+package aws
2
+
3
+import (
4
+ "net/http"
5
+ "time"
6
+)
7
+
8
+// V4Signer:
9
+// Exporting methods for testing
10
+
11
+func (s *V4Signer) RequestTime(req *http.Request) time.Time {
12
+ return s.requestTime(req)
13
+}
14
+
15
+func (s *V4Signer) CanonicalRequest(req *http.Request) string {
16
+ return s.canonicalRequest(req, "")
17
+}
18
+
19
+func (s *V4Signer) StringToSign(t time.Time, creq string) string {
20
+ return s.stringToSign(t, creq)
21
+}
22
+
23
+func (s *V4Signer) Signature(t time.Time, sts string) string {
24
+ return s.signature(t, sts)
25
+}
26
+
27
+func (s *V4Signer) Authorization(header http.Header, t time.Time, signature string) string {
28
+ return s.authorization(header, t, signature)
29
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/aws/regions.go
new
+231
@@ -0,0 +1,231 @@
1
+package aws
2
+
3
+var USGovWest = Region{
4
+ "us-gov-west-1",
5
+ "https://ec2.us-gov-west-1.amazonaws.com",
6
+ "https://s3-fips-us-gov-west-1.amazonaws.com",
7
+ "",
8
+ true,
9
+ true,
10
+ "",
11
+ "https://sns.us-gov-west-1.amazonaws.com",
12
+ "https://sqs.us-gov-west-1.amazonaws.com",
13
+ "",
14
+ "https://iam.us-gov.amazonaws.com",
15
+ "https://elasticloadbalancing.us-gov-west-1.amazonaws.com",
16
+ "https://dynamodb.us-gov-west-1.amazonaws.com",
17
+ ServiceInfo{"https://monitoring.us-gov-west-1.amazonaws.com", V2Signature},
18
+ "https://autoscaling.us-gov-west-1.amazonaws.com",
19
+ ServiceInfo{"https://rds.us-gov-west-1.amazonaws.com", V2Signature},
20
+ "",
21
+ "https://sts.amazonaws.com",
22
+ "https://cloudformation.us-gov-west-1.amazonaws.com",
23
+ "",
24
+}
25
+
26
+var USEast = Region{
27
+ "us-east-1",
28
+ "https://ec2.us-east-1.amazonaws.com",
29
+ "https://s3.amazonaws.com",
30
+ "",
31
+ false,
32
+ false,
33
+ "https://sdb.amazonaws.com",
34
+ "https://sns.us-east-1.amazonaws.com",
35
+ "https://sqs.us-east-1.amazonaws.com",
36
+ "https://email.us-east-1.amazonaws.com",
37
+ "https://iam.amazonaws.com",
38
+ "https://elasticloadbalancing.us-east-1.amazonaws.com",
39
+ "https://dynamodb.us-east-1.amazonaws.com",
40
+ ServiceInfo{"https://monitoring.us-east-1.amazonaws.com", V2Signature},
41
+ "https://autoscaling.us-east-1.amazonaws.com",
42
+ ServiceInfo{"https://rds.us-east-1.amazonaws.com", V2Signature},
43
+ "https://kinesis.us-east-1.amazonaws.com",
44
+ "https://sts.amazonaws.com",
45
+ "https://cloudformation.us-east-1.amazonaws.com",
46
+ "https://elasticache.us-east-1.amazonaws.com",
47
+}
48
+
49
+var USWest = Region{
50
+ "us-west-1",
51
+ "https://ec2.us-west-1.amazonaws.com",
52
+ "https://s3-us-west-1.amazonaws.com",
53
+ "",
54
+ true,
55
+ true,
56
+ "https://sdb.us-west-1.amazonaws.com",
57
+ "https://sns.us-west-1.amazonaws.com",
58
+ "https://sqs.us-west-1.amazonaws.com",
59
+ "",
60
+ "https://iam.amazonaws.com",
61
+ "https://elasticloadbalancing.us-west-1.amazonaws.com",
62
+ "https://dynamodb.us-west-1.amazonaws.com",
63
+ ServiceInfo{"https://monitoring.us-west-1.amazonaws.com", V2Signature},
64
+ "https://autoscaling.us-west-1.amazonaws.com",
65
+ ServiceInfo{"https://rds.us-west-1.amazonaws.com", V2Signature},
66
+ "",
67
+ "https://sts.amazonaws.com",
68
+ "https://cloudformation.us-west-1.amazonaws.com",
69
+ "https://elasticache.us-west-1.amazonaws.com",
70
+}
71
+
72
+var USWest2 = Region{
73
+ "us-west-2",
74
+ "https://ec2.us-west-2.amazonaws.com",
75
+ "https://s3-us-west-2.amazonaws.com",
76
+ "",
77
+ true,
78
+ true,
79
+ "https://sdb.us-west-2.amazonaws.com",
80
+ "https://sns.us-west-2.amazonaws.com",
81
+ "https://sqs.us-west-2.amazonaws.com",
82
+ "https://email.us-west-2.amazonaws.com",
83
+ "https://iam.amazonaws.com",
84
+ "https://elasticloadbalancing.us-west-2.amazonaws.com",
85
+ "https://dynamodb.us-west-2.amazonaws.com",
86
+ ServiceInfo{"https://monitoring.us-west-2.amazonaws.com", V2Signature},
87
+ "https://autoscaling.us-west-2.amazonaws.com",
88
+ ServiceInfo{"https://rds.us-west-2.amazonaws.com", V2Signature},
89
+ "https://kinesis.us-west-2.amazonaws.com",
90
+ "https://sts.amazonaws.com",
91
+ "https://cloudformation.us-west-2.amazonaws.com",
92
+ "https://elasticache.us-west-2.amazonaws.com",
93
+}
94
+
95
+var EUWest = Region{
96
+ "eu-west-1",
97
+ "https://ec2.eu-west-1.amazonaws.com",
98
+ "https://s3-eu-west-1.amazonaws.com",
99
+ "",
100
+ true,
101
+ true,
102
+ "https://sdb.eu-west-1.amazonaws.com",
103
+ "https://sns.eu-west-1.amazonaws.com",
104
+ "https://sqs.eu-west-1.amazonaws.com",
105
+ "https://email.eu-west-1.amazonaws.com",
106
+ "https://iam.amazonaws.com",
107
+ "https://elasticloadbalancing.eu-west-1.amazonaws.com",
108
+ "https://dynamodb.eu-west-1.amazonaws.com",
109
+ ServiceInfo{"https://monitoring.eu-west-1.amazonaws.com", V2Signature},
110
+ "https://autoscaling.eu-west-1.amazonaws.com",
111
+ ServiceInfo{"https://rds.eu-west-1.amazonaws.com", V2Signature},
112
+ "https://kinesis.eu-west-1.amazonaws.com",
113
+ "https://sts.amazonaws.com",
114
+ "https://cloudformation.eu-west-1.amazonaws.com",
115
+ "https://elasticache.eu-west-1.amazonaws.com",
116
+}
117
+
118
+var EUCentral = Region{
119
+ "eu-central-1",
120
+ "https://ec2.eu-central-1.amazonaws.com",
121
+ "https://s3-eu-central-1.amazonaws.com",
122
+ "",
123
+ true,
124
+ true,
125
+ "https://sdb.eu-central-1.amazonaws.com",
126
+ "https://sns.eu-central-1.amazonaws.com",
127
+ "https://sqs.eu-central-1.amazonaws.com",
128
+ "",
129
+ "https://iam.amazonaws.com",
130
+ "https://elasticloadbalancing.eu-central-1.amazonaws.com",
131
+ "https://dynamodb.eu-central-1.amazonaws.com",
132
+ ServiceInfo{"https://monitoring.eu-central-1.amazonaws.com", V2Signature},
133
+ "https://autoscaling.eu-central-1.amazonaws.com",
134
+ ServiceInfo{"https://rds.eu-central-1.amazonaws.com", V2Signature},
135
+ "https://kinesis.eu-central-1.amazonaws.com",
136
+ "https://sts.amazonaws.com",
137
+ "https://cloudformation.eu-central-1.amazonaws.com",
138
+ "",
139
+}
140
+
141
+var APSoutheast = Region{
142
+ "ap-southeast-1",
143
+ "https://ec2.ap-southeast-1.amazonaws.com",
144
+ "https://s3-ap-southeast-1.amazonaws.com",
145
+ "",
146
+ true,
147
+ true,
148
+ "https://sdb.ap-southeast-1.amazonaws.com",
149
+ "https://sns.ap-southeast-1.amazonaws.com",
150
+ "https://sqs.ap-southeast-1.amazonaws.com",
151
+ "",
152
+ "https://iam.amazonaws.com",
153
+ "https://elasticloadbalancing.ap-southeast-1.amazonaws.com",
154
+ "https://dynamodb.ap-southeast-1.amazonaws.com",
155
+ ServiceInfo{"https://monitoring.ap-southeast-1.amazonaws.com", V2Signature},
156
+ "https://autoscaling.ap-southeast-1.amazonaws.com",
157
+ ServiceInfo{"https://rds.ap-southeast-1.amazonaws.com", V2Signature},
158
+ "https://kinesis.ap-southeast-1.amazonaws.com",
159
+ "https://sts.amazonaws.com",
160
+ "https://cloudformation.ap-southeast-1.amazonaws.com",
161
+ "https://elasticache.ap-southeast-1.amazonaws.com",
162
+}
163
+
164
+var APSoutheast2 = Region{
165
+ "ap-southeast-2",
166
+ "https://ec2.ap-southeast-2.amazonaws.com",
167
+ "https://s3-ap-southeast-2.amazonaws.com",
168
+ "",
169
+ true,
170
+ true,
171
+ "https://sdb.ap-southeast-2.amazonaws.com",
172
+ "https://sns.ap-southeast-2.amazonaws.com",
173
+ "https://sqs.ap-southeast-2.amazonaws.com",
174
+ "",
175
+ "https://iam.amazonaws.com",
176
+ "https://elasticloadbalancing.ap-southeast-2.amazonaws.com",
177
+ "https://dynamodb.ap-southeast-2.amazonaws.com",
178
+ ServiceInfo{"https://monitoring.ap-southeast-2.amazonaws.com", V2Signature},
179
+ "https://autoscaling.ap-southeast-2.amazonaws.com",
180
+ ServiceInfo{"https://rds.ap-southeast-2.amazonaws.com", V2Signature},
181
+ "https://kinesis.ap-southeast-2.amazonaws.com",
182
+ "https://sts.amazonaws.com",
183
+ "https://cloudformation.ap-southeast-2.amazonaws.com",
184
+ "https://elasticache.ap-southeast-2.amazonaws.com",
185
+}
186
+
187
+var APNortheast = Region{
188
+ "ap-northeast-1",
189
+ "https://ec2.ap-northeast-1.amazonaws.com",
190
+ "https://s3-ap-northeast-1.amazonaws.com",
191
+ "",
192
+ true,
193
+ true,
194
+ "https://sdb.ap-northeast-1.amazonaws.com",
195
+ "https://sns.ap-northeast-1.amazonaws.com",
196
+ "https://sqs.ap-northeast-1.amazonaws.com",
197
+ "",
198
+ "https://iam.amazonaws.com",
199
+ "https://elasticloadbalancing.ap-northeast-1.amazonaws.com",
200
+ "https://dynamodb.ap-northeast-1.amazonaws.com",
201
+ ServiceInfo{"https://monitoring.ap-northeast-1.amazonaws.com", V2Signature},
202
+ "https://autoscaling.ap-northeast-1.amazonaws.com",
203
+ ServiceInfo{"https://rds.ap-northeast-1.amazonaws.com", V2Signature},
204
+ "https://kinesis.ap-northeast-1.amazonaws.com",
205
+ "https://sts.amazonaws.com",
206
+ "https://cloudformation.ap-northeast-1.amazonaws.com",
207
+ "https://elasticache.ap-northeast-1.amazonaws.com",
208
+}
209
+
210
+var SAEast = Region{
211
+ "sa-east-1",
212
+ "https://ec2.sa-east-1.amazonaws.com",
213
+ "https://s3-sa-east-1.amazonaws.com",
214
+ "",
215
+ true,
216
+ true,
217
+ "https://sdb.sa-east-1.amazonaws.com",
218
+ "https://sns.sa-east-1.amazonaws.com",
219
+ "https://sqs.sa-east-1.amazonaws.com",
220
+ "",
221
+ "https://iam.amazonaws.com",
222
+ "https://elasticloadbalancing.sa-east-1.amazonaws.com",
223
+ "https://dynamodb.sa-east-1.amazonaws.com",
224
+ ServiceInfo{"https://monitoring.sa-east-1.amazonaws.com", V2Signature},
225
+ "https://autoscaling.sa-east-1.amazonaws.com",
226
+ ServiceInfo{"https://rds.sa-east-1.amazonaws.com", V2Signature},
227
+ "",
228
+ "https://sts.amazonaws.com",
229
+ "https://cloudformation.sa-east-1.amazonaws.com",
230
+ "https://elasticache.sa-east-1.amazonaws.com",
231
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/aws/retry.go
new
+136
@@ -0,0 +1,136 @@
1
+package aws
2
+
3
+import (
4
+ "math/rand"
5
+ "net"
6
+ "net/http"
7
+ "time"
8
+)
9
+
10
+const (
11
+ maxDelay = 20 * time.Second
12
+ defaultScale = 300 * time.Millisecond
13
+ throttlingScale = 500 * time.Millisecond
14
+ throttlingScaleRange = throttlingScale / 4
15
+ defaultMaxRetries = 3
16
+ dynamoDBScale = 25 * time.Millisecond
17
+ dynamoDBMaxRetries = 10
18
+)
19
+
20
+// A RetryPolicy encapsulates a strategy for implementing client retries.
21
+//
22
+// Default implementations are provided which match the AWS SDKs.
23
+type RetryPolicy interface {
24
+ // ShouldRetry returns whether a client should retry a failed request.
25
+ ShouldRetry(target string, r *http.Response, err error, numRetries int) bool
26
+
27
+ // Delay returns the time a client should wait before issuing a retry.
28
+ Delay(target string, r *http.Response, err error, numRetries int) time.Duration
29
+}
30
+
31
+// DefaultRetryPolicy implements the AWS SDK default retry policy.
32
+//
33
+// It will retry up to 3 times, and uses an exponential backoff with a scale
34
+// factor of 300ms (300ms, 600ms, 1200ms). If the retry is because of
35
+// throttling, the delay will also include some randomness.
36
+//
37
+// See https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/retry/PredefinedRetryPolicies.java#L90.
38
+type DefaultRetryPolicy struct {
39
+}
40
+
41
+// ShouldRetry implements the RetryPolicy ShouldRetry method.
42
+func (policy DefaultRetryPolicy) ShouldRetry(target string, r *http.Response, err error, numRetries int) bool {
43
+ return shouldRetry(r, err, numRetries, defaultMaxRetries)
44
+}
45
+
46
+// Delay implements the RetryPolicy Delay method.
47
+func (policy DefaultRetryPolicy) Delay(target string, r *http.Response, err error, numRetries int) time.Duration {
48
+ scale := defaultScale
49
+ if err, ok := err.(*Error); ok && isThrottlingException(err) {
50
+ scale = throttlingScale + time.Duration(rand.Int63n(int64(throttlingScaleRange)))
51
+ }
52
+ return exponentialBackoff(numRetries, scale)
53
+}
54
+
55
+// DynamoDBRetryPolicy implements the AWS SDK DynamoDB retry policy.
56
+//
57
+// It will retry up to 10 times, and uses an exponential backoff with a scale
58
+// factor of 25ms (25ms, 50ms, 100ms, ...).
59
+//
60
+// See https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/retry/PredefinedRetryPolicies.java#L103.
61
+type DynamoDBRetryPolicy struct {
62
+}
63
+
64
+// ShouldRetry implements the RetryPolicy ShouldRetry method.
65
+func (policy DynamoDBRetryPolicy) ShouldRetry(target string, r *http.Response, err error, numRetries int) bool {
66
+ return shouldRetry(r, err, numRetries, dynamoDBMaxRetries)
67
+}
68
+
69
+// Delay implements the RetryPolicy Delay method.
70
+func (policy DynamoDBRetryPolicy) Delay(target string, r *http.Response, err error, numRetries int) time.Duration {
71
+ return exponentialBackoff(numRetries, dynamoDBScale)
72
+}
73
+
74
+// NeverRetryPolicy never retries requests and returns immediately on failure.
75
+type NeverRetryPolicy struct {
76
+}
77
+
78
+// ShouldRetry implements the RetryPolicy ShouldRetry method.
79
+func (policy NeverRetryPolicy) ShouldRetry(target string, r *http.Response, err error, numRetries int) bool {
80
+ return false
81
+}
82
+
83
+// Delay implements the RetryPolicy Delay method.
84
+func (policy NeverRetryPolicy) Delay(target string, r *http.Response, err error, numRetries int) time.Duration {
85
+ return time.Duration(0)
86
+}
87
+
88
+// shouldRetry determines if we should retry the request.
89
+//
90
+// See http://docs.aws.amazon.com/general/latest/gr/api-retries.html.
91
+func shouldRetry(r *http.Response, err error, numRetries int, maxRetries int) bool {
92
+ // Once we've exceeded the max retry attempts, game over.
93
+ if numRetries >= maxRetries {
94
+ return false
95
+ }
96
+
97
+ // Always retry temporary network errors.
98
+ if err, ok := err.(net.Error); ok && err.Temporary() {
99
+ return true
100
+ }
101
+
102
+ // Always retry 5xx responses.
103
+ if r != nil && r.StatusCode >= 500 {
104
+ return true
105
+ }
106
+
107
+ // Always retry throttling exceptions.
108
+ if err, ok := err.(ServiceError); ok && isThrottlingException(err) {
109
+ return true
110
+ }
111
+
112
+ // Other classes of failures indicate a problem with the request. Retrying
113
+ // won't help.
114
+ return false
115
+}
116
+
117
+func exponentialBackoff(numRetries int, scale time.Duration) time.Duration {
118
+ if numRetries < 0 {
119
+ return time.Duration(0)
120
+ }
121
+
122
+ delay := (1 << uint(numRetries)) * scale
123
+ if delay > maxDelay {
124
+ return maxDelay
125
+ }
126
+ return delay
127
+}
128
+
129
+func isThrottlingException(err ServiceError) bool {
130
+ switch err.ErrorCode() {
131
+ case "Throttling", "ThrottlingException", "ProvisionedThroughputExceededException":
132
+ return true
133
+ default:
134
+ return false
135
+ }
136
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/aws/retry_test.go
new
+303
@@ -0,0 +1,303 @@
1
+package aws
2
+
3
+import (
4
+ "math/rand"
5
+ "net"
6
+ "net/http"
7
+ "testing"
8
+ "time"
9
+)
10
+
11
+type testInput struct {
12
+ res *http.Response
13
+ err error
14
+ numRetries int
15
+}
16
+
17
+type testResult struct {
18
+ shouldRetry bool
19
+ delay time.Duration
20
+}
21
+
22
+type testCase struct {
23
+ input testInput
24
+ defaultResult testResult
25
+ dynamoDBResult testResult
26
+}
27
+
28
+var testCases = []testCase{
29
+ // Test nil fields
30
+ testCase{
31
+ input: testInput{
32
+ err: nil,
33
+ res: nil,
34
+ numRetries: 0,
35
+ },
36
+ defaultResult: testResult{
37
+ shouldRetry: false,
38
+ delay: 300 * time.Millisecond,
39
+ },
40
+ dynamoDBResult: testResult{
41
+ shouldRetry: false,
42
+ delay: 25 * time.Millisecond,
43
+ },
44
+ },
45
+ // Test 3 different throttling exceptions
46
+ testCase{
47
+ input: testInput{
48
+ err: &Error{
49
+ Code: "Throttling",
50
+ },
51
+ numRetries: 0,
52
+ },
53
+ defaultResult: testResult{
54
+ shouldRetry: true,
55
+ delay: 617165505 * time.Nanosecond, // account for randomness with known seed
56
+ },
57
+ dynamoDBResult: testResult{
58
+ shouldRetry: true,
59
+ delay: 25 * time.Millisecond,
60
+ },
61
+ },
62
+ testCase{
63
+ input: testInput{
64
+ err: &Error{
65
+ Code: "ThrottlingException",
66
+ },
67
+ numRetries: 0,
68
+ },
69
+ defaultResult: testResult{
70
+ shouldRetry: true,
71
+ delay: 579393152 * time.Nanosecond, // account for randomness with known seed
72
+ },
73
+ dynamoDBResult: testResult{
74
+ shouldRetry: true,
75
+ delay: 25 * time.Millisecond,
76
+ },
77
+ },
78
+ testCase{
79
+ input: testInput{
80
+ err: &Error{
81
+ Code: "ProvisionedThroughputExceededException",
82
+ },
83
+ numRetries: 1,
84
+ },
85
+ defaultResult: testResult{
86
+ shouldRetry: true,
87
+ delay: 1105991654 * time.Nanosecond, // account for randomness with known seed
88
+ },
89
+ dynamoDBResult: testResult{
90
+ shouldRetry: true,
91
+ delay: 50 * time.Millisecond,
92
+ },
93
+ },
94
+ // Test a fake throttling exception
95
+ testCase{
96
+ input: testInput{
97
+ err: &Error{
98
+ Code: "MyMadeUpThrottlingCode",
99
+ },
100
+ numRetries: 0,
101
+ },
102
+ defaultResult: testResult{
103
+ shouldRetry: false,
104
+ delay: 300 * time.Millisecond,
105
+ },
106
+ dynamoDBResult: testResult{
107
+ shouldRetry: false,
108
+ delay: 25 * time.Millisecond,
109
+ },
110
+ },
111
+ // Test 5xx errors
112
+ testCase{
113
+ input: testInput{
114
+ res: &http.Response{
115
+ StatusCode: http.StatusInternalServerError,
116
+ },
117
+ numRetries: 1,
118
+ },
119
+ defaultResult: testResult{
120
+ shouldRetry: true,
121
+ delay: 600 * time.Millisecond,
122
+ },
123
+ dynamoDBResult: testResult{
124
+ shouldRetry: true,
125
+ delay: 50 * time.Millisecond,
126
+ },
127
+ },
128
+ testCase{
129
+ input: testInput{
130
+ res: &http.Response{
131
+ StatusCode: http.StatusServiceUnavailable,
132
+ },
133
+ numRetries: 1,
134
+ },
135
+ defaultResult: testResult{
136
+ shouldRetry: true,
137
+ delay: 600 * time.Millisecond,
138
+ },
139
+ dynamoDBResult: testResult{
140
+ shouldRetry: true,
141
+ delay: 50 * time.Millisecond,
142
+ },
143
+ },
144
+ // Test a random 400 error
145
+ testCase{
146
+ input: testInput{
147
+ res: &http.Response{
148
+ StatusCode: http.StatusNotFound,
149
+ },
150
+ numRetries: 1,
151
+ },
152
+ defaultResult: testResult{
153
+ shouldRetry: false,
154
+ delay: 600 * time.Millisecond,
155
+ },
156
+ dynamoDBResult: testResult{
157
+ shouldRetry: false,
158
+ delay: 50 * time.Millisecond,
159
+ },
160
+ },
161
+ // Test a temporary net.Error
162
+ testCase{
163
+ input: testInput{
164
+ res: &http.Response{},
165
+ err: &net.DNSError{
166
+ IsTimeout: true,
167
+ },
168
+ numRetries: 2,
169
+ },
170
+ defaultResult: testResult{
171
+ shouldRetry: true,
172
+ delay: 1200 * time.Millisecond,
173
+ },
174
+ dynamoDBResult: testResult{
175
+ shouldRetry: true,
176
+ delay: 100 * time.Millisecond,
177
+ },
178
+ },
179
+ // Test a non-temporary net.Error
180
+ testCase{
181
+ input: testInput{
182
+ res: &http.Response{},
183
+ err: &net.DNSError{
184
+ IsTimeout: false,
185
+ },
186
+ numRetries: 3,
187
+ },
188
+ defaultResult: testResult{
189
+ shouldRetry: false,
190
+ delay: 2400 * time.Millisecond,
191
+ },
192
+ dynamoDBResult: testResult{
193
+ shouldRetry: false,
194
+ delay: 200 * time.Millisecond,
195
+ },
196
+ },
197
+ // Assert failure after hitting max default retries
198
+ testCase{
199
+ input: testInput{
200
+ err: &Error{
201
+ Code: "ProvisionedThroughputExceededException",
202
+ },
203
+ numRetries: defaultMaxRetries,
204
+ },
205
+ defaultResult: testResult{
206
+ shouldRetry: false,
207
+ delay: 4313582352 * time.Nanosecond, // account for randomness with known seed
208
+ },
209
+ dynamoDBResult: testResult{
210
+ shouldRetry: true,
211
+ delay: 200 * time.Millisecond,
212
+ },
213
+ },
214
+ // Assert failure after hitting max DynamoDB retries
215
+ testCase{
216
+ input: testInput{
217
+ err: &Error{
218
+ Code: "ProvisionedThroughputExceededException",
219
+ },
220
+ numRetries: dynamoDBMaxRetries,
221
+ },
222
+ defaultResult: testResult{
223
+ shouldRetry: false,
224
+ delay: maxDelay,
225
+ },
226
+ dynamoDBResult: testResult{
227
+ shouldRetry: false,
228
+ delay: maxDelay,
229
+ },
230
+ },
231
+ // Assert we never go over the maxDelay value
232
+ testCase{
233
+ input: testInput{
234
+ numRetries: 25,
235
+ },
236
+ defaultResult: testResult{
237
+ shouldRetry: false,
238
+ delay: maxDelay,
239
+ },
240
+ dynamoDBResult: testResult{
241
+ shouldRetry: false,
242
+ delay: maxDelay,
243
+ },
244
+ },
245
+}
246
+
247
+func TestDefaultRetryPolicy(t *testing.T) {
248
+ rand.Seed(0)
249
+ var policy RetryPolicy
250
+ policy = &DefaultRetryPolicy{}
251
+ for _, test := range testCases {
252
+ res := test.input.res
253
+ err := test.input.err
254
+ numRetries := test.input.numRetries
255
+
256
+ shouldRetry := policy.ShouldRetry("", res, err, numRetries)
257
+ if shouldRetry != test.defaultResult.shouldRetry {
258
+ t.Errorf("ShouldRetry returned %v, expected %v res=%#v err=%#v numRetries=%d", shouldRetry, test.defaultResult.shouldRetry, res, err, numRetries)
259
+ }
260
+ delay := policy.Delay("", res, err, numRetries)
261
+ if delay != test.defaultResult.delay {
262
+ t.Errorf("Delay returned %v, expected %v res=%#v err=%#v numRetries=%d", delay, test.defaultResult.delay, res, err, numRetries)
263
+ }
264
+ }
265
+}
266
+
267
+func TestDynamoDBRetryPolicy(t *testing.T) {
268
+ var policy RetryPolicy
269
+ policy = &DynamoDBRetryPolicy{}
270
+ for _, test := range testCases {
271
+ res := test.input.res
272
+ err := test.input.err
273
+ numRetries := test.input.numRetries
274
+
275
+ shouldRetry := policy.ShouldRetry("", res, err, numRetries)
276
+ if shouldRetry != test.dynamoDBResult.shouldRetry {
277
+ t.Errorf("ShouldRetry returned %v, expected %v res=%#v err=%#v numRetries=%d", shouldRetry, test.dynamoDBResult.shouldRetry, res, err, numRetries)
278
+ }
279
+ delay := policy.Delay("", res, err, numRetries)
280
+ if delay != test.dynamoDBResult.delay {
281
+ t.Errorf("Delay returned %v, expected %v res=%#v err=%#v numRetries=%d", delay, test.dynamoDBResult.delay, res, err, numRetries)
282
+ }
283
+ }
284
+}
285
+
286
+func TestNeverRetryPolicy(t *testing.T) {
287
+ var policy RetryPolicy
288
+ policy = &NeverRetryPolicy{}
289
+ for _, test := range testCases {
290
+ res := test.input.res
291
+ err := test.input.err
292
+ numRetries := test.input.numRetries
293
+
294
+ shouldRetry := policy.ShouldRetry("", res, err, numRetries)
295
+ if shouldRetry {
296
+ t.Errorf("ShouldRetry returned %v, expected %v res=%#v err=%#v numRetries=%d", shouldRetry, false, res, err, numRetries)
297
+ }
298
+ delay := policy.Delay("", res, err, numRetries)
299
+ if delay != time.Duration(0) {
300
+ t.Errorf("Delay returned %v, expected %v res=%#v err=%#v numRetries=%d", delay, time.Duration(0), res, err, numRetries)
301
+ }
302
+ }
303
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/aws/sign.go
new
+413
@@ -0,0 +1,413 @@
1
+package aws
2
+
3
+import (
4
+ "bytes"
5
+ "crypto/hmac"
6
+ "crypto/sha256"
7
+ "encoding/base64"
8
+ "fmt"
9
+ "io/ioutil"
10
+ "net/http"
11
+ "net/url"
12
+ "path"
13
+ "sort"
14
+ "strings"
15
+ "time"
16
+)
17
+
18
+type V2Signer struct {
19
+ auth Auth
20
+ service ServiceInfo
21
+ host string
22
+}
23
+
24
+var b64 = base64.StdEncoding
25
+
26
+func NewV2Signer(auth Auth, service ServiceInfo) (*V2Signer, error) {
27
+ u, err := url.Parse(service.Endpoint)
28
+ if err != nil {
29
+ return nil, err
30
+ }
31
+ return &V2Signer{auth: auth, service: service, host: u.Host}, nil
32
+}
33
+
34
+func (s *V2Signer) Sign(method, path string, params map[string]string) {
35
+ params["AWSAccessKeyId"] = s.auth.AccessKey
36
+ params["SignatureVersion"] = "2"
37
+ params["SignatureMethod"] = "HmacSHA256"
38
+ if s.auth.Token() != "" {
39
+ params["SecurityToken"] = s.auth.Token()
40
+ }
41
+
42
+ // AWS specifies that the parameters in a signed request must
43
+ // be provided in the natural order of the keys. This is distinct
44
+ // from the natural order of the encoded value of key=value.
45
+ // Percent and gocheck.Equals affect the sorting order.
46
+ var keys, sarray []string
47
+ for k, _ := range params {
48
+ keys = append(keys, k)
49
+ }
50
+ sort.Strings(keys)
51
+ for _, k := range keys {
52
+ sarray = append(sarray, Encode(k)+"="+Encode(params[k]))
53
+ }
54
+ joined := strings.Join(sarray, "&")
55
+ payload := method + "\n" + s.host + "\n" + path + "\n" + joined
56
+ hash := hmac.New(sha256.New, []byte(s.auth.SecretKey))
57
+ hash.Write([]byte(payload))
58
+ signature := make([]byte, b64.EncodedLen(hash.Size()))
59
+ b64.Encode(signature, hash.Sum(nil))
60
+
61
+ params["Signature"] = string(signature)
62
+}
63
+
64
+// Common date formats for signing requests
65
+const (
66
+ ISO8601BasicFormat = "20060102T150405Z"
67
+ ISO8601BasicFormatShort = "20060102"
68
+)
69
+
70
+type Route53Signer struct {
71
+ auth Auth
72
+}
73
+
74
+func NewRoute53Signer(auth Auth) *Route53Signer {
75
+ return &Route53Signer{auth: auth}
76
+}
77
+
78
+// getCurrentDate fetches the date stamp from the aws servers to
79
+// ensure the auth headers are within 5 minutes of the server time
80
+func (s *Route53Signer) getCurrentDate() string {
81
+ response, err := http.Get("https://route53.amazonaws.com/date")
82
+ if err != nil {
83
+ fmt.Print("Unable to get date from amazon: ", err)
84
+ return ""
85
+ }
86
+
87
+ response.Body.Close()
88
+ return response.Header.Get("Date")
89
+}
90
+
91
+// Creates the authorize signature based on the date stamp and secret key
92
+func (s *Route53Signer) getHeaderAuthorize(message string) string {
93
+ hmacSha256 := hmac.New(sha256.New, []byte(s.auth.SecretKey))
94
+ hmacSha256.Write([]byte(message))
95
+ cryptedString := hmacSha256.Sum(nil)
96
+
97
+ return base64.StdEncoding.EncodeToString(cryptedString)
98
+}
99
+
100
+// Adds all the required headers for AWS Route53 API to the request
101
+// including the authorization
102
+func (s *Route53Signer) Sign(req *http.Request) {
103
+ date := s.getCurrentDate()
104
+ authHeader := fmt.Sprintf("AWS3-HTTPS AWSAccessKeyId=%s,Algorithm=%s,Signature=%s",
105
+ s.auth.AccessKey, "HmacSHA256", s.getHeaderAuthorize(date))
106
+
107
+ req.Header.Set("Host", req.Host)
108
+ req.Header.Set("X-Amzn-Authorization", authHeader)
109
+ req.Header.Set("X-Amz-Date", date)
110
+ req.Header.Set("Content-Type", "application/xml")
111
+ if s.auth.Token() != "" {
112
+ req.Header.Set("X-Amzn-Security-Token", s.auth.Token())
113
+ }
114
+}
115
+
116
+/*
117
+The V4Signer encapsulates all of the functionality to sign a request with the AWS
118
+Signature Version 4 Signing Process. (http://goo.gl/u1OWZz)
119
+*/
120
+type V4Signer struct {
121
+ auth Auth
122
+ serviceName string
123
+ region Region
124
+ // Add the x-amz-content-sha256 header
125
+ IncludeXAmzContentSha256 bool
126
+}
127
+
128
+/*
129
+Return a new instance of a V4Signer capable of signing AWS requests.
130
+*/
131
+func NewV4Signer(auth Auth, serviceName string, region Region) *V4Signer {
132
+ return &V4Signer{
133
+ auth: auth,
134
+ serviceName: serviceName,
135
+ region: region,
136
+ IncludeXAmzContentSha256: false,
137
+ }
138
+}
139
+
140
+/*
141
+Sign a request according to the AWS Signature Version 4 Signing Process. (http://goo.gl/u1OWZz)
142
+
143
+The signed request will include an "x-amz-date" header with a current timestamp if a valid "x-amz-date"
144
+or "date" header was not available in the original request. In addition, AWS Signature Version 4 requires
145
+the "host" header to be a signed header, therefor the Sign method will manually set a "host" header from
146
+the request.Host.
147
+
148
+The signed request will include a new "Authorization" header indicating that the request has been signed.
149
+
150
+Any changes to the request after signing the request will invalidate the signature.
151
+*/
152
+func (s *V4Signer) Sign(req *http.Request) {
153
+ req.Header.Set("host", req.Host) // host header must be included as a signed header
154
+ t := s.requestTime(req) // Get request time
155
+
156
+ payloadHash := ""
157
+
158
+ if _, ok := req.Form["X-Amz-Expires"]; ok {
159
+ // We are authenticating the the request by using query params
160
+ // (also known as pre-signing a url, http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html)
161
+ payloadHash = "UNSIGNED-PAYLOAD"
162
+ req.Header.Del("x-amz-date")
163
+
164
+ req.Form["X-Amz-SignedHeaders"] = []string{s.signedHeaders(req.Header)}
165
+ req.Form["X-Amz-Algorithm"] = []string{"AWS4-HMAC-SHA256"}
166
+ req.Form["X-Amz-Credential"] = []string{s.auth.AccessKey + "/" + s.credentialScope(t)}
167
+ req.Form["X-Amz-Date"] = []string{t.Format(ISO8601BasicFormat)}
168
+ req.URL.RawQuery = req.Form.Encode()
169
+ } else {
170
+ payloadHash = s.payloadHash(req)
171
+ if s.IncludeXAmzContentSha256 {
172
+ req.Header.Set("x-amz-content-sha256", payloadHash) // x-amz-content-sha256 contains the payload hash
173
+ }
174
+ }
175
+ creq := s.canonicalRequest(req, payloadHash) // Build canonical request
176
+ sts := s.stringToSign(t, creq) // Build string to sign
177
+ signature := s.signature(t, sts) // Calculate the AWS Signature Version 4
178
+ auth := s.authorization(req.Header, t, signature) // Create Authorization header value
179
+
180
+ if _, ok := req.Form["X-Amz-Expires"]; ok {
181
+ req.Form["X-Amz-Signature"] = []string{signature}
182
+ } else {
183
+ req.Header.Set("Authorization", auth) // Add Authorization header to request
184
+ }
185
+ return
186
+}
187
+
188
+/*
189
+requestTime method will parse the time from the request "x-amz-date" or "date" headers.
190
+If the "x-amz-date" header is present, that will take priority over the "date" header.
191
+If neither header is defined or we are unable to parse either header as a valid date
192
+then we will create a new "x-amz-date" header with the current time.
193
+*/
194
+func (s *V4Signer) requestTime(req *http.Request) time.Time {
195
+
196
+ // Get "x-amz-date" header
197
+ date := req.Header.Get("x-amz-date")
198
+
199
+ // Attempt to parse as ISO8601BasicFormat
200
+ t, err := time.Parse(ISO8601BasicFormat, date)
201
+ if err == nil {
202
+ return t
203
+ }
204
+
205
+ // Attempt to parse as http.TimeFormat
206
+ t, err = time.Parse(http.TimeFormat, date)
207
+ if err == nil {
208
+ req.Header.Set("x-amz-date", t.Format(ISO8601BasicFormat))
209
+ return t
210
+ }
211
+
212
+ // Get "date" header
213
+ date = req.Header.Get("date")
214
+
215
+ // Attempt to parse as http.TimeFormat
216
+ t, err = time.Parse(http.TimeFormat, date)
217
+ if err == nil {
218
+ return t
219
+ }
220
+
221
+ // Create a current time header to be used
222
+ t = time.Now().UTC()
223
+ req.Header.Set("x-amz-date", t.Format(ISO8601BasicFormat))
224
+ return t
225
+}
226
+
227
+/*
228
+canonicalRequest method creates the canonical request according to Task 1 of the AWS Signature Version 4 Signing Process. (http://goo.gl/eUUZ3S)
229
+
230
+ CanonicalRequest =
231
+ HTTPRequestMethod + '\n' +
232
+ CanonicalURI + '\n' +
233
+ CanonicalQueryString + '\n' +
234
+ CanonicalHeaders + '\n' +
235
+ SignedHeaders + '\n' +
236
+ HexEncode(Hash(Payload))
237
+
238
+payloadHash is optional; use the empty string and it will be calculated from the request
239
+*/
240
+func (s *V4Signer) canonicalRequest(req *http.Request, payloadHash string) string {
241
+ if payloadHash == "" {
242
+ payloadHash = s.payloadHash(req)
243
+ }
244
+ c := new(bytes.Buffer)
245
+ fmt.Fprintf(c, "%s\n", req.Method)
246
+ fmt.Fprintf(c, "%s\n", s.canonicalURI(req.URL))
247
+ fmt.Fprintf(c, "%s\n", s.canonicalQueryString(req.URL))
248
+ fmt.Fprintf(c, "%s\n\n", s.canonicalHeaders(req.Header))
249
+ fmt.Fprintf(c, "%s\n", s.signedHeaders(req.Header))
250
+ fmt.Fprintf(c, "%s", payloadHash)
251
+ return c.String()
252
+}
253
+
254
+func (s *V4Signer) canonicalURI(u *url.URL) string {
255
+ u = &url.URL{Path: u.Path}
256
+ canonicalPath := u.String()
257
+
258
+ slash := strings.HasSuffix(canonicalPath, "/")
259
+ canonicalPath = path.Clean(canonicalPath)
260
+
261
+ if canonicalPath == "" || canonicalPath == "." {
262
+ canonicalPath = "/"
263
+ }
264
+
265
+ if canonicalPath != "/" && slash {
266
+ canonicalPath += "/"
267
+ }
268
+
269
+ return canonicalPath
270
+}
271
+
272
+func (s *V4Signer) canonicalQueryString(u *url.URL) string {
273
+ var a []string
274
+ for k, vs := range u.Query() {
275
+ k = url.QueryEscape(k)
276
+ for _, v := range vs {
277
+ if v == "" {
278
+ a = append(a, k+"=")
279
+ } else {
280
+ v = url.QueryEscape(v)
281
+ a = append(a, k+"="+v)
282
+ }
283
+ }
284
+ }
285
+ sort.Strings(a)
286
+ return strings.Join(a, "&")
287
+}
288
+
289
+func (s *V4Signer) canonicalHeaders(h http.Header) string {
290
+ i, a, lowerCase := 0, make([]string, len(h)), make(map[string][]string)
291
+
292
+ for k, v := range h {
293
+ lowerCase[strings.ToLower(k)] = v
294
+ }
295
+
296
+ var keys []string
297
+ for k := range lowerCase {
298
+ keys = append(keys, k)
299
+ }
300
+ sort.Strings(keys)
301
+
302
+ for _, k := range keys {
303
+ v := lowerCase[k]
304
+ for j, w := range v {
305
+ v[j] = strings.Trim(w, " ")
306
+ }
307
+ sort.Strings(v)
308
+ a[i] = strings.ToLower(k) + ":" + strings.Join(v, ",")
309
+ i++
310
+ }
311
+ return strings.Join(a, "\n")
312
+}
313
+
314
+func (s *V4Signer) signedHeaders(h http.Header) string {
315
+ i, a := 0, make([]string, len(h))
316
+ for k, _ := range h {
317
+ a[i] = strings.ToLower(k)
318
+ i++
319
+ }
320
+ sort.Strings(a)
321
+ return strings.Join(a, ";")
322
+}
323
+
324
+func (s *V4Signer) payloadHash(req *http.Request) string {
325
+ var b []byte
326
+ if req.Body == nil {
327
+ b = []byte("")
328
+ } else {
329
+ var err error
330
+ b, err = ioutil.ReadAll(req.Body)
331
+ if err != nil {
332
+ // TODO: I REALLY DON'T LIKE THIS PANIC!!!!
333
+ panic(err)
334
+ }
335
+ }
336
+ req.Body = ioutil.NopCloser(bytes.NewBuffer(b))
337
+ return s.hash(string(b))
338
+}
339
+
340
+/*
341
+stringToSign method creates the string to sign accorting to Task 2 of the AWS Signature Version 4 Signing Process. (http://goo.gl/es1PAu)
342
+
343
+ StringToSign =
344
+ Algorithm + '\n' +
345
+ RequestDate + '\n' +
346
+ CredentialScope + '\n' +
347
+ HexEncode(Hash(CanonicalRequest))
348
+*/
349
+func (s *V4Signer) stringToSign(t time.Time, creq string) string {
350
+ w := new(bytes.Buffer)
351
+ fmt.Fprint(w, "AWS4-HMAC-SHA256\n")
352
+ fmt.Fprintf(w, "%s\n", t.Format(ISO8601BasicFormat))
353
+ fmt.Fprintf(w, "%s\n", s.credentialScope(t))
354
+ fmt.Fprintf(w, "%s", s.hash(creq))
355
+ return w.String()
356
+}
357
+
358
+func (s *V4Signer) credentialScope(t time.Time) string {
359
+ return fmt.Sprintf("%s/%s/%s/aws4_request", t.Format(ISO8601BasicFormatShort), s.region.Name, s.serviceName)
360
+}
361
+
362
+/*
363
+signature method calculates the AWS Signature Version 4 according to Task 3 of the AWS Signature Version 4 Signing Process. (http://goo.gl/j0Yqe1)
364
+
365
+ signature = HexEncode(HMAC(derived-signing-key, string-to-sign))
366
+*/
367
+func (s *V4Signer) signature(t time.Time, sts string) string {
368
+ h := s.hmac(s.derivedKey(t), []byte(sts))
369
+ return fmt.Sprintf("%x", h)
370
+}
371
+
372
+/*
373
+derivedKey method derives a signing key to be used for signing a request.
374
+
375
+ kSecret = Your AWS Secret Access Key
376
+ kDate = HMAC("AWS4" + kSecret, Date)
377
+ kRegion = HMAC(kDate, Region)
378
+ kService = HMAC(kRegion, Service)
379
+ kSigning = HMAC(kService, "aws4_request")
380
+*/
381
+func (s *V4Signer) derivedKey(t time.Time) []byte {
382
+ h := s.hmac([]byte("AWS4"+s.auth.SecretKey), []byte(t.Format(ISO8601BasicFormatShort)))
383
+ h = s.hmac(h, []byte(s.region.Name))
384
+ h = s.hmac(h, []byte(s.serviceName))
385
+ h = s.hmac(h, []byte("aws4_request"))
386
+ return h
387
+}
388
+
389
+/*
390
+authorization method generates the authorization header value.
391
+*/
392
+func (s *V4Signer) authorization(header http.Header, t time.Time, signature string) string {
393
+ w := new(bytes.Buffer)
394
+ fmt.Fprint(w, "AWS4-HMAC-SHA256 ")
395
+ fmt.Fprintf(w, "Credential=%s/%s, ", s.auth.AccessKey, s.credentialScope(t))
396
+ fmt.Fprintf(w, "SignedHeaders=%s, ", s.signedHeaders(header))
397
+ fmt.Fprintf(w, "Signature=%s", signature)
398
+ return w.String()
399
+}
400
+
401
+// hash method calculates the sha256 hash for a given string
402
+func (s *V4Signer) hash(in string) string {
403
+ h := sha256.New()
404
+ fmt.Fprintf(h, "%s", in)
405
+ return fmt.Sprintf("%x", h.Sum(nil))
406
+}
407
+
408
+// hmac method calculates the sha256 hmac for a given slice of bytes
409
+func (s *V4Signer) hmac(key, data []byte) []byte {
410
+ h := hmac.New(sha256.New, key)
411
+ h.Write(data)
412
+ return h.Sum(nil)
413
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/aws/sign_test.go
new
+569
@@ -0,0 +1,569 @@
1
+package aws_test
2
+
3
+import (
4
+ "fmt"
5
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/aws"
6
+ "gopkg.in/check.v1"
7
+ "net/http"
8
+ "strings"
9
+ "time"
10
+)
11
+
12
+var _ = check.Suite(&V4SignerSuite{})
13
+
14
+type V4SignerSuite struct {
15
+ auth aws.Auth
16
+ region aws.Region
17
+ cases []V4SignerSuiteCase
18
+}
19
+
20
+type V4SignerSuiteCase struct {
21
+ label string
22
+ request V4SignerSuiteCaseRequest
23
+ canonicalRequest string
24
+ stringToSign string
25
+ signature string
26
+ authorization string
27
+}
28
+
29
+type V4SignerSuiteCaseRequest struct {
30
+ method string
31
+ host string
32
+ url string
33
+ headers []string
34
+ body string
35
+}
36
+
37
+func (s *V4SignerSuite) SetUpSuite(c *check.C) {
38
+ s.auth = aws.Auth{AccessKey: "AKIDEXAMPLE", SecretKey: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"}
39
+ s.region = aws.USEast
40
+
41
+ // Test cases from the Signature Version 4 Test Suite (http://goo.gl/nguvs0)
42
+ s.cases = append(s.cases,
43
+
44
+ // get-header-key-duplicate
45
+ V4SignerSuiteCase{
46
+ label: "get-header-key-duplicate",
47
+ request: V4SignerSuiteCaseRequest{
48
+ method: "POST",
49
+ host: "host.foo.com",
50
+ url: "/",
51
+ headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT", "ZOO:zoobar", "zoo:foobar", "zoo:zoobar"},
52
+ },
53
+ canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\nzoo:foobar,zoobar,zoobar\n\ndate;host;zoo\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
54
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n3c52f0eaae2b61329c0a332e3fa15842a37bc5812cf4d80eb64784308850e313",
55
+ signature: "54afcaaf45b331f81cd2edb974f7b824ff4dd594cbbaa945ed636b48477368ed",
56
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host;zoo, Signature=54afcaaf45b331f81cd2edb974f7b824ff4dd594cbbaa945ed636b48477368ed",
57
+ },
58
+
59
+ // get-header-value-order
60
+ V4SignerSuiteCase{
61
+ label: "get-header-value-order",
62
+ request: V4SignerSuiteCaseRequest{
63
+ method: "POST",
64
+ host: "host.foo.com",
65
+ url: "/",
66
+ headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT", "p:z", "p:a", "p:p", "p:a"},
67
+ },
68
+ canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\np:a,a,p,z\n\ndate;host;p\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
69
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n94c0389fefe0988cbbedc8606f0ca0b485b48da010d09fc844b45b697c8924fe",
70
+ signature: "d2973954263943b11624a11d1c963ca81fb274169c7868b2858c04f083199e3d",
71
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host;p, Signature=d2973954263943b11624a11d1c963ca81fb274169c7868b2858c04f083199e3d",
72
+ },
73
+
74
+ // get-header-value-trim
75
+ V4SignerSuiteCase{
76
+ label: "get-header-value-trim",
77
+ request: V4SignerSuiteCaseRequest{
78
+ method: "POST",
79
+ host: "host.foo.com",
80
+ url: "/",
81
+ headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT", "p: phfft "},
82
+ },
83
+ canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\np:phfft\n\ndate;host;p\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
84
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\ndddd1902add08da1ac94782b05f9278c08dc7468db178a84f8950d93b30b1f35",
85
+ signature: "debf546796015d6f6ded8626f5ce98597c33b47b9164cf6b17b4642036fcb592",
86
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host;p, Signature=debf546796015d6f6ded8626f5ce98597c33b47b9164cf6b17b4642036fcb592",
87
+ },
88
+
89
+ // get-empty
90
+ V4SignerSuiteCase{
91
+ label: "get-relative-relative",
92
+ request: V4SignerSuiteCaseRequest{
93
+ method: "GET",
94
+ host: "host.foo.com",
95
+ url: "",
96
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
97
+ },
98
+ canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
99
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
100
+ signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
101
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
102
+ },
103
+
104
+ // get-single-relative
105
+ V4SignerSuiteCase{
106
+ label: "get-relative-relative",
107
+ request: V4SignerSuiteCaseRequest{
108
+ method: "GET",
109
+ host: "host.foo.com",
110
+ url: "/.",
111
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
112
+ },
113
+ canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
114
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
115
+ signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
116
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
117
+ },
118
+
119
+ // get-multiple-relative
120
+ V4SignerSuiteCase{
121
+ label: "get-relative-relative",
122
+ request: V4SignerSuiteCaseRequest{
123
+ method: "GET",
124
+ host: "host.foo.com",
125
+ url: "/./././",
126
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
127
+ },
128
+ canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
129
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
130
+ signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
131
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
132
+ },
133
+
134
+ // get-relative-relative
135
+ V4SignerSuiteCase{
136
+ label: "get-relative-relative",
137
+ request: V4SignerSuiteCaseRequest{
138
+ method: "GET",
139
+ host: "host.foo.com",
140
+ url: "/foo/bar/../..",
141
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
142
+ },
143
+ canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
144
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
145
+ signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
146
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
147
+ },
148
+
149
+ // get-relative
150
+ V4SignerSuiteCase{
151
+ label: "get-relative",
152
+ request: V4SignerSuiteCaseRequest{
153
+ method: "GET",
154
+ host: "host.foo.com",
155
+ url: "/foo/..",
156
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
157
+ },
158
+ canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
159
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
160
+ signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
161
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
162
+ },
163
+
164
+ // get-slash-dot-slash
165
+ V4SignerSuiteCase{
166
+ label: "get-slash-dot-slash",
167
+ request: V4SignerSuiteCaseRequest{
168
+ method: "GET",
169
+ host: "host.foo.com",
170
+ url: "/./",
171
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
172
+ },
173
+ canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
174
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
175
+ signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
176
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
177
+ },
178
+
179
+ // get-slash-pointless-dot
180
+ V4SignerSuiteCase{
181
+ label: "get-slash-pointless-dot",
182
+ request: V4SignerSuiteCaseRequest{
183
+ method: "GET",
184
+ host: "host.foo.com",
185
+ url: "/./foo",
186
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
187
+ },
188
+ canonicalRequest: "GET\n/foo\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
189
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n8021a97572ee460f87ca67f4e8c0db763216d84715f5424a843a5312a3321e2d",
190
+ signature: "910e4d6c9abafaf87898e1eb4c929135782ea25bb0279703146455745391e63a",
191
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=910e4d6c9abafaf87898e1eb4c929135782ea25bb0279703146455745391e63a",
192
+ },
193
+
194
+ // get-slash
195
+ V4SignerSuiteCase{
196
+ label: "get-slash",
197
+ request: V4SignerSuiteCaseRequest{
198
+ method: "GET",
199
+ host: "host.foo.com",
200
+ url: "//",
201
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
202
+ },
203
+ canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
204
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
205
+ signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
206
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
207
+ },
208
+
209
+ // get-slashes
210
+ V4SignerSuiteCase{
211
+ label: "get-slashes",
212
+ request: V4SignerSuiteCaseRequest{
213
+ method: "GET",
214
+ host: "host.foo.com",
215
+ url: "//foo//",
216
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
217
+ },
218
+ canonicalRequest: "GET\n/foo/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
219
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n6bb4476ee8745730c9cb79f33a0c70baa6d8af29c0077fa12e4e8f1dd17e7098",
220
+ signature: "b00392262853cfe3201e47ccf945601079e9b8a7f51ee4c3d9ee4f187aa9bf19",
221
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b00392262853cfe3201e47ccf945601079e9b8a7f51ee4c3d9ee4f187aa9bf19",
222
+ },
223
+
224
+ // get-space
225
+ V4SignerSuiteCase{
226
+ label: "get-space",
227
+ request: V4SignerSuiteCaseRequest{
228
+ method: "GET",
229
+ host: "host.foo.com",
230
+ url: "/%20/foo",
231
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
232
+ },
233
+ canonicalRequest: "GET\n/%20/foo\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
234
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n69c45fb9fe3fd76442b5086e50b2e9fec8298358da957b293ef26e506fdfb54b",
235
+ signature: "f309cfbd10197a230c42dd17dbf5cca8a0722564cb40a872d25623cfa758e374",
236
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=f309cfbd10197a230c42dd17dbf5cca8a0722564cb40a872d25623cfa758e374",
237
+ },
238
+
239
+ // get-unreserved
240
+ V4SignerSuiteCase{
241
+ label: "get-unreserved",
242
+ request: V4SignerSuiteCaseRequest{
243
+ method: "GET",
244
+ host: "host.foo.com",
245
+ url: "/-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz",
246
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
247
+ },
248
+ canonicalRequest: "GET\n/-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
249
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\ndf63ee3247c0356c696a3b21f8d8490b01fa9cd5bc6550ef5ef5f4636b7b8901",
250
+ signature: "830cc36d03f0f84e6ee4953fbe701c1c8b71a0372c63af9255aa364dd183281e",
251
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=830cc36d03f0f84e6ee4953fbe701c1c8b71a0372c63af9255aa364dd183281e",
252
+ },
253
+
254
+ // get-utf8
255
+ V4SignerSuiteCase{
256
+ label: "get-utf8",
257
+ request: V4SignerSuiteCaseRequest{
258
+ method: "GET",
259
+ host: "host.foo.com",
260
+ url: "/%E1%88%B4",
261
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
262
+ },
263
+ canonicalRequest: "GET\n/%E1%88%B4\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
264
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n27ba31df5dbc6e063d8f87d62eb07143f7f271c5330a917840586ac1c85b6f6b",
265
+ signature: "8d6634c189aa8c75c2e51e106b6b5121bed103fdb351f7d7d4381c738823af74",
266
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=8d6634c189aa8c75c2e51e106b6b5121bed103fdb351f7d7d4381c738823af74",
267
+ },
268
+
269
+ // get-vanilla-empty-query-key
270
+ V4SignerSuiteCase{
271
+ label: "get-vanilla-empty-query-key",
272
+ request: V4SignerSuiteCaseRequest{
273
+ method: "GET",
274
+ host: "host.foo.com",
275
+ url: "/?foo=bar",
276
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
277
+ },
278
+ canonicalRequest: "GET\n/\nfoo=bar\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
279
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n0846c2945b0832deb7a463c66af5c4f8bd54ec28c438e67a214445b157c9ddf8",
280
+ signature: "56c054473fd260c13e4e7393eb203662195f5d4a1fada5314b8b52b23f985e9f",
281
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=56c054473fd260c13e4e7393eb203662195f5d4a1fada5314b8b52b23f985e9f",
282
+ },
283
+
284
+ // get-vanilla-query-order-key-case
285
+ V4SignerSuiteCase{
286
+ label: "get-vanilla-query-order-key-case",
287
+ request: V4SignerSuiteCaseRequest{
288
+ method: "GET",
289
+ host: "host.foo.com",
290
+ url: "/?foo=Zoo&foo=aha",
291
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
292
+ },
293
+ canonicalRequest: "GET\n/\nfoo=Zoo&foo=aha\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
294
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\ne25f777ba161a0f1baf778a87faf057187cf5987f17953320e3ca399feb5f00d",
295
+ signature: "be7148d34ebccdc6423b19085378aa0bee970bdc61d144bd1a8c48c33079ab09",
296
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=be7148d34ebccdc6423b19085378aa0bee970bdc61d144bd1a8c48c33079ab09",
297
+ },
298
+
299
+ // get-vanilla-query-order-key
300
+ V4SignerSuiteCase{
301
+ label: "get-vanilla-query-order-key",
302
+ request: V4SignerSuiteCaseRequest{
303
+ method: "GET",
304
+ host: "host.foo.com",
305
+ url: "/?a=foo&b=foo",
306
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
307
+ },
308
+ canonicalRequest: "GET\n/\na=foo&b=foo\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
309
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n2f23d14fe13caebf6dfda346285c6d9c14f49eaca8f5ec55c627dd7404f7a727",
310
+ signature: "0dc122f3b28b831ab48ba65cb47300de53fbe91b577fe113edac383730254a3b",
311
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=0dc122f3b28b831ab48ba65cb47300de53fbe91b577fe113edac383730254a3b",
312
+ },
313
+
314
+ // get-vanilla-query-order-value
315
+ V4SignerSuiteCase{
316
+ label: "get-vanilla-query-order-value",
317
+ request: V4SignerSuiteCaseRequest{
318
+ method: "GET",
319
+ host: "host.foo.com",
320
+ url: "/?foo=b&foo=a",
321
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
322
+ },
323
+ canonicalRequest: "GET\n/\nfoo=a&foo=b\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
324
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n33dffc220e89131f8f6157a35c40903daa658608d9129ff9489e5cf5bbd9b11b",
325
+ signature: "feb926e49e382bec75c9d7dcb2a1b6dc8aa50ca43c25d2bc51143768c0875acc",
326
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=feb926e49e382bec75c9d7dcb2a1b6dc8aa50ca43c25d2bc51143768c0875acc",
327
+ },
328
+
329
+ // get-vanilla-query-unreserved
330
+ V4SignerSuiteCase{
331
+ label: "get-vanilla-query-unreserved",
332
+ request: V4SignerSuiteCaseRequest{
333
+ method: "GET",
334
+ host: "host.foo.com",
335
+ url: "/?-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz=-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz",
336
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
337
+ },
338
+ canonicalRequest: "GET\n/\n-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz=-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
339
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\nd2578f3156d4c9d180713d1ff20601d8a3eed0dd35447d24603d7d67414bd6b5",
340
+ signature: "f1498ddb4d6dae767d97c466fb92f1b59a2c71ca29ac954692663f9db03426fb",
341
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=f1498ddb4d6dae767d97c466fb92f1b59a2c71ca29ac954692663f9db03426fb",
342
+ },
343
+
344
+ // get-vanilla-query
345
+ V4SignerSuiteCase{
346
+ label: "get-vanilla-query",
347
+ request: V4SignerSuiteCaseRequest{
348
+ method: "GET",
349
+ host: "host.foo.com",
350
+ url: "/",
351
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
352
+ },
353
+ canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
354
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
355
+ signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
356
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
357
+ },
358
+
359
+ // get-vanilla-ut8-query
360
+ V4SignerSuiteCase{
361
+ label: "get-vanilla-ut8-query",
362
+ request: V4SignerSuiteCaseRequest{
363
+ method: "GET",
364
+ host: "host.foo.com",
365
+ url: "/?ሴ=bar",
366
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
367
+ },
368
+ canonicalRequest: "GET\n/\n%E1%88%B4=bar\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
369
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\nde5065ff39c131e6c2e2bd19cd9345a794bf3b561eab20b8d97b2093fc2a979e",
370
+ signature: "6fb359e9a05394cc7074e0feb42573a2601abc0c869a953e8c5c12e4e01f1a8c",
371
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=6fb359e9a05394cc7074e0feb42573a2601abc0c869a953e8c5c12e4e01f1a8c",
372
+ },
373
+
374
+ // get-vanilla
375
+ V4SignerSuiteCase{
376
+ label: "get-vanilla",
377
+ request: V4SignerSuiteCaseRequest{
378
+ method: "GET",
379
+ host: "host.foo.com",
380
+ url: "/",
381
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
382
+ },
383
+ canonicalRequest: "GET\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
384
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n366b91fb121d72a00f46bbe8d395f53a102b06dfb7e79636515208ed3fa606b1",
385
+ signature: "b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
386
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b27ccfbfa7df52a200ff74193ca6e32d4b48b8856fab7ebf1c595d0670a7e470",
387
+ },
388
+
389
+ // post-header-key-case
390
+ V4SignerSuiteCase{
391
+ label: "post-header-key-case",
392
+ request: V4SignerSuiteCaseRequest{
393
+ method: "POST",
394
+ host: "host.foo.com",
395
+ url: "/",
396
+ headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT"},
397
+ },
398
+ canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
399
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n05da62cee468d24ae84faff3c39f1b85540de60243c1bcaace39c0a2acc7b2c4",
400
+ signature: "22902d79e148b64e7571c3565769328423fe276eae4b26f83afceda9e767f726",
401
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=22902d79e148b64e7571c3565769328423fe276eae4b26f83afceda9e767f726",
402
+ },
403
+
404
+ // post-header-key-sort
405
+ V4SignerSuiteCase{
406
+ label: "post-header-key-sort",
407
+ request: V4SignerSuiteCaseRequest{
408
+ method: "POST",
409
+ host: "host.foo.com",
410
+ url: "/",
411
+ headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT", "ZOO:zoobar"},
412
+ },
413
+ canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\nzoo:zoobar\n\ndate;host;zoo\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
414
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n34e1bddeb99e76ee01d63b5e28656111e210529efeec6cdfd46a48e4c734545d",
415
+ signature: "b7a95a52518abbca0964a999a880429ab734f35ebbf1235bd79a5de87756dc4a",
416
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host;zoo, Signature=b7a95a52518abbca0964a999a880429ab734f35ebbf1235bd79a5de87756dc4a",
417
+ },
418
+
419
+ // post-header-value-case
420
+ V4SignerSuiteCase{
421
+ label: "post-header-value-case",
422
+ request: V4SignerSuiteCaseRequest{
423
+ method: "POST",
424
+ host: "host.foo.com",
425
+ url: "/",
426
+ headers: []string{"DATE:Mon, 09 Sep 2011 23:36:00 GMT", "zoo:ZOOBAR"},
427
+ },
428
+ canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\nzoo:ZOOBAR\n\ndate;host;zoo\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
429
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n3aae6d8274b8c03e2cc96fc7d6bda4b9bd7a0a184309344470b2c96953e124aa",
430
+ signature: "273313af9d0c265c531e11db70bbd653f3ba074c1009239e8559d3987039cad7",
431
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host;zoo, Signature=273313af9d0c265c531e11db70bbd653f3ba074c1009239e8559d3987039cad7",
432
+ },
433
+
434
+ // post-vanilla-empty-query-value
435
+ V4SignerSuiteCase{
436
+ label: "post-vanilla-empty-query-value",
437
+ request: V4SignerSuiteCaseRequest{
438
+ method: "POST",
439
+ host: "host.foo.com",
440
+ url: "/?foo=bar",
441
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
442
+ },
443
+ canonicalRequest: "POST\n/\nfoo=bar\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
444
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\ncd4f39132d8e60bb388831d734230460872b564871c47f5de62e62d1a68dbe1e",
445
+ signature: "b6e3b79003ce0743a491606ba1035a804593b0efb1e20a11cba83f8c25a57a92",
446
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b6e3b79003ce0743a491606ba1035a804593b0efb1e20a11cba83f8c25a57a92",
447
+ },
448
+
449
+ // post-vanilla-query
450
+ V4SignerSuiteCase{
451
+ label: "post-vanilla-query",
452
+ request: V4SignerSuiteCaseRequest{
453
+ method: "POST",
454
+ host: "host.foo.com",
455
+ url: "/?foo=bar",
456
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
457
+ },
458
+ canonicalRequest: "POST\n/\nfoo=bar\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
459
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\ncd4f39132d8e60bb388831d734230460872b564871c47f5de62e62d1a68dbe1e",
460
+ signature: "b6e3b79003ce0743a491606ba1035a804593b0efb1e20a11cba83f8c25a57a92",
461
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=b6e3b79003ce0743a491606ba1035a804593b0efb1e20a11cba83f8c25a57a92",
462
+ },
463
+
464
+ // post-vanilla
465
+ V4SignerSuiteCase{
466
+ label: "post-vanilla",
467
+ request: V4SignerSuiteCaseRequest{
468
+ method: "POST",
469
+ host: "host.foo.com",
470
+ url: "/",
471
+ headers: []string{"Date:Mon, 09 Sep 2011 23:36:00 GMT"},
472
+ },
473
+ canonicalRequest: "POST\n/\n\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ndate;host\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
474
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n05da62cee468d24ae84faff3c39f1b85540de60243c1bcaace39c0a2acc7b2c4",
475
+ signature: "22902d79e148b64e7571c3565769328423fe276eae4b26f83afceda9e767f726",
476
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=date;host, Signature=22902d79e148b64e7571c3565769328423fe276eae4b26f83afceda9e767f726",
477
+ },
478
+
479
+ // post-x-www-form-urlencoded-parameters
480
+ V4SignerSuiteCase{
481
+ label: "post-x-www-form-urlencoded-parameters",
482
+ request: V4SignerSuiteCaseRequest{
483
+ method: "POST",
484
+ host: "host.foo.com",
485
+ url: "/",
486
+ headers: []string{"Content-Type:application/x-www-form-urlencoded; charset=utf8", "Date:Mon, 09 Sep 2011 23:36:00 GMT"},
487
+ body: "foo=bar",
488
+ },
489
+ canonicalRequest: "POST\n/\n\ncontent-type:application/x-www-form-urlencoded; charset=utf8\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ncontent-type;date;host\n3ba8907e7a252327488df390ed517c45b96dead033600219bdca7107d1d3f88a",
490
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\nc4115f9e54b5cecf192b1eaa23b8e88ed8dc5391bd4fde7b3fff3d9c9fe0af1f",
491
+ signature: "b105eb10c6d318d2294de9d49dd8b031b55e3c3fe139f2e637da70511e9e7b71",
492
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=content-type;date;host, Signature=b105eb10c6d318d2294de9d49dd8b031b55e3c3fe139f2e637da70511e9e7b71",
493
+ },
494
+
495
+ // post-x-www-form-urlencoded
496
+ V4SignerSuiteCase{
497
+ label: "post-x-www-form-urlencoded",
498
+ request: V4SignerSuiteCaseRequest{
499
+ method: "POST",
500
+ host: "host.foo.com",
501
+ url: "/",
502
+ headers: []string{"Content-Type:application/x-www-form-urlencoded", "Date:Mon, 09 Sep 2011 23:36:00 GMT"},
503
+ body: "foo=bar",
504
+ },
505
+ canonicalRequest: "POST\n/\n\ncontent-type:application/x-www-form-urlencoded\ndate:Mon, 09 Sep 2011 23:36:00 GMT\nhost:host.foo.com\n\ncontent-type;date;host\n3ba8907e7a252327488df390ed517c45b96dead033600219bdca7107d1d3f88a",
506
+ stringToSign: "AWS4-HMAC-SHA256\n20110909T233600Z\n20110909/us-east-1/host/aws4_request\n4c5c6e4b52fb5fb947a8733982a8a5a61b14f04345cbfe6e739236c76dd48f74",
507
+ signature: "5a15b22cf462f047318703b92e6f4f38884e4a7ab7b1d6426ca46a8bd1c26cbc",
508
+ authorization: "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20110909/us-east-1/host/aws4_request, SignedHeaders=content-type;date;host, Signature=5a15b22cf462f047318703b92e6f4f38884e4a7ab7b1d6426ca46a8bd1c26cbc",
509
+ },
510
+ )
511
+}
512
+
513
+func (s *V4SignerSuite) TestCases(c *check.C) {
514
+ signer := aws.NewV4Signer(s.auth, "host", s.region)
515
+
516
+ for _, testCase := range s.cases {
517
+
518
+ req, err := http.NewRequest(testCase.request.method, "http://"+testCase.request.host+testCase.request.url, strings.NewReader(testCase.request.body))
519
+ c.Assert(err, check.IsNil, check.Commentf("Testcase: %s", testCase.label))
520
+ for _, v := range testCase.request.headers {
521
+ h := strings.SplitN(v, ":", 2)
522
+ req.Header.Add(h[0], h[1])
523
+ }
524
+ req.Header.Set("host", req.Host)
525
+
526
+ t := signer.RequestTime(req)
527
+
528
+ canonicalRequest := signer.CanonicalRequest(req)
529
+ c.Check(canonicalRequest, check.Equals, testCase.canonicalRequest, check.Commentf("Testcase: %s", testCase.label))
530
+
531
+ stringToSign := signer.StringToSign(t, canonicalRequest)
532
+ c.Check(stringToSign, check.Equals, testCase.stringToSign, check.Commentf("Testcase: %s", testCase.label))
533
+
534
+ signature := signer.Signature(t, stringToSign)
535
+ c.Check(signature, check.Equals, testCase.signature, check.Commentf("Testcase: %s", testCase.label))
536
+
537
+ authorization := signer.Authorization(req.Header, t, signature)
538
+ c.Check(authorization, check.Equals, testCase.authorization, check.Commentf("Testcase: %s", testCase.label))
539
+
540
+ signer.Sign(req)
541
+ c.Check(req.Header.Get("Authorization"), check.Equals, testCase.authorization, check.Commentf("Testcase: %s", testCase.label))
542
+ }
543
+}
544
+
545
+func ExampleV4Signer() {
546
+ // Get auth from env vars
547
+ auth, err := aws.EnvAuth()
548
+ if err != nil {
549
+ fmt.Println(err)
550
+ }
551
+
552
+ // Create a signer with the auth, name of the service, and aws region
553
+ signer := aws.NewV4Signer(auth, "dynamodb", aws.USEast)
554
+
555
+ // Create a request
556
+ req, err := http.NewRequest("POST", aws.USEast.DynamoDBEndpoint, strings.NewReader("sample_request"))
557
+ if err != nil {
558
+ fmt.Println(err)
559
+ }
560
+
561
+ // Date or x-amz-date header is required to sign a request
562
+ req.Header.Add("Date", time.Now().UTC().Format(http.TimeFormat))
563
+
564
+ // Sign the request
565
+ signer.Sign(req)
566
+
567
+ // Issue signed request
568
+ http.DefaultClient.Do(req)
569
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/s3/export_test.go
new
+27
@@ -0,0 +1,27 @@
1
+package s3
2
+
3
+import (
4
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/aws"
5
+)
6
+
7
+var originalStrategy = attempts
8
+
9
+func SetAttemptStrategy(s *aws.AttemptStrategy) {
10
+ if s == nil {
11
+ attempts = originalStrategy
12
+ } else {
13
+ attempts = *s
14
+ }
15
+}
16
+
17
+func Sign(auth aws.Auth, method, path string, params, headers map[string][]string) {
18
+ sign(auth, method, path, params, headers)
19
+}
20
+
21
+func SetListPartsMax(n int) {
22
+ listPartsMax = n
23
+}
24
+
25
+func SetListMultiMax(n int) {
26
+ listMultiMax = n
27
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/s3/lifecycle.go
new
+202
@@ -0,0 +1,202 @@
1
+package s3
2
+
3
+import (
4
+ "crypto/md5"
5
+ "encoding/base64"
6
+ "encoding/xml"
7
+ "net/url"
8
+ "strconv"
9
+ "time"
10
+)
11
+
12
+// Implements an interface for s3 bucket lifecycle configuration
13
+// See goo.gl/d0bbDf for details.
14
+
15
+const (
16
+ LifecycleRuleStatusEnabled = "Enabled"
17
+ LifecycleRuleStatusDisabled = "Disabled"
18
+ LifecycleRuleDateFormat = "2006-01-02"
19
+ StorageClassGlacier = "GLACIER"
20
+)
21
+
22
+type Expiration struct {
23
+ Days *uint `xml:"Days,omitempty"`
24
+ Date string `xml:"Date,omitempty"`
25
+}
26
+
27
+// Returns Date as a time.Time.
28
+func (r *Expiration) ParseDate() (time.Time, error) {
29
+ return time.Parse(LifecycleRuleDateFormat, r.Date)
30
+}
31
+
32
+type Transition struct {
33
+ Days *uint `xml:"Days,omitempty"`
34
+ Date string `xml:"Date,omitempty"`
35
+ StorageClass string `xml:"StorageClass"`
36
+}
37
+
38
+// Returns Date as a time.Time.
39
+func (r *Transition) ParseDate() (time.Time, error) {
40
+ return time.Parse(LifecycleRuleDateFormat, r.Date)
41
+}
42
+
43
+type NoncurrentVersionExpiration struct {
44
+ Days *uint `xml:"NoncurrentDays,omitempty"`
45
+}
46
+
47
+type NoncurrentVersionTransition struct {
48
+ Days *uint `xml:"NoncurrentDays,omitempty"`
49
+ StorageClass string `xml:"StorageClass"`
50
+}
51
+
52
+type LifecycleRule struct {
53
+ ID string `xml:"ID"`
54
+ Prefix string `xml:"Prefix"`
55
+ Status string `xml:"Status"`
56
+ NoncurrentVersionTransition *NoncurrentVersionTransition `xml:"NoncurrentVersionTransition,omitempty"`
57
+ NoncurrentVersionExpiration *NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"`
58
+ Transition *Transition `xml:"Transition,omitempty"`
59
+ Expiration *Expiration `xml:"Expiration,omitempty"`
60
+}
61
+
62
+// Create a lifecycle rule with arbitrary identifier id and object name prefix
63
+// for which the rules should apply.
64
+func NewLifecycleRule(id, prefix string) *LifecycleRule {
65
+ rule := &LifecycleRule{
66
+ ID: id,
67
+ Prefix: prefix,
68
+ Status: LifecycleRuleStatusEnabled,
69
+ }
70
+ return rule
71
+}
72
+
73
+// Adds a transition rule in days. Overwrites any previous transition rule.
74
+func (r *LifecycleRule) SetTransitionDays(days uint) {
75
+ r.Transition = &Transition{
76
+ Days: &days,
77
+ StorageClass: StorageClassGlacier,
78
+ }
79
+}
80
+
81
+// Adds a transition rule as a date. Overwrites any previous transition rule.
82
+func (r *LifecycleRule) SetTransitionDate(date time.Time) {
83
+ r.Transition = &Transition{
84
+ Date: date.Format(LifecycleRuleDateFormat),
85
+ StorageClass: StorageClassGlacier,
86
+ }
87
+}
88
+
89
+// Adds an expiration rule in days. Overwrites any previous expiration rule.
90
+// Days must be > 0.
91
+func (r *LifecycleRule) SetExpirationDays(days uint) {
92
+ r.Expiration = &Expiration{
93
+ Days: &days,
94
+ }
95
+}
96
+
97
+// Adds an expiration rule as a date. Overwrites any previous expiration rule.
98
+func (r *LifecycleRule) SetExpirationDate(date time.Time) {
99
+ r.Expiration = &Expiration{
100
+ Date: date.Format(LifecycleRuleDateFormat),
101
+ }
102
+}
103
+
104
+// Adds a noncurrent version transition rule. Overwrites any previous
105
+// noncurrent version transition rule.
106
+func (r *LifecycleRule) SetNoncurrentVersionTransitionDays(days uint) {
107
+ r.NoncurrentVersionTransition = &NoncurrentVersionTransition{
108
+ Days: &days,
109
+ StorageClass: StorageClassGlacier,
110
+ }
111
+}
112
+
113
+// Adds a noncurrent version expiration rule. Days must be > 0. Overwrites
114
+// any previous noncurrent version expiration rule.
115
+func (r *LifecycleRule) SetNoncurrentVersionExpirationDays(days uint) {
116
+ r.NoncurrentVersionExpiration = &NoncurrentVersionExpiration{
117
+ Days: &days,
118
+ }
119
+}
120
+
121
+// Marks the rule as disabled.
122
+func (r *LifecycleRule) Disable() {
123
+ r.Status = LifecycleRuleStatusDisabled
124
+}
125
+
126
+// Marks the rule as enabled (default).
127
+func (r *LifecycleRule) Enable() {
128
+ r.Status = LifecycleRuleStatusEnabled
129
+}
130
+
131
+type LifecycleConfiguration struct {
132
+ XMLName xml.Name `xml:"LifecycleConfiguration"`
133
+ Rules *[]*LifecycleRule `xml:"Rule,omitempty"`
134
+}
135
+
136
+// Adds a LifecycleRule to the configuration.
137
+func (c *LifecycleConfiguration) AddRule(r *LifecycleRule) {
138
+ var rules []*LifecycleRule
139
+ if c.Rules != nil {
140
+ rules = *c.Rules
141
+ }
142
+ rules = append(rules, r)
143
+ c.Rules = &rules
144
+}
145
+
146
+// Sets the bucket's lifecycle configuration.
147
+func (b *Bucket) PutLifecycleConfiguration(c *LifecycleConfiguration) error {
148
+ doc, err := xml.Marshal(c)
149
+ if err != nil {
150
+ return err
151
+ }
152
+
153
+ buf := makeXmlBuffer(doc)
154
+ digest := md5.New()
155
+ size, err := digest.Write(buf.Bytes())
156
+ if err != nil {
157
+ return err
158
+ }
159
+
160
+ headers := map[string][]string{
161
+ "Content-Length": {strconv.FormatInt(int64(size), 10)},
162
+ "Content-MD5": {base64.StdEncoding.EncodeToString(digest.Sum(nil))},
163
+ }
164
+
165
+ req := &request{
166
+ path: "/",
167
+ method: "PUT",
168
+ bucket: b.Name,
169
+ headers: headers,
170
+ payload: buf,
171
+ params: url.Values{"lifecycle": {""}},
172
+ }
173
+
174
+ return b.S3.queryV4Sign(req, nil)
175
+}
176
+
177
+// Retrieves the lifecycle configuration for the bucket. AWS returns an error
178
+// if no lifecycle found.
179
+func (b *Bucket) GetLifecycleConfiguration() (*LifecycleConfiguration, error) {
180
+ req := &request{
181
+ method: "GET",
182
+ bucket: b.Name,
183
+ path: "/",
184
+ params: url.Values{"lifecycle": {""}},
185
+ }
186
+
187
+ conf := &LifecycleConfiguration{}
188
+ err := b.S3.queryV4Sign(req, conf)
189
+ return conf, err
190
+}
191
+
192
+// Delete the bucket's lifecycle configuration.
193
+func (b *Bucket) DeleteLifecycleConfiguration() error {
194
+ req := &request{
195
+ method: "DELETE",
196
+ bucket: b.Name,
197
+ path: "/",
198
+ params: url.Values{"lifecycle": {""}},
199
+ }
200
+
201
+ return b.S3.queryV4Sign(req, nil)
202
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/s3/lifecycle_test.go
new
+205
@@ -0,0 +1,205 @@
1
+package s3_test
2
+
3
+import (
4
+ "encoding/xml"
5
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/s3"
6
+ "gopkg.in/check.v1"
7
+ "io/ioutil"
8
+ "net/http"
9
+ "strings"
10
+ "time"
11
+)
12
+
13
+func (s *S) TestLifecycleConfiguration(c *check.C) {
14
+ date, err := time.Parse(s3.LifecycleRuleDateFormat, "2014-09-10")
15
+ c.Check(err, check.IsNil)
16
+
17
+ conf := &s3.LifecycleConfiguration{}
18
+
19
+ rule := s3.NewLifecycleRule("transition-days", "/")
20
+ rule.SetTransitionDays(7)
21
+ conf.AddRule(rule)
22
+
23
+ rule = s3.NewLifecycleRule("transition-date", "/")
24
+ rule.SetTransitionDate(date)
25
+ conf.AddRule(rule)
26
+
27
+ rule = s3.NewLifecycleRule("expiration-days", "")
28
+ rule.SetExpirationDays(1)
29
+ conf.AddRule(rule)
30
+
31
+ rule = s3.NewLifecycleRule("expiration-date", "")
32
+ rule.SetExpirationDate(date)
33
+ conf.AddRule(rule)
34
+
35
+ rule = s3.NewLifecycleRule("noncurrent-transition", "")
36
+ rule.SetNoncurrentVersionTransitionDays(11)
37
+ conf.AddRule(rule)
38
+
39
+ rule = s3.NewLifecycleRule("noncurrent-expiration", "")
40
+ rule.SetNoncurrentVersionExpirationDays(1011)
41
+
42
+ // Test Disable() and Enable() toggling
43
+ c.Check(rule.Status, check.Equals, s3.LifecycleRuleStatusEnabled)
44
+ rule.Disable()
45
+ c.Check(rule.Status, check.Equals, s3.LifecycleRuleStatusDisabled)
46
+ rule.Enable()
47
+ c.Check(rule.Status, check.Equals, s3.LifecycleRuleStatusEnabled)
48
+ rule.Disable()
49
+ c.Check(rule.Status, check.Equals, s3.LifecycleRuleStatusDisabled)
50
+
51
+ conf.AddRule(rule)
52
+
53
+ doc, err := xml.MarshalIndent(conf, "", " ")
54
+ c.Check(err, check.IsNil)
55
+
56
+ expectedDoc := `<LifecycleConfiguration>
57
+ <Rule>
58
+ <ID>transition-days</ID>
59
+ <Prefix>/</Prefix>
60
+ <Status>Enabled</Status>
61
+ <Transition>
62
+ <Days>7</Days>
63
+ <StorageClass>GLACIER</StorageClass>
64
+ </Transition>
65
+ </Rule>
66
+ <Rule>
67
+ <ID>transition-date</ID>
68
+ <Prefix>/</Prefix>
69
+ <Status>Enabled</Status>
70
+ <Transition>
71
+ <Date>2014-09-10</Date>
72
+ <StorageClass>GLACIER</StorageClass>
73
+ </Transition>
74
+ </Rule>
75
+ <Rule>
76
+ <ID>expiration-days</ID>
77
+ <Prefix></Prefix>
78
+ <Status>Enabled</Status>
79
+ <Expiration>
80
+ <Days>1</Days>
81
+ </Expiration>
82
+ </Rule>
83
+ <Rule>
84
+ <ID>expiration-date</ID>
85
+ <Prefix></Prefix>
86
+ <Status>Enabled</Status>
87
+ <Expiration>
88
+ <Date>2014-09-10</Date>
89
+ </Expiration>
90
+ </Rule>
91
+ <Rule>
92
+ <ID>noncurrent-transition</ID>
93
+ <Prefix></Prefix>
94
+ <Status>Enabled</Status>
95
+ <NoncurrentVersionTransition>
96
+ <NoncurrentDays>11</NoncurrentDays>
97
+ <StorageClass>GLACIER</StorageClass>
98
+ </NoncurrentVersionTransition>
99
+ </Rule>
100
+ <Rule>
101
+ <ID>noncurrent-expiration</ID>
102
+ <Prefix></Prefix>
103
+ <Status>Disabled</Status>
104
+ <NoncurrentVersionExpiration>
105
+ <NoncurrentDays>1011</NoncurrentDays>
106
+ </NoncurrentVersionExpiration>
107
+ </Rule>
108
+</LifecycleConfiguration>`
109
+
110
+ c.Check(string(doc), check.Equals, expectedDoc)
111
+
112
+ // Unmarshalling test
113
+ conf2 := &s3.LifecycleConfiguration{}
114
+ err = xml.Unmarshal(doc, conf2)
115
+ c.Check(err, check.IsNil)
116
+ s.checkLifecycleConfigurationEqual(c, conf, conf2)
117
+}
118
+
119
+func (s *S) checkLifecycleConfigurationEqual(c *check.C, conf, conf2 *s3.LifecycleConfiguration) {
120
+ c.Check(len(*conf2.Rules), check.Equals, len(*conf.Rules))
121
+ for i, rule := range *conf2.Rules {
122
+ confRules := *conf.Rules
123
+ c.Check(rule, check.DeepEquals, confRules[i])
124
+ }
125
+}
126
+
127
+func (s *S) checkLifecycleRequest(c *check.C, req *http.Request) {
128
+ // ?lifecycle= is the only query param
129
+ v, ok := req.Form["lifecycle"]
130
+ c.Assert(ok, check.Equals, true)
131
+ c.Assert(v, check.HasLen, 1)
132
+ c.Assert(v[0], check.Equals, "")
133
+
134
+ c.Assert(req.Header["X-Amz-Date"], check.HasLen, 1)
135
+ c.Assert(req.Header["X-Amz-Date"][0], check.Not(check.Equals), "")
136
+
137
+ // Lifecycle methods require V4 auth
138
+ usesV4 := strings.HasPrefix(req.Header["Authorization"][0], "AWS4-HMAC-SHA256")
139
+ c.Assert(usesV4, check.Equals, true)
140
+}
141
+
142
+func (s *S) TestPutLifecycleConfiguration(c *check.C) {
143
+ testServer.Response(200, nil, "")
144
+
145
+ conf := &s3.LifecycleConfiguration{}
146
+ rule := s3.NewLifecycleRule("id", "")
147
+ rule.SetTransitionDays(7)
148
+ conf.AddRule(rule)
149
+
150
+ doc, err := xml.Marshal(conf)
151
+ c.Check(err, check.IsNil)
152
+
153
+ b := s.s3.Bucket("bucket")
154
+ err = b.PutLifecycleConfiguration(conf)
155
+ c.Assert(err, check.IsNil)
156
+
157
+ req := testServer.WaitRequest()
158
+ c.Assert(req.Method, check.Equals, "PUT")
159
+ c.Assert(req.URL.Path, check.Equals, "/bucket/")
160
+ c.Assert(req.Header["Content-Md5"], check.HasLen, 1)
161
+ c.Assert(req.Header["Content-Md5"][0], check.Not(check.Equals), "")
162
+ s.checkLifecycleRequest(c, req)
163
+
164
+ // Check we sent the correct xml serialization
165
+ data, err := ioutil.ReadAll(req.Body)
166
+ req.Body.Close()
167
+ c.Assert(err, check.IsNil)
168
+ header := "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
169
+ c.Assert(string(data), check.Equals, header+string(doc))
170
+}
171
+
172
+func (s *S) TestGetLifecycleConfiguration(c *check.C) {
173
+ conf := &s3.LifecycleConfiguration{}
174
+ rule := s3.NewLifecycleRule("id", "")
175
+ rule.SetTransitionDays(7)
176
+ conf.AddRule(rule)
177
+
178
+ doc, err := xml.Marshal(conf)
179
+ c.Check(err, check.IsNil)
180
+
181
+ testServer.Response(200, nil, string(doc))
182
+
183
+ b := s.s3.Bucket("bucket")
184
+ conf2, err := b.GetLifecycleConfiguration()
185
+ c.Check(err, check.IsNil)
186
+
187
+ req := testServer.WaitRequest()
188
+ c.Assert(req.Method, check.Equals, "GET")
189
+ c.Assert(req.URL.Path, check.Equals, "/bucket/")
190
+ s.checkLifecycleRequest(c, req)
191
+ s.checkLifecycleConfigurationEqual(c, conf, conf2)
192
+}
193
+
194
+func (s *S) TestDeleteLifecycleConfiguration(c *check.C) {
195
+ testServer.Response(200, nil, "")
196
+
197
+ b := s.s3.Bucket("bucket")
198
+ err := b.DeleteLifecycleConfiguration()
199
+ c.Check(err, check.IsNil)
200
+
201
+ req := testServer.WaitRequest()
202
+ c.Assert(req.Method, check.Equals, "DELETE")
203
+ c.Assert(req.URL.Path, check.Equals, "/bucket/")
204
+ s.checkLifecycleRequest(c, req)
205
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/s3/multi.go
new
+464
@@ -0,0 +1,464 @@
1
+package s3
2
+
3
+import (
4
+ "bytes"
5
+ "crypto/md5"
6
+ "encoding/base64"
7
+ "encoding/hex"
8
+ "encoding/xml"
9
+ "errors"
10
+ "io"
11
+ "net/url"
12
+ "sort"
13
+ "strconv"
14
+ "strings"
15
+)
16
+
17
+// Multi represents an unfinished multipart upload.
18
+//
19
+// Multipart uploads allow sending big objects in smaller chunks.
20
+// After all parts have been sent, the upload must be explicitly
21
+// completed by calling Complete with the list of parts.
22
+//
23
+// See http://goo.gl/vJfTG for an overview of multipart uploads.
24
+type Multi struct {
25
+ Bucket *Bucket
26
+ Key string
27
+ UploadId string
28
+}
29
+
30
+// That's the default. Here just for testing.
31
+var listMultiMax = 1000
32
+
33
+type listMultiResp struct {
34
+ NextKeyMarker string
35
+ NextUploadIdMarker string
36
+ IsTruncated bool
37
+ Upload []Multi
38
+ CommonPrefixes []string `xml:"CommonPrefixes>Prefix"`
39
+}
40
+
41
+// ListMulti returns the list of unfinished multipart uploads in b.
42
+//
43
+// The prefix parameter limits the response to keys that begin with the
44
+// specified prefix. You can use prefixes to separate a bucket into different
45
+// groupings of keys (to get the feeling of folders, for example).
46
+//
47
+// The delim parameter causes the response to group all of the keys that
48
+// share a common prefix up to the next delimiter in a single entry within
49
+// the CommonPrefixes field. You can use delimiters to separate a bucket
50
+// into different groupings of keys, similar to how folders would work.
51
+//
52
+// See http://goo.gl/ePioY for details.
53
+func (b *Bucket) ListMulti(prefix, delim string) (multis []*Multi, prefixes []string, err error) {
54
+ params := map[string][]string{
55
+ "uploads": {""},
56
+ "max-uploads": {strconv.FormatInt(int64(listMultiMax), 10)},
57
+ "prefix": {prefix},
58
+ "delimiter": {delim},
59
+ }
60
+ for attempt := attempts.Start(); attempt.Next(); {
61
+ req := &request{
62
+ method: "GET",
63
+ bucket: b.Name,
64
+ params: params,
65
+ }
66
+ var resp listMultiResp
67
+ err := b.S3.query(req, &resp)
68
+ if shouldRetry(err) && attempt.HasNext() {
69
+ continue
70
+ }
71
+ if err != nil {
72
+ return nil, nil, err
73
+ }
74
+ for i := range resp.Upload {
75
+ multi := &resp.Upload[i]
76
+ multi.Bucket = b
77
+ multis = append(multis, multi)
78
+ }
79
+ prefixes = append(prefixes, resp.CommonPrefixes...)
80
+ if !resp.IsTruncated {
81
+ return multis, prefixes, nil
82
+ }
83
+ params["key-marker"] = []string{resp.NextKeyMarker}
84
+ params["upload-id-marker"] = []string{resp.NextUploadIdMarker}
85
+ attempt = attempts.Start() // Last request worked.
86
+ }
87
+ panic("unreachable")
88
+}
89
+
90
+// Multi returns a multipart upload handler for the provided key
91
+// inside b. If a multipart upload exists for key, it is returned,
92
+// otherwise a new multipart upload is initiated with contType and perm.
93
+func (b *Bucket) Multi(key, contType string, perm ACL, options Options) (*Multi, error) {
94
+ multis, _, err := b.ListMulti(key, "")
95
+ if err != nil && !hasCode(err, "NoSuchUpload") {
96
+ return nil, err
97
+ }
98
+ for _, m := range multis {
99
+ if m.Key == key {
100
+ return m, nil
101
+ }
102
+ }
103
+ return b.InitMulti(key, contType, perm, options)
104
+}
105
+
106
+// InitMulti initializes a new multipart upload at the provided
107
+// key inside b and returns a value for manipulating it.
108
+//
109
+// See http://goo.gl/XP8kL for details.
110
+func (b *Bucket) InitMulti(key string, contType string, perm ACL, options Options) (*Multi, error) {
111
+ headers := map[string][]string{
112
+ "Content-Type": {contType},
113
+ "Content-Length": {"0"},
114
+ "x-amz-acl": {string(perm)},
115
+ }
116
+ options.addHeaders(headers)
117
+ params := map[string][]string{
118
+ "uploads": {""},
119
+ }
120
+ req := &request{
121
+ method: "POST",
122
+ bucket: b.Name,
123
+ path: key,
124
+ headers: headers,
125
+ params: params,
126
+ }
127
+ var err error
128
+ var resp struct {
129
+ UploadId string `xml:"UploadId"`
130
+ }
131
+ for attempt := attempts.Start(); attempt.Next(); {
132
+ err = b.S3.query(req, &resp)
133
+ if !shouldRetry(err) {
134
+ break
135
+ }
136
+ }
137
+ if err != nil {
138
+ return nil, err
139
+ }
140
+ return &Multi{Bucket: b, Key: key, UploadId: resp.UploadId}, nil
141
+}
142
+
143
+func (m *Multi) PutPartCopy(n int, options CopyOptions, source string) (*CopyObjectResult, Part, error) {
144
+ headers := map[string][]string{
145
+ "x-amz-copy-source": {url.QueryEscape(source)},
146
+ }
147
+ options.addHeaders(headers)
148
+ params := map[string][]string{
149
+ "uploadId": {m.UploadId},
150
+ "partNumber": {strconv.FormatInt(int64(n), 10)},
151
+ }
152
+
153
+ sourceBucket := m.Bucket.S3.Bucket(strings.TrimRight(strings.SplitAfterN(source, "/", 2)[0], "/"))
154
+ sourceMeta, err := sourceBucket.Head(strings.SplitAfterN(source, "/", 2)[1], nil)
155
+ if err != nil {
156
+ return nil, Part{}, err
157
+ }
158
+
159
+ for attempt := attempts.Start(); attempt.Next(); {
160
+ req := &request{
161
+ method: "PUT",
162
+ bucket: m.Bucket.Name,
163
+ path: m.Key,
164
+ headers: headers,
165
+ params: params,
166
+ }
167
+ resp := &CopyObjectResult{}
168
+ err = m.Bucket.S3.query(req, resp)
169
+ if shouldRetry(err) && attempt.HasNext() {
170
+ continue
171
+ }
172
+ if err != nil {
173
+ return nil, Part{}, err
174
+ }
175
+ if resp.ETag == "" {
176
+ return nil, Part{}, errors.New("part upload succeeded with no ETag")
177
+ }
178
+ return resp, Part{n, resp.ETag, sourceMeta.ContentLength}, nil
179
+ }
180
+ panic("unreachable")
181
+}
182
+
183
+// PutPart sends part n of the multipart upload, reading all the content from r.
184
+// Each part, except for the last one, must be at least 5MB in size.
185
+//
186
+// See http://goo.gl/pqZer for details.
187
+func (m *Multi) PutPart(n int, r io.ReadSeeker) (Part, error) {
188
+ partSize, _, md5b64, err := seekerInfo(r)
189
+ if err != nil {
190
+ return Part{}, err
191
+ }
192
+ return m.putPart(n, r, partSize, md5b64)
193
+}
194
+
195
+func (m *Multi) putPart(n int, r io.ReadSeeker, partSize int64, md5b64 string) (Part, error) {
196
+ headers := map[string][]string{
197
+ "Content-Length": {strconv.FormatInt(partSize, 10)},
198
+ "Content-MD5": {md5b64},
199
+ }
200
+ params := map[string][]string{
201
+ "uploadId": {m.UploadId},
202
+ "partNumber": {strconv.FormatInt(int64(n), 10)},
203
+ }
204
+ for attempt := attempts.Start(); attempt.Next(); {
205
+ _, err := r.Seek(0, 0)
206
+ if err != nil {
207
+ return Part{}, err
208
+ }
209
+ req := &request{
210
+ method: "PUT",
211
+ bucket: m.Bucket.Name,
212
+ path: m.Key,
213
+ headers: headers,
214
+ params: params,
215
+ payload: r,
216
+ }
217
+ err = m.Bucket.S3.prepare(req)
218
+ if err != nil {
219
+ return Part{}, err
220
+ }
221
+ resp, err := m.Bucket.S3.run(req, nil)
222
+ if shouldRetry(err) && attempt.HasNext() {
223
+ continue
224
+ }
225
+ if err != nil {
226
+ return Part{}, err
227
+ }
228
+ etag := resp.Header.Get("ETag")
229
+ if etag == "" {
230
+ return Part{}, errors.New("part upload succeeded with no ETag")
231
+ }
232
+ return Part{n, etag, partSize}, nil
233
+ }
234
+ panic("unreachable")
235
+}
236
+
237
+func seekerInfo(r io.ReadSeeker) (size int64, md5hex string, md5b64 string, err error) {
238
+ _, err = r.Seek(0, 0)
239
+ if err != nil {
240
+ return 0, "", "", err
241
+ }
242
+ digest := md5.New()
243
+ size, err = io.Copy(digest, r)
244
+ if err != nil {
245
+ return 0, "", "", err
246
+ }
247
+ sum := digest.Sum(nil)
248
+ md5hex = hex.EncodeToString(sum)
249
+ md5b64 = base64.StdEncoding.EncodeToString(sum)
250
+ return size, md5hex, md5b64, nil
251
+}
252
+
253
+type Part struct {
254
+ N int `xml:"PartNumber"`
255
+ ETag string
256
+ Size int64
257
+}
258
+
259
+type partSlice []Part
260
+
261
+func (s partSlice) Len() int { return len(s) }
262
+func (s partSlice) Less(i, j int) bool { return s[i].N < s[j].N }
263
+func (s partSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
264
+
265
+type listPartsResp struct {
266
+ NextPartNumberMarker string
267
+ IsTruncated bool
268
+ Part []Part
269
+}
270
+
271
+// That's the default. Here just for testing.
272
+var listPartsMax = 1000
273
+
274
+// Kept for backcompatability. See the documentation for ListPartsFull
275
+func (m *Multi) ListParts() ([]Part, error) {
276
+ return m.ListPartsFull(0, listPartsMax)
277
+}
278
+
279
+// ListParts returns the list of previously uploaded parts in m,
280
+// ordered by part number (Only parts with higher part numbers than
281
+// partNumberMarker will be listed). Only up to maxParts parts will be
282
+// returned.
283
+//
284
+// See http://goo.gl/ePioY for details.
285
+func (m *Multi) ListPartsFull(partNumberMarker int, maxParts int) ([]Part, error) {
286
+ if maxParts > listPartsMax {
287
+ maxParts = listPartsMax
288
+ }
289
+
290
+ params := map[string][]string{
291
+ "uploadId": {m.UploadId},
292
+ "max-parts": {strconv.FormatInt(int64(maxParts), 10)},
293
+ "part-number-marker": {strconv.FormatInt(int64(partNumberMarker), 10)},
294
+ }
295
+ var parts partSlice
296
+ for attempt := attempts.Start(); attempt.Next(); {
297
+ req := &request{
298
+ method: "GET",
299
+ bucket: m.Bucket.Name,
300
+ path: m.Key,
301
+ params: params,
302
+ }
303
+ var resp listPartsResp
304
+ err := m.Bucket.S3.query(req, &resp)
305
+ if shouldRetry(err) && attempt.HasNext() {
306
+ continue
307
+ }
308
+ if err != nil {
309
+ return nil, err
310
+ }
311
+ parts = append(parts, resp.Part...)
312
+ if !resp.IsTruncated {
313
+ sort.Sort(parts)
314
+ return parts, nil
315
+ }
316
+ params["part-number-marker"] = []string{resp.NextPartNumberMarker}
317
+ attempt = attempts.Start() // Last request worked.
318
+ }
319
+ panic("unreachable")
320
+}
321
+
322
+type ReaderAtSeeker interface {
323
+ io.ReaderAt
324
+ io.ReadSeeker
325
+}
326
+
327
+// PutAll sends all of r via a multipart upload with parts no larger
328
+// than partSize bytes, which must be set to at least 5MB.
329
+// Parts previously uploaded are either reused if their checksum
330
+// and size match the new part, or otherwise overwritten with the
331
+// new content.
332
+// PutAll returns all the parts of m (reused or not).
333
+func (m *Multi) PutAll(r ReaderAtSeeker, partSize int64) ([]Part, error) {
334
+ old, err := m.ListParts()
335
+ if err != nil && !hasCode(err, "NoSuchUpload") {
336
+ return nil, err
337
+ }
338
+ reuse := 0 // Index of next old part to consider reusing.
339
+ current := 1 // Part number of latest good part handled.
340
+ totalSize, err := r.Seek(0, 2)
341
+ if err != nil {
342
+ return nil, err
343
+ }
344
+ first := true // Must send at least one empty part if the file is empty.
345
+ var result []Part
346
+NextSection:
347
+ for offset := int64(0); offset < totalSize || first; offset += partSize {
348
+ first = false
349
+ if offset+partSize > totalSize {
350
+ partSize = totalSize - offset
351
+ }
352
+ section := io.NewSectionReader(r, offset, partSize)
353
+ _, md5hex, md5b64, err := seekerInfo(section)
354
+ if err != nil {
355
+ return nil, err
356
+ }
357
+ for reuse < len(old) && old[reuse].N <= current {
358
+ // Looks like this part was already sent.
359
+ part := &old[reuse]
360
+ etag := `"` + md5hex + `"`
361
+ if part.N == current && part.Size == partSize && part.ETag == etag {
362
+ // Checksum matches. Reuse the old part.
363
+ result = append(result, *part)
364
+ current++
365
+ continue NextSection
366
+ }
367
+ reuse++
368
+ }
369
+
370
+ // Part wasn't found or doesn't match. Send it.
371
+ part, err := m.putPart(current, section, partSize, md5b64)
372
+ if err != nil {
373
+ return nil, err
374
+ }
375
+ result = append(result, part)
376
+ current++
377
+ }
378
+ return result, nil
379
+}
380
+
381
+type completeUpload struct {
382
+ XMLName xml.Name `xml:"CompleteMultipartUpload"`
383
+ Parts completeParts `xml:"Part"`
384
+}
385
+
386
+type completePart struct {
387
+ PartNumber int
388
+ ETag string
389
+}
390
+
391
+type completeParts []completePart
392
+
393
+func (p completeParts) Len() int { return len(p) }
394
+func (p completeParts) Less(i, j int) bool { return p[i].PartNumber < p[j].PartNumber }
395
+func (p completeParts) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
396
+
397
+// Complete assembles the given previously uploaded parts into the
398
+// final object. This operation may take several minutes.
399
+//
400
+// See http://goo.gl/2Z7Tw for details.
401
+func (m *Multi) Complete(parts []Part) error {
402
+ params := map[string][]string{
403
+ "uploadId": {m.UploadId},
404
+ }
405
+ c := completeUpload{}
406
+ for _, p := range parts {
407
+ c.Parts = append(c.Parts, completePart{p.N, p.ETag})
408
+ }
409
+ sort.Sort(c.Parts)
410
+ data, err := xml.Marshal(&c)
411
+ if err != nil {
412
+ return err
413
+ }
414
+ for attempt := attempts.Start(); attempt.Next(); {
415
+ req := &request{
416
+ method: "POST",
417
+ bucket: m.Bucket.Name,
418
+ path: m.Key,
419
+ params: params,
420
+ payload: bytes.NewReader(data),
421
+ }
422
+ err := m.Bucket.S3.query(req, nil)
423
+ if shouldRetry(err) && attempt.HasNext() {
424
+ continue
425
+ }
426
+ return err
427
+ }
428
+ panic("unreachable")
429
+}
430
+
431
+// Abort deletes an unifinished multipart upload and any previously
432
+// uploaded parts for it.
433
+//
434
+// After a multipart upload is aborted, no additional parts can be
435
+// uploaded using it. However, if any part uploads are currently in
436
+// progress, those part uploads might or might not succeed. As a result,
437
+// it might be necessary to abort a given multipart upload multiple
438
+// times in order to completely free all storage consumed by all parts.
439
+//
440
+// NOTE: If the described scenario happens to you, please report back to
441
+// the goamz authors with details. In the future such retrying should be
442
+// handled internally, but it's not clear what happens precisely (Is an
443
+// error returned? Is the issue completely undetectable?).
444
+//
445
+// See http://goo.gl/dnyJw for details.
446
+func (m *Multi) Abort() error {
447
+ params := map[string][]string{
448
+ "uploadId": {m.UploadId},
449
+ }
450
+ for attempt := attempts.Start(); attempt.Next(); {
451
+ req := &request{
452
+ method: "DELETE",
453
+ bucket: m.Bucket.Name,
454
+ path: m.Key,
455
+ params: params,
456
+ }
457
+ err := m.Bucket.S3.query(req, nil)
458
+ if shouldRetry(err) && attempt.HasNext() {
459
+ continue
460
+ }
461
+ return err
462
+ }
463
+ panic("unreachable")
464
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/s3/multi_test.go
new
+425
@@ -0,0 +1,425 @@
1
+package s3_test
2
+
3
+import (
4
+ "encoding/xml"
5
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/s3"
6
+ "gopkg.in/check.v1"
7
+ "io"
8
+ "io/ioutil"
9
+ "strings"
10
+)
11
+
12
+func (s *S) TestInitMulti(c *check.C) {
13
+ testServer.Response(200, nil, InitMultiResultDump)
14
+ b := s.s3.Bucket("sample")
15
+
16
+ metadata := make(map[string][]string)
17
+ metadata["key1"] = []string{"value1"}
18
+ metadata["key2"] = []string{"value2"}
19
+ options := s3.Options{
20
+ SSE: true,
21
+ Meta: metadata,
22
+ ContentEncoding: "text/utf8",
23
+ CacheControl: "no-cache",
24
+ RedirectLocation: "http://github.com/crowdmob/goamz",
25
+ ContentMD5: "0000000000000000",
26
+ }
27
+
28
+ multi, err := b.InitMulti("multi", "text/plain", s3.Private, options)
29
+ c.Assert(err, check.IsNil)
30
+
31
+ req := testServer.WaitRequest()
32
+ c.Assert(req.Method, check.Equals, "POST")
33
+ c.Assert(req.URL.Path, check.Equals, "/sample/multi")
34
+ c.Assert(req.Header["Content-Type"], check.DeepEquals, []string{"text/plain"})
35
+ c.Assert(req.Header["X-Amz-Acl"], check.DeepEquals, []string{"private"})
36
+ c.Assert(req.Form["uploads"], check.DeepEquals, []string{""})
37
+
38
+ c.Assert(req.Header["X-Amz-Server-Side-Encryption"], check.DeepEquals, []string{"AES256"})
39
+ c.Assert(req.Header["Content-Encoding"], check.DeepEquals, []string{"text/utf8"})
40
+ c.Assert(req.Header["Cache-Control"], check.DeepEquals, []string{"no-cache"})
41
+ c.Assert(req.Header["Content-Md5"], check.DeepEquals, []string{"0000000000000000"})
42
+ c.Assert(req.Header["X-Amz-Website-Redirect-Location"], check.DeepEquals, []string{"http://github.com/crowdmob/goamz"})
43
+ c.Assert(req.Header["X-Amz-Meta-Key1"], check.DeepEquals, []string{"value1"})
44
+ c.Assert(req.Header["X-Amz-Meta-Key2"], check.DeepEquals, []string{"value2"})
45
+
46
+ c.Assert(multi.UploadId, check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
47
+}
48
+
49
+func (s *S) TestMultiNoPreviousUpload(c *check.C) {
50
+ // Don't retry the NoSuchUpload error.
51
+ s.DisableRetries()
52
+
53
+ testServer.Response(404, nil, NoSuchUploadErrorDump)
54
+ testServer.Response(200, nil, InitMultiResultDump)
55
+
56
+ b := s.s3.Bucket("sample")
57
+
58
+ multi, err := b.Multi("multi", "text/plain", s3.Private, s3.Options{})
59
+ c.Assert(err, check.IsNil)
60
+
61
+ req := testServer.WaitRequest()
62
+ c.Assert(req.Method, check.Equals, "GET")
63
+ c.Assert(req.URL.Path, check.Equals, "/sample/")
64
+ c.Assert(req.Form["uploads"], check.DeepEquals, []string{""})
65
+ c.Assert(req.Form["prefix"], check.DeepEquals, []string{"multi"})
66
+
67
+ req = testServer.WaitRequest()
68
+ c.Assert(req.Method, check.Equals, "POST")
69
+ c.Assert(req.URL.Path, check.Equals, "/sample/multi")
70
+ c.Assert(req.Form["uploads"], check.DeepEquals, []string{""})
71
+
72
+ c.Assert(multi.UploadId, check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
73
+}
74
+
75
+func (s *S) TestMultiReturnOld(c *check.C) {
76
+ testServer.Response(200, nil, ListMultiResultDump)
77
+
78
+ b := s.s3.Bucket("sample")
79
+
80
+ multi, err := b.Multi("multi1", "text/plain", s3.Private, s3.Options{})
81
+ c.Assert(err, check.IsNil)
82
+ c.Assert(multi.Key, check.Equals, "multi1")
83
+ c.Assert(multi.UploadId, check.Equals, "iUVug89pPvSswrikD")
84
+
85
+ req := testServer.WaitRequest()
86
+ c.Assert(req.Method, check.Equals, "GET")
87
+ c.Assert(req.URL.Path, check.Equals, "/sample/")
88
+ c.Assert(req.Form["uploads"], check.DeepEquals, []string{""})
89
+ c.Assert(req.Form["prefix"], check.DeepEquals, []string{"multi1"})
90
+}
91
+
92
+func (s *S) TestListParts(c *check.C) {
93
+ testServer.Response(200, nil, InitMultiResultDump)
94
+ testServer.Response(200, nil, ListPartsResultDump1)
95
+ testServer.Response(404, nil, NoSuchUploadErrorDump) // :-(
96
+ testServer.Response(200, nil, ListPartsResultDump2)
97
+
98
+ b := s.s3.Bucket("sample")
99
+
100
+ multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
101
+ c.Assert(err, check.IsNil)
102
+
103
+ parts, err := multi.ListParts()
104
+ c.Assert(err, check.IsNil)
105
+ c.Assert(parts, check.HasLen, 3)
106
+ c.Assert(parts[0].N, check.Equals, 1)
107
+ c.Assert(parts[0].Size, check.Equals, int64(5))
108
+ c.Assert(parts[0].ETag, check.Equals, `"ffc88b4ca90a355f8ddba6b2c3b2af5c"`)
109
+ c.Assert(parts[1].N, check.Equals, 2)
110
+ c.Assert(parts[1].Size, check.Equals, int64(5))
111
+ c.Assert(parts[1].ETag, check.Equals, `"d067a0fa9dc61a6e7195ca99696b5a89"`)
112
+ c.Assert(parts[2].N, check.Equals, 3)
113
+ c.Assert(parts[2].Size, check.Equals, int64(5))
114
+ c.Assert(parts[2].ETag, check.Equals, `"49dcd91231f801159e893fb5c6674985"`)
115
+ testServer.WaitRequest()
116
+ req := testServer.WaitRequest()
117
+ c.Assert(req.Method, check.Equals, "GET")
118
+ c.Assert(req.URL.Path, check.Equals, "/sample/multi")
119
+ c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
120
+ c.Assert(req.Form["max-parts"], check.DeepEquals, []string{"1000"})
121
+
122
+ testServer.WaitRequest() // The internal error.
123
+ req = testServer.WaitRequest()
124
+ c.Assert(req.Method, check.Equals, "GET")
125
+ c.Assert(req.URL.Path, check.Equals, "/sample/multi")
126
+ c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
127
+ c.Assert(req.Form["max-parts"], check.DeepEquals, []string{"1000"})
128
+ c.Assert(req.Form["part-number-marker"], check.DeepEquals, []string{"2"})
129
+}
130
+
131
+func (s *S) TestPutPart(c *check.C) {
132
+ headers := map[string]string{
133
+ "ETag": `"26f90efd10d614f100252ff56d88dad8"`,
134
+ }
135
+ testServer.Response(200, nil, InitMultiResultDump)
136
+ testServer.Response(200, headers, "")
137
+
138
+ b := s.s3.Bucket("sample")
139
+
140
+ multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
141
+ c.Assert(err, check.IsNil)
142
+
143
+ part, err := multi.PutPart(1, strings.NewReader("<part 1>"))
144
+ c.Assert(err, check.IsNil)
145
+ c.Assert(part.N, check.Equals, 1)
146
+ c.Assert(part.Size, check.Equals, int64(8))
147
+ c.Assert(part.ETag, check.Equals, headers["ETag"])
148
+
149
+ testServer.WaitRequest()
150
+ req := testServer.WaitRequest()
151
+ c.Assert(req.Method, check.Equals, "PUT")
152
+ c.Assert(req.URL.Path, check.Equals, "/sample/multi")
153
+ c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
154
+ c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"1"})
155
+ c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"8"})
156
+ c.Assert(req.Header["Content-Md5"], check.DeepEquals, []string{"JvkO/RDWFPEAJS/1bYja2A=="})
157
+}
158
+
159
+func (s *S) TestPutPartCopy(c *check.C) {
160
+ testServer.Response(200, nil, InitMultiResultDump)
161
+ // PutPartCopy makes a Head request internally to verify access to the source object
162
+ // and obtain its size
163
+ testServer.Response(200, nil, "content")
164
+ testServer.Response(200, nil, PutCopyResultDump)
165
+
166
+ b := s.s3.Bucket("sample")
167
+
168
+ multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
169
+ c.Assert(err, check.IsNil)
170
+
171
+ res, part, err := multi.PutPartCopy(1, s3.CopyOptions{}, "source-bucket/\u00FCber-fil\u00E9.jpg")
172
+ c.Assert(err, check.IsNil)
173
+ c.Assert(part.N, check.Equals, 1)
174
+ c.Assert(part.Size, check.Equals, int64(7))
175
+ c.Assert(res, check.DeepEquals, &s3.CopyObjectResult{
176
+ ETag: `"9b2cf535f27731c974343645a3985328"`,
177
+ LastModified: `2009-10-28T22:32:00`})
178
+
179
+ // Verify the Head request
180
+ req := testServer.WaitRequest()
181
+ c.Assert(req.Method, check.Equals, "POST")
182
+ c.Assert(req.URL.Path, check.Equals, "/sample/multi")
183
+ c.Assert(req.Header["Date"], check.Not(check.Equals), "")
184
+ c.Assert(err, check.IsNil)
185
+
186
+ testServer.WaitRequest()
187
+ req = testServer.WaitRequest()
188
+ c.Assert(req.Method, check.Equals, "PUT")
189
+ c.Assert(req.URL.Path, check.Equals, "/sample/multi")
190
+ c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
191
+ c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"1"})
192
+ c.Assert(req.Header["X-Amz-Copy-Source"], check.DeepEquals, []string{`source-bucket%2F%C3%BCber-fil%C3%A9.jpg`})
193
+}
194
+
195
+func readAll(r io.Reader) string {
196
+ data, err := ioutil.ReadAll(r)
197
+ if err != nil {
198
+ panic(err)
199
+ }
200
+ return string(data)
201
+}
202
+
203
+func (s *S) TestPutAllNoPreviousUpload(c *check.C) {
204
+ // Don't retry the NoSuchUpload error.
205
+ s.DisableRetries()
206
+
207
+ etag1 := map[string]string{"ETag": `"etag1"`}
208
+ etag2 := map[string]string{"ETag": `"etag2"`}
209
+ etag3 := map[string]string{"ETag": `"etag3"`}
210
+ testServer.Response(200, nil, InitMultiResultDump)
211
+ testServer.Response(404, nil, NoSuchUploadErrorDump)
212
+ testServer.Response(200, etag1, "")
213
+ testServer.Response(200, etag2, "")
214
+ testServer.Response(200, etag3, "")
215
+
216
+ b := s.s3.Bucket("sample")
217
+
218
+ multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
219
+ c.Assert(err, check.IsNil)
220
+
221
+ parts, err := multi.PutAll(strings.NewReader("part1part2last"), 5)
222
+ c.Assert(parts, check.HasLen, 3)
223
+ c.Assert(parts[0].ETag, check.Equals, `"etag1"`)
224
+ c.Assert(parts[1].ETag, check.Equals, `"etag2"`)
225
+ c.Assert(parts[2].ETag, check.Equals, `"etag3"`)
226
+ c.Assert(err, check.IsNil)
227
+
228
+ // Init
229
+ testServer.WaitRequest()
230
+
231
+ // List old parts. Won't find anything.
232
+ req := testServer.WaitRequest()
233
+ c.Assert(req.Method, check.Equals, "GET")
234
+ c.Assert(req.URL.Path, check.Equals, "/sample/multi")
235
+
236
+ // Send part 1.
237
+ req = testServer.WaitRequest()
238
+ c.Assert(req.Method, check.Equals, "PUT")
239
+ c.Assert(req.URL.Path, check.Equals, "/sample/multi")
240
+ c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"1"})
241
+ c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"5"})
242
+ c.Assert(readAll(req.Body), check.Equals, "part1")
243
+
244
+ // Send part 2.
245
+ req = testServer.WaitRequest()
246
+ c.Assert(req.Method, check.Equals, "PUT")
247
+ c.Assert(req.URL.Path, check.Equals, "/sample/multi")
248
+ c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"2"})
249
+ c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"5"})
250
+ c.Assert(readAll(req.Body), check.Equals, "part2")
251
+
252
+ // Send part 3 with shorter body.
253
+ req = testServer.WaitRequest()
254
+ c.Assert(req.Method, check.Equals, "PUT")
255
+ c.Assert(req.URL.Path, check.Equals, "/sample/multi")
256
+ c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"3"})
257
+ c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"4"})
258
+ c.Assert(readAll(req.Body), check.Equals, "last")
259
+}
260
+
261
+func (s *S) TestPutAllZeroSizeFile(c *check.C) {
262
+ // Don't retry the NoSuchUpload error.
263
+ s.DisableRetries()
264
+
265
+ etag1 := map[string]string{"ETag": `"etag1"`}
266
+ testServer.Response(200, nil, InitMultiResultDump)
267
+ testServer.Response(404, nil, NoSuchUploadErrorDump)
268
+ testServer.Response(200, etag1, "")
269
+
270
+ b := s.s3.Bucket("sample")
271
+
272
+ multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
273
+ c.Assert(err, check.IsNil)
274
+
275
+ // Must send at least one part, so that completing it will work.
276
+ parts, err := multi.PutAll(strings.NewReader(""), 5)
277
+ c.Assert(parts, check.HasLen, 1)
278
+ c.Assert(parts[0].ETag, check.Equals, `"etag1"`)
279
+ c.Assert(err, check.IsNil)
280
+
281
+ // Init
282
+ testServer.WaitRequest()
283
+
284
+ // List old parts. Won't find anything.
285
+ req := testServer.WaitRequest()
286
+ c.Assert(req.Method, check.Equals, "GET")
287
+ c.Assert(req.URL.Path, check.Equals, "/sample/multi")
288
+
289
+ // Send empty part.
290
+ req = testServer.WaitRequest()
291
+ c.Assert(req.Method, check.Equals, "PUT")
292
+ c.Assert(req.URL.Path, check.Equals, "/sample/multi")
293
+ c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"1"})
294
+ c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"0"})
295
+ c.Assert(readAll(req.Body), check.Equals, "")
296
+}
297
+
298
+func (s *S) TestPutAllResume(c *check.C) {
299
+ etag2 := map[string]string{"ETag": `"etag2"`}
300
+ testServer.Response(200, nil, InitMultiResultDump)
301
+ testServer.Response(200, nil, ListPartsResultDump1)
302
+ testServer.Response(200, nil, ListPartsResultDump2)
303
+ testServer.Response(200, etag2, "")
304
+
305
+ b := s.s3.Bucket("sample")
306
+
307
+ multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
308
+ c.Assert(err, check.IsNil)
309
+
310
+ // "part1" and "part3" match the checksums in ResultDump1.
311
+ // The middle one is a mismatch (it refers to "part2").
312
+ parts, err := multi.PutAll(strings.NewReader("part1partXpart3"), 5)
313
+ c.Assert(parts, check.HasLen, 3)
314
+ c.Assert(parts[0].N, check.Equals, 1)
315
+ c.Assert(parts[0].Size, check.Equals, int64(5))
316
+ c.Assert(parts[0].ETag, check.Equals, `"ffc88b4ca90a355f8ddba6b2c3b2af5c"`)
317
+ c.Assert(parts[1].N, check.Equals, 2)
318
+ c.Assert(parts[1].Size, check.Equals, int64(5))
319
+ c.Assert(parts[1].ETag, check.Equals, `"etag2"`)
320
+ c.Assert(parts[2].N, check.Equals, 3)
321
+ c.Assert(parts[2].Size, check.Equals, int64(5))
322
+ c.Assert(parts[2].ETag, check.Equals, `"49dcd91231f801159e893fb5c6674985"`)
323
+ c.Assert(err, check.IsNil)
324
+
325
+ // Init
326
+ testServer.WaitRequest()
327
+
328
+ // List old parts, broken in two requests.
329
+ for i := 0; i < 2; i++ {
330
+ req := testServer.WaitRequest()
331
+ c.Assert(req.Method, check.Equals, "GET")
332
+ c.Assert(req.URL.Path, check.Equals, "/sample/multi")
333
+ }
334
+
335
+ // Send part 2, as it didn't match the checksum.
336
+ req := testServer.WaitRequest()
337
+ c.Assert(req.Method, check.Equals, "PUT")
338
+ c.Assert(req.URL.Path, check.Equals, "/sample/multi")
339
+ c.Assert(req.Form["partNumber"], check.DeepEquals, []string{"2"})
340
+ c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"5"})
341
+ c.Assert(readAll(req.Body), check.Equals, "partX")
342
+}
343
+
344
+func (s *S) TestMultiComplete(c *check.C) {
345
+ testServer.Response(200, nil, InitMultiResultDump)
346
+ // Note the 200 response. Completing will hold the connection on some
347
+ // kind of long poll, and may return a late error even after a 200.
348
+ testServer.Response(200, nil, InternalErrorDump)
349
+ testServer.Response(200, nil, "")
350
+
351
+ b := s.s3.Bucket("sample")
352
+
353
+ multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
354
+ c.Assert(err, check.IsNil)
355
+
356
+ err = multi.Complete([]s3.Part{{2, `"ETag2"`, 32}, {1, `"ETag1"`, 64}})
357
+ c.Assert(err, check.IsNil)
358
+
359
+ testServer.WaitRequest()
360
+ req := testServer.WaitRequest()
361
+ c.Assert(req.Method, check.Equals, "POST")
362
+ c.Assert(req.URL.Path, check.Equals, "/sample/multi")
363
+ c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
364
+
365
+ var payload struct {
366
+ XMLName xml.Name
367
+ Part []struct {
368
+ PartNumber int
369
+ ETag string
370
+ }
371
+ }
372
+
373
+ dec := xml.NewDecoder(req.Body)
374
+ err = dec.Decode(&payload)
375
+ c.Assert(err, check.IsNil)
376
+
377
+ c.Assert(payload.XMLName.Local, check.Equals, "CompleteMultipartUpload")
378
+ c.Assert(len(payload.Part), check.Equals, 2)
379
+ c.Assert(payload.Part[0].PartNumber, check.Equals, 1)
380
+ c.Assert(payload.Part[0].ETag, check.Equals, `"ETag1"`)
381
+ c.Assert(payload.Part[1].PartNumber, check.Equals, 2)
382
+ c.Assert(payload.Part[1].ETag, check.Equals, `"ETag2"`)
383
+}
384
+
385
+func (s *S) TestMultiAbort(c *check.C) {
386
+ testServer.Response(200, nil, InitMultiResultDump)
387
+ testServer.Response(200, nil, "")
388
+
389
+ b := s.s3.Bucket("sample")
390
+
391
+ multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
392
+ c.Assert(err, check.IsNil)
393
+
394
+ err = multi.Abort()
395
+ c.Assert(err, check.IsNil)
396
+
397
+ testServer.WaitRequest()
398
+ req := testServer.WaitRequest()
399
+ c.Assert(req.Method, check.Equals, "DELETE")
400
+ c.Assert(req.URL.Path, check.Equals, "/sample/multi")
401
+ c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
402
+}
403
+
404
+func (s *S) TestListMulti(c *check.C) {
405
+ testServer.Response(200, nil, ListMultiResultDump)
406
+
407
+ b := s.s3.Bucket("sample")
408
+
409
+ multis, prefixes, err := b.ListMulti("", "/")
410
+ c.Assert(err, check.IsNil)
411
+ c.Assert(prefixes, check.DeepEquals, []string{"a/", "b/"})
412
+ c.Assert(multis, check.HasLen, 2)
413
+ c.Assert(multis[0].Key, check.Equals, "multi1")
414
+ c.Assert(multis[0].UploadId, check.Equals, "iUVug89pPvSswrikD")
415
+ c.Assert(multis[1].Key, check.Equals, "multi2")
416
+ c.Assert(multis[1].UploadId, check.Equals, "DkirwsSvPp98guVUi")
417
+
418
+ req := testServer.WaitRequest()
419
+ c.Assert(req.Method, check.Equals, "GET")
420
+ c.Assert(req.URL.Path, check.Equals, "/sample/")
421
+ c.Assert(req.Form["uploads"], check.DeepEquals, []string{""})
422
+ c.Assert(req.Form["prefix"], check.DeepEquals, []string{""})
423
+ c.Assert(req.Form["delimiter"], check.DeepEquals, []string{"/"})
424
+ c.Assert(req.Form["max-uploads"], check.DeepEquals, []string{"1000"})
425
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/s3/responses_test.go
new
+239
@@ -0,0 +1,239 @@
1
+package s3_test
2
+
3
+var PutCopyResultDump = `
4
+<?xml version="1.0" encoding="UTF-8"?>
5
+<CopyObjectResult>
6
+ <LastModified>2009-10-28T22:32:00</LastModified>
7
+ <ETag>"9b2cf535f27731c974343645a3985328"</ETag>
8
+</CopyObjectResult>
9
+`
10
+
11
+var GetObjectErrorDump = `
12
+<?xml version="1.0" encoding="UTF-8"?>
13
+<Error><Code>NoSuchBucket</Code><Message>The specified bucket does not exist</Message>
14
+<BucketName>non-existent-bucket</BucketName><RequestId>3F1B667FAD71C3D8</RequestId>
15
+<HostId>L4ee/zrm1irFXY5F45fKXIRdOf9ktsKY/8TDVawuMK2jWRb1RF84i1uBzkdNqS5D</HostId></Error>
16
+`
17
+
18
+var GetListResultDump1 = `
19
+<?xml version="1.0" encoding="UTF-8"?>
20
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
21
+ <Name>quotes</Name>
22
+ <Prefix>N</Prefix>
23
+ <IsTruncated>false</IsTruncated>
24
+ <Contents>
25
+ <Key>Nelson</Key>
26
+ <LastModified>2006-01-01T12:00:00.000Z</LastModified>
27
+ <ETag>"828ef3fdfa96f00ad9f27c383fc9ac7f"</ETag>
28
+ <Size>5</Size>
29
+ <StorageClass>STANDARD</StorageClass>
30
+ <Owner>
31
+ <ID>bcaf161ca5fb16fd081034f</ID>
32
+ <DisplayName>webfile</DisplayName>
33
+ </Owner>
34
+ </Contents>
35
+ <Contents>
36
+ <Key>Neo</Key>
37
+ <LastModified>2006-01-01T12:00:00.000Z</LastModified>
38
+ <ETag>"828ef3fdfa96f00ad9f27c383fc9ac7f"</ETag>
39
+ <Size>4</Size>
40
+ <StorageClass>STANDARD</StorageClass>
41
+ <Owner>
42
+ <ID>bcaf1ffd86a5fb16fd081034f</ID>
43
+ <DisplayName>webfile</DisplayName>
44
+ </Owner>
45
+ </Contents>
46
+</ListBucketResult>
47
+`
48
+
49
+var GetListResultDump2 = `
50
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
51
+ <Name>example-bucket</Name>
52
+ <Prefix>photos/2006/</Prefix>
53
+ <Marker>some-marker</Marker>
54
+ <MaxKeys>1000</MaxKeys>
55
+ <Delimiter>/</Delimiter>
56
+ <IsTruncated>false</IsTruncated>
57
+
58
+ <CommonPrefixes>
59
+ <Prefix>photos/2006/feb/</Prefix>
60
+ </CommonPrefixes>
61
+ <CommonPrefixes>
62
+ <Prefix>photos/2006/jan/</Prefix>
63
+ </CommonPrefixes>
64
+</ListBucketResult>
65
+`
66
+
67
+var InitMultiResultDump = `
68
+<?xml version="1.0" encoding="UTF-8"?>
69
+<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
70
+ <Bucket>sample</Bucket>
71
+ <Key>multi</Key>
72
+ <UploadId>JNbR_cMdwnGiD12jKAd6WK2PUkfj2VxA7i4nCwjE6t71nI9Tl3eVDPFlU0nOixhftH7I17ZPGkV3QA.l7ZD.QQ--</UploadId>
73
+</InitiateMultipartUploadResult>
74
+`
75
+
76
+var ListPartsResultDump1 = `
77
+<?xml version="1.0" encoding="UTF-8"?>
78
+<ListPartsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
79
+ <Bucket>sample</Bucket>
80
+ <Key>multi</Key>
81
+ <UploadId>JNbR_cMdwnGiD12jKAd6WK2PUkfj2VxA7i4nCwjE6t71nI9Tl3eVDPFlU0nOixhftH7I17ZPGkV3QA.l7ZD.QQ--</UploadId>
82
+ <Initiator>
83
+ <ID>bb5c0f63b0b25f2d099c</ID>
84
+ <DisplayName>joe</DisplayName>
85
+ </Initiator>
86
+ <Owner>
87
+ <ID>bb5c0f63b0b25f2d099c</ID>
88
+ <DisplayName>joe</DisplayName>
89
+ </Owner>
90
+ <StorageClass>STANDARD</StorageClass>
91
+ <PartNumberMarker>0</PartNumberMarker>
92
+ <NextPartNumberMarker>2</NextPartNumberMarker>
93
+ <MaxParts>2</MaxParts>
94
+ <IsTruncated>true</IsTruncated>
95
+ <Part>
96
+ <PartNumber>1</PartNumber>
97
+ <LastModified>2013-01-30T13:45:51.000Z</LastModified>
98
+ <ETag>"ffc88b4ca90a355f8ddba6b2c3b2af5c"</ETag>
99
+ <Size>5</Size>
100
+ </Part>
101
+ <Part>
102
+ <PartNumber>2</PartNumber>
103
+ <LastModified>2013-01-30T13:45:52.000Z</LastModified>
104
+ <ETag>"d067a0fa9dc61a6e7195ca99696b5a89"</ETag>
105
+ <Size>5</Size>
106
+ </Part>
107
+</ListPartsResult>
108
+`
109
+
110
+var ListPartsResultDump2 = `
111
+<?xml version="1.0" encoding="UTF-8"?>
112
+<ListPartsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
113
+ <Bucket>sample</Bucket>
114
+ <Key>multi</Key>
115
+ <UploadId>JNbR_cMdwnGiD12jKAd6WK2PUkfj2VxA7i4nCwjE6t71nI9Tl3eVDPFlU0nOixhftH7I17ZPGkV3QA.l7ZD.QQ--</UploadId>
116
+ <Initiator>
117
+ <ID>bb5c0f63b0b25f2d099c</ID>
118
+ <DisplayName>joe</DisplayName>
119
+ </Initiator>
120
+ <Owner>
121
+ <ID>bb5c0f63b0b25f2d099c</ID>
122
+ <DisplayName>joe</DisplayName>
123
+ </Owner>
124
+ <StorageClass>STANDARD</StorageClass>
125
+ <PartNumberMarker>2</PartNumberMarker>
126
+ <NextPartNumberMarker>3</NextPartNumberMarker>
127
+ <MaxParts>2</MaxParts>
128
+ <IsTruncated>false</IsTruncated>
129
+ <Part>
130
+ <PartNumber>3</PartNumber>
131
+ <LastModified>2013-01-30T13:46:50.000Z</LastModified>
132
+ <ETag>"49dcd91231f801159e893fb5c6674985"</ETag>
133
+ <Size>5</Size>
134
+ </Part>
135
+</ListPartsResult>
136
+`
137
+
138
+var ListMultiResultDump = `
139
+<?xml version="1.0"?>
140
+<ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
141
+ <Bucket>goamz-test-bucket-us-east-1-akiajk3wyewhctyqbf7a</Bucket>
142
+ <KeyMarker/>
143
+ <UploadIdMarker/>
144
+ <NextKeyMarker>multi1</NextKeyMarker>
145
+ <NextUploadIdMarker>iUVug89pPvSswrikD72p8uO62EzhNtpDxRmwC5WSiWDdK9SfzmDqe3xpP1kMWimyimSnz4uzFc3waVM5ufrKYQ--</NextUploadIdMarker>
146
+ <Delimiter>/</Delimiter>
147
+ <MaxUploads>1000</MaxUploads>
148
+ <IsTruncated>false</IsTruncated>
149
+ <Upload>
150
+ <Key>multi1</Key>
151
+ <UploadId>iUVug89pPvSswrikD</UploadId>
152
+ <Initiator>
153
+ <ID>bb5c0f63b0b25f2d0</ID>
154
+ <DisplayName>gustavoniemeyer</DisplayName>
155
+ </Initiator>
156
+ <Owner>
157
+ <ID>bb5c0f63b0b25f2d0</ID>
158
+ <DisplayName>gustavoniemeyer</DisplayName>
159
+ </Owner>
160
+ <StorageClass>STANDARD</StorageClass>
161
+ <Initiated>2013-01-30T18:15:47.000Z</Initiated>
162
+ </Upload>
163
+ <Upload>
164
+ <Key>multi2</Key>
165
+ <UploadId>DkirwsSvPp98guVUi</UploadId>
166
+ <Initiator>
167
+ <ID>bb5c0f63b0b25f2d0</ID>
168
+ <DisplayName>joe</DisplayName>
169
+ </Initiator>
170
+ <Owner>
171
+ <ID>bb5c0f63b0b25f2d0</ID>
172
+ <DisplayName>joe</DisplayName>
173
+ </Owner>
174
+ <StorageClass>STANDARD</StorageClass>
175
+ <Initiated>2013-01-30T18:15:47.000Z</Initiated>
176
+ </Upload>
177
+ <CommonPrefixes>
178
+ <Prefix>a/</Prefix>
179
+ </CommonPrefixes>
180
+ <CommonPrefixes>
181
+ <Prefix>b/</Prefix>
182
+ </CommonPrefixes>
183
+</ListMultipartUploadsResult>
184
+`
185
+
186
+var NoSuchUploadErrorDump = `
187
+<?xml version="1.0" encoding="UTF-8"?>
188
+<Error>
189
+ <Code>NoSuchUpload</Code>
190
+ <Message>Not relevant</Message>
191
+ <BucketName>sample</BucketName>
192
+ <RequestId>3F1B667FAD71C3D8</RequestId>
193
+ <HostId>kjhwqk</HostId>
194
+</Error>
195
+`
196
+
197
+var InternalErrorDump = `
198
+<?xml version="1.0" encoding="UTF-8"?>
199
+<Error>
200
+ <Code>InternalError</Code>
201
+ <Message>Not relevant</Message>
202
+ <BucketName>sample</BucketName>
203
+ <RequestId>3F1B667FAD71C3D8</RequestId>
204
+ <HostId>kjhwqk</HostId>
205
+</Error>
206
+`
207
+
208
+var GetServiceDump = `
209
+<?xml version="1.0" encoding="UTF-8"?>
210
+<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
211
+ <Owner>
212
+ <ID>bcaf1ffd86f461ca5fb16fd081034f</ID>
213
+ <DisplayName>webfile</DisplayName>
214
+ </Owner>
215
+ <Buckets>
216
+ <Bucket>
217
+ <Name>quotes</Name>
218
+ <CreationDate>2006-02-03T16:45:09.000Z</CreationDate>
219
+ </Bucket>
220
+ <Bucket>
221
+ <Name>samples</Name>
222
+ <CreationDate>2006-02-03T16:41:58.000Z</CreationDate>
223
+ </Bucket>
224
+ </Buckets>
225
+</ListAllMyBucketsResult>
226
+`
227
+
228
+var GetLocationUsStandard = `
229
+<?xml version="1.0" encoding="UTF-8"?>
230
+<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>
231
+`
232
+
233
+var GetLocationUsWest1 = `
234
+<?xml version="1.0" encoding="UTF-8"?>
235
+<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">us-west-1</LocationConstraint>
236
+`
237
+
238
+var BucketWebsiteConfigurationDump = `<?xml version="1.0" encoding="UTF-8"?>
239
+<WebsiteConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><RedirectAllRequestsTo><HostName>example.com</HostName></RedirectAllRequestsTo></WebsiteConfiguration>`
Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3.go
new
+1248
@@ -0,0 +1,1248 @@
1
+//
2
+// goamz - Go packages to interact with the Amazon Web Services.
3
+//
4
+// https://wiki.ubuntu.com/goamz
5
+//
6
+// Copyright (c) 2011 Canonical Ltd.
7
+//
8
+// Written by Gustavo Niemeyer <gustavo.niemeyer@canonical.com>
9
+//
10
+
11
+package s3
12
+
13
+import (
14
+ "bytes"
15
+ "crypto/hmac"
16
+ "crypto/md5"
17
+ "crypto/sha1"
18
+ "encoding/base64"
19
+ "encoding/xml"
20
+ "fmt"
21
+ "io"
22
+ "io/ioutil"
23
+ "log"
24
+ "net"
25
+ "net/http"
26
+ "net/http/httputil"
27
+ "net/url"
28
+ "strconv"
29
+ "strings"
30
+ "time"
31
+
32
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/aws"
33
+)
34
+
35
+const debug = false
36
+
37
+// The S3 type encapsulates operations with an S3 region.
38
+type S3 struct {
39
+ aws.Auth
40
+ aws.Region
41
+ ConnectTimeout time.Duration
42
+ ReadTimeout time.Duration
43
+ Signature int
44
+ private byte // Reserve the right of using private data.
45
+}
46
+
47
+// The Bucket type encapsulates operations with an S3 bucket.
48
+type Bucket struct {
49
+ *S3
50
+ Name string
51
+}
52
+
53
+// The Owner type represents the owner of the object in an S3 bucket.
54
+type Owner struct {
55
+ ID string
56
+ DisplayName string
57
+}
58
+
59
+// Fold options into an Options struct
60
+//
61
+type Options struct {
62
+ SSE bool
63
+ SSECustomerAlgorithm string
64
+ SSECustomerKey string
65
+ SSECustomerKeyMD5 string
66
+ Meta map[string][]string
67
+ ContentEncoding string
68
+ CacheControl string
69
+ RedirectLocation string
70
+ ContentMD5 string
71
+ ContentDisposition string
72
+ Range string
73
+ // What else?
74
+ //// The following become headers so they are []strings rather than strings... I think
75
+ // x-amz-storage-class []string
76
+}
77
+
78
+type CopyOptions struct {
79
+ Options
80
+ CopySourceOptions string
81
+ MetadataDirective string
82
+ ContentType string
83
+}
84
+
85
+// CopyObjectResult is the output from a Copy request
86
+type CopyObjectResult struct {
87
+ ETag string
88
+ LastModified string
89
+}
90
+
91
+var attempts = aws.AttemptStrategy{
92
+ Min: 5,
93
+ Total: 5 * time.Second,
94
+ Delay: 200 * time.Millisecond,
95
+}
96
+
97
+// New creates a new S3.
98
+func New(auth aws.Auth, region aws.Region) *S3 {
99
+ return &S3{auth, region, 0, 0, 0, aws.V2Signature}
100
+}
101
+
102
+// Bucket returns a Bucket with the given name.
103
+func (s3 *S3) Bucket(name string) *Bucket {
104
+ if s3.Region.S3BucketEndpoint != "" || s3.Region.S3LowercaseBucket {
105
+ name = strings.ToLower(name)
106
+ }
107
+ return &Bucket{s3, name}
108
+}
109
+
110
+type BucketInfo struct {
111
+ Name string
112
+ CreationDate string
113
+}
114
+
115
+type GetServiceResp struct {
116
+ Owner Owner
117
+ Buckets []BucketInfo `xml:">Bucket"`
118
+}
119
+
120
+// GetService gets a list of all buckets owned by an account.
121
+//
122
+// See http://goo.gl/wbHkGj for details.
123
+func (s3 *S3) GetService() (*GetServiceResp, error) {
124
+ bucket := s3.Bucket("")
125
+
126
+ r, err := bucket.Get("")
127
+ if err != nil {
128
+ return nil, err
129
+ }
130
+
131
+ // Parse the XML response.
132
+ var resp GetServiceResp
133
+ if err = xml.Unmarshal(r, &resp); err != nil {
134
+ return nil, err
135
+ }
136
+
137
+ return &resp, nil
138
+}
139
+
140
+var createBucketConfiguration = `<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
141
+ <LocationConstraint>%s</LocationConstraint>
142
+</CreateBucketConfiguration>`
143
+
144
+// locationConstraint returns an io.Reader specifying a LocationConstraint if
145
+// required for the region.
146
+//
147
+// See http://goo.gl/bh9Kq for details.
148
+func (s3 *S3) locationConstraint() io.Reader {
149
+ constraint := ""
150
+ if s3.Region.S3LocationConstraint {
151
+ constraint = fmt.Sprintf(createBucketConfiguration, s3.Region.Name)
152
+ }
153
+ return strings.NewReader(constraint)
154
+}
155
+
156
+type ACL string
157
+
158
+const (
159
+ Private = ACL("private")
160
+ PublicRead = ACL("public-read")
161
+ PublicReadWrite = ACL("public-read-write")
162
+ AuthenticatedRead = ACL("authenticated-read")
163
+ BucketOwnerRead = ACL("bucket-owner-read")
164
+ BucketOwnerFull = ACL("bucket-owner-full-control")
165
+)
166
+
167
+// PutBucket creates a new bucket.
168
+//
169
+// See http://goo.gl/ndjnR for details.
170
+func (b *Bucket) PutBucket(perm ACL) error {
171
+ headers := map[string][]string{
172
+ "x-amz-acl": {string(perm)},
173
+ }
174
+ req := &request{
175
+ method: "PUT",
176
+ bucket: b.Name,
177
+ path: "/",
178
+ headers: headers,
179
+ payload: b.locationConstraint(),
180
+ }
181
+ return b.S3.query(req, nil)
182
+}
183
+
184
+// DelBucket removes an existing S3 bucket. All objects in the bucket must
185
+// be removed before the bucket itself can be removed.
186
+//
187
+// See http://goo.gl/GoBrY for details.
188
+func (b *Bucket) DelBucket() (err error) {
189
+ req := &request{
190
+ method: "DELETE",
191
+ bucket: b.Name,
192
+ path: "/",
193
+ }
194
+ for attempt := attempts.Start(); attempt.Next(); {
195
+ err = b.S3.query(req, nil)
196
+ if !shouldRetry(err) {
197
+ break
198
+ }
199
+ }
200
+ return err
201
+}
202
+
203
+// Get retrieves an object from an S3 bucket.
204
+//
205
+// See http://goo.gl/isCO7 for details.
206
+func (b *Bucket) Get(path string) (data []byte, err error) {
207
+ body, err := b.GetReader(path)
208
+ if err != nil {
209
+ return nil, err
210
+ }
211
+ data, err = ioutil.ReadAll(body)
212
+ body.Close()
213
+ return data, err
214
+}
215
+
216
+// GetReader retrieves an object from an S3 bucket,
217
+// returning the body of the HTTP response.
218
+// It is the caller's responsibility to call Close on rc when
219
+// finished reading.
220
+func (b *Bucket) GetReader(path string) (rc io.ReadCloser, err error) {
221
+ resp, err := b.GetResponse(path)
222
+ if resp != nil {
223
+ return resp.Body, err
224
+ }
225
+ return nil, err
226
+}
227
+
228
+// GetResponse retrieves an object from an S3 bucket,
229
+// returning the HTTP response.
230
+// It is the caller's responsibility to call Close on rc when
231
+// finished reading
232
+func (b *Bucket) GetResponse(path string) (resp *http.Response, err error) {
233
+ return b.GetResponseWithHeaders(path, make(http.Header))
234
+}
235
+
236
+// GetReaderWithHeaders retrieves an object from an S3 bucket
237
+// Accepts custom headers to be sent as the second parameter
238
+// returning the body of the HTTP response.
239
+// It is the caller's responsibility to call Close on rc when
240
+// finished reading
241
+func (b *Bucket) GetResponseWithHeaders(path string, headers map[string][]string) (resp *http.Response, err error) {
242
+ req := &request{
243
+ bucket: b.Name,
244
+ path: path,
245
+ headers: headers,
246
+ }
247
+ err = b.S3.prepare(req)
248
+ if err != nil {
249
+ return nil, err
250
+ }
251
+ for attempt := attempts.Start(); attempt.Next(); {
252
+ resp, err := b.S3.run(req, nil)
253
+ if shouldRetry(err) && attempt.HasNext() {
254
+ continue
255
+ }
256
+ if err != nil {
257
+ return nil, err
258
+ }
259
+ return resp, nil
260
+ }
261
+ panic("unreachable")
262
+}
263
+
264
+// Exists checks whether or not an object exists on an S3 bucket using a HEAD request.
265
+func (b *Bucket) Exists(path string) (exists bool, err error) {
266
+ req := &request{
267
+ method: "HEAD",
268
+ bucket: b.Name,
269
+ path: path,
270
+ }
271
+ err = b.S3.prepare(req)
272
+ if err != nil {
273
+ return
274
+ }
275
+ for attempt := attempts.Start(); attempt.Next(); {
276
+ resp, err := b.S3.run(req, nil)
277
+
278
+ if shouldRetry(err) && attempt.HasNext() {
279
+ continue
280
+ }
281
+
282
+ if err != nil {
283
+ // We can treat a 403 or 404 as non existance
284
+ if e, ok := err.(*Error); ok && (e.StatusCode == 403 || e.StatusCode == 404) {
285
+ return false, nil
286
+ }
287
+ return false, err
288
+ }
289
+
290
+ if resp.StatusCode/100 == 2 {
291
+ exists = true
292
+ }
293
+ if resp.Body != nil {
294
+ resp.Body.Close()
295
+ }
296
+ return exists, err
297
+ }
298
+ return false, fmt.Errorf("S3 Currently Unreachable")
299
+}
300
+
301
+// Head HEADs an object in the S3 bucket, returns the response with
302
+// no body see http://bit.ly/17K1ylI
303
+func (b *Bucket) Head(path string, headers map[string][]string) (*http.Response, error) {
304
+ req := &request{
305
+ method: "HEAD",
306
+ bucket: b.Name,
307
+ path: path,
308
+ headers: headers,
309
+ }
310
+ err := b.S3.prepare(req)
311
+ if err != nil {
312
+ return nil, err
313
+ }
314
+
315
+ for attempt := attempts.Start(); attempt.Next(); {
316
+ resp, err := b.S3.run(req, nil)
317
+ if shouldRetry(err) && attempt.HasNext() {
318
+ continue
319
+ }
320
+ if err != nil {
321
+ return nil, err
322
+ }
323
+ return resp, err
324
+ }
325
+ return nil, fmt.Errorf("S3 Currently Unreachable")
326
+}
327
+
328
+// Put inserts an object into the S3 bucket.
329
+//
330
+// See http://goo.gl/FEBPD for details.
331
+func (b *Bucket) Put(path string, data []byte, contType string, perm ACL, options Options) error {
332
+ body := bytes.NewBuffer(data)
333
+ return b.PutReader(path, body, int64(len(data)), contType, perm, options)
334
+}
335
+
336
+// PutCopy puts a copy of an object given by the key path into bucket b using b.Path as the target key
337
+func (b *Bucket) PutCopy(path string, perm ACL, options CopyOptions, source string) (*CopyObjectResult, error) {
338
+ headers := map[string][]string{
339
+ "x-amz-acl": {string(perm)},
340
+ "x-amz-copy-source": {url.QueryEscape(source)},
341
+ }
342
+ options.addHeaders(headers)
343
+ req := &request{
344
+ method: "PUT",
345
+ bucket: b.Name,
346
+ path: path,
347
+ headers: headers,
348
+ }
349
+ resp := &CopyObjectResult{}
350
+ err := b.S3.query(req, resp)
351
+ if err != nil {
352
+ return resp, err
353
+ }
354
+ return resp, nil
355
+}
356
+
357
+// PutReader inserts an object into the S3 bucket by consuming data
358
+// from r until EOF.
359
+func (b *Bucket) PutReader(path string, r io.Reader, length int64, contType string, perm ACL, options Options) error {
360
+ headers := map[string][]string{
361
+ "Content-Length": {strconv.FormatInt(length, 10)},
362
+ "Content-Type": {contType},
363
+ "x-amz-acl": {string(perm)},
364
+ }
365
+ options.addHeaders(headers)
366
+ req := &request{
367
+ method: "PUT",
368
+ bucket: b.Name,
369
+ path: path,
370
+ headers: headers,
371
+ payload: r,
372
+ }
373
+ return b.S3.query(req, nil)
374
+}
375
+
376
+// addHeaders adds o's specified fields to headers
377
+func (o Options) addHeaders(headers map[string][]string) {
378
+ if o.SSE {
379
+ headers["x-amz-server-side-encryption"] = []string{"AES256"}
380
+ } else if len(o.SSECustomerAlgorithm) != 0 && len(o.SSECustomerKey) != 0 && len(o.SSECustomerKeyMD5) != 0 {
381
+ // Amazon-managed keys and customer-managed keys are mutually exclusive
382
+ headers["x-amz-server-side-encryption-customer-algorithm"] = []string{o.SSECustomerAlgorithm}
383
+ headers["x-amz-server-side-encryption-customer-key"] = []string{o.SSECustomerKey}
384
+ headers["x-amz-server-side-encryption-customer-key-MD5"] = []string{o.SSECustomerKeyMD5}
385
+ }
386
+ if len(o.Range) != 0 {
387
+ headers["Range"] = []string{o.Range}
388
+ }
389
+ if len(o.ContentEncoding) != 0 {
390
+ headers["Content-Encoding"] = []string{o.ContentEncoding}
391
+ }
392
+ if len(o.CacheControl) != 0 {
393
+ headers["Cache-Control"] = []string{o.CacheControl}
394
+ }
395
+ if len(o.ContentMD5) != 0 {
396
+ headers["Content-MD5"] = []string{o.ContentMD5}
397
+ }
398
+ if len(o.RedirectLocation) != 0 {
399
+ headers["x-amz-website-redirect-location"] = []string{o.RedirectLocation}
400
+ }
401
+ if len(o.ContentDisposition) != 0 {
402
+ headers["Content-Disposition"] = []string{o.ContentDisposition}
403
+ }
404
+ for k, v := range o.Meta {
405
+ headers["x-amz-meta-"+k] = v
406
+ }
407
+}
408
+
409
+// addHeaders adds o's specified fields to headers
410
+func (o CopyOptions) addHeaders(headers map[string][]string) {
411
+ o.Options.addHeaders(headers)
412
+ if len(o.MetadataDirective) != 0 {
413
+ headers["x-amz-metadata-directive"] = []string{o.MetadataDirective}
414
+ }
415
+ if len(o.CopySourceOptions) != 0 {
416
+ headers["x-amz-copy-source-range"] = []string{o.CopySourceOptions}
417
+ }
418
+ if len(o.ContentType) != 0 {
419
+ headers["Content-Type"] = []string{o.ContentType}
420
+ }
421
+}
422
+
423
+func makeXmlBuffer(doc []byte) *bytes.Buffer {
424
+ buf := new(bytes.Buffer)
425
+ buf.WriteString(xml.Header)
426
+ buf.Write(doc)
427
+ return buf
428
+}
429
+
430
+type IndexDocument struct {
431
+ Suffix string `xml:"Suffix"`
432
+}
433
+
434
+type ErrorDocument struct {
435
+ Key string `xml:"Key"`
436
+}
437
+
438
+type RoutingRule struct {
439
+ ConditionKeyPrefixEquals string `xml:"Condition>KeyPrefixEquals"`
440
+ RedirectReplaceKeyPrefixWith string `xml:"Redirect>ReplaceKeyPrefixWith,omitempty"`
441
+ RedirectReplaceKeyWith string `xml:"Redirect>ReplaceKeyWith,omitempty"`
442
+}
443
+
444
+type RedirectAllRequestsTo struct {
445
+ HostName string `xml:"HostName"`
446
+ Protocol string `xml:"Protocol,omitempty"`
447
+}
448
+
449
+type WebsiteConfiguration struct {
450
+ XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ WebsiteConfiguration"`
451
+ IndexDocument *IndexDocument `xml:"IndexDocument,omitempty"`
452
+ ErrorDocument *ErrorDocument `xml:"ErrorDocument,omitempty"`
453
+ RoutingRules *[]RoutingRule `xml:"RoutingRules>RoutingRule,omitempty"`
454
+ RedirectAllRequestsTo *RedirectAllRequestsTo `xml:"RedirectAllRequestsTo,omitempty"`
455
+}
456
+
457
+// PutBucketWebsite configures a bucket as a website.
458
+//
459
+// See http://goo.gl/TpRlUy for details.
460
+func (b *Bucket) PutBucketWebsite(configuration WebsiteConfiguration) error {
461
+ doc, err := xml.Marshal(configuration)
462
+ if err != nil {
463
+ return err
464
+ }
465
+
466
+ buf := makeXmlBuffer(doc)
467
+
468
+ return b.PutBucketSubresource("website", buf, int64(buf.Len()))
469
+}
470
+
471
+func (b *Bucket) PutBucketSubresource(subresource string, r io.Reader, length int64) error {
472
+ headers := map[string][]string{
473
+ "Content-Length": {strconv.FormatInt(length, 10)},
474
+ }
475
+ req := &request{
476
+ path: "/",
477
+ method: "PUT",
478
+ bucket: b.Name,
479
+ headers: headers,
480
+ payload: r,
481
+ params: url.Values{subresource: {""}},
482
+ }
483
+
484
+ return b.S3.query(req, nil)
485
+}
486
+
487
+// Del removes an object from the S3 bucket.
488
+//
489
+// See http://goo.gl/APeTt for details.
490
+func (b *Bucket) Del(path string) error {
491
+ req := &request{
492
+ method: "DELETE",
493
+ bucket: b.Name,
494
+ path: path,
495
+ }
496
+ return b.S3.query(req, nil)
497
+}
498
+
499
+type Delete struct {
500
+ Quiet bool `xml:"Quiet,omitempty"`
501
+ Objects []Object `xml:"Object"`
502
+}
503
+
504
+type Object struct {
505
+ Key string `xml:"Key"`
506
+ VersionId string `xml:"VersionId,omitempty"`
507
+}
508
+
509
+// DelMulti removes up to 1000 objects from the S3 bucket.
510
+//
511
+// See http://goo.gl/jx6cWK for details.
512
+func (b *Bucket) DelMulti(objects Delete) error {
513
+ doc, err := xml.Marshal(objects)
514
+ if err != nil {
515
+ return err
516
+ }
517
+
518
+ buf := makeXmlBuffer(doc)
519
+ digest := md5.New()
520
+ size, err := digest.Write(buf.Bytes())
521
+ if err != nil {
522
+ return err
523
+ }
524
+
525
+ headers := map[string][]string{
526
+ "Content-Length": {strconv.FormatInt(int64(size), 10)},
527
+ "Content-MD5": {base64.StdEncoding.EncodeToString(digest.Sum(nil))},
528
+ "Content-Type": {"text/xml"},
529
+ }
530
+ req := &request{
531
+ path: "/",
532
+ method: "POST",
533
+ params: url.Values{"delete": {""}},
534
+ bucket: b.Name,
535
+ headers: headers,
536
+ payload: buf,
537
+ }
538
+
539
+ return b.S3.query(req, nil)
540
+}
541
+
542
+// The ListResp type holds the results of a List bucket operation.
543
+type ListResp struct {
544
+ Name string
545
+ Prefix string
546
+ Delimiter string
547
+ Marker string
548
+ MaxKeys int
549
+ // IsTruncated is true if the results have been truncated because
550
+ // there are more keys and prefixes than can fit in MaxKeys.
551
+ // N.B. this is the opposite sense to that documented (incorrectly) in
552
+ // http://goo.gl/YjQTc
553
+ IsTruncated bool
554
+ Contents []Key
555
+ CommonPrefixes []string `xml:">Prefix"`
556
+ // if IsTruncated is true, pass NextMarker as marker argument to List()
557
+ // to get the next set of keys
558
+ NextMarker string
559
+}
560
+
561
+// The Key type represents an item stored in an S3 bucket.
562
+type Key struct {
563
+ Key string
564
+ LastModified string
565
+ Size int64
566
+ // ETag gives the hex-encoded MD5 sum of the contents,
567
+ // surrounded with double-quotes.
568
+ ETag string
569
+ StorageClass string
570
+ Owner Owner
571
+}
572
+
573
+// List returns information about objects in an S3 bucket.
574
+//
575
+// The prefix parameter limits the response to keys that begin with the
576
+// specified prefix.
577
+//
578
+// The delim parameter causes the response to group all of the keys that
579
+// share a common prefix up to the next delimiter in a single entry within
580
+// the CommonPrefixes field. You can use delimiters to separate a bucket
581
+// into different groupings of keys, similar to how folders would work.
582
+//
583
+// The marker parameter specifies the key to start with when listing objects
584
+// in a bucket. Amazon S3 lists objects in alphabetical order and
585
+// will return keys alphabetically greater than the marker.
586
+//
587
+// The max parameter specifies how many keys + common prefixes to return in
588
+// the response. The default is 1000.
589
+//
590
+// For example, given these keys in a bucket:
591
+//
592
+// index.html
593
+// index2.html
594
+// photos/2006/January/sample.jpg
595
+// photos/2006/February/sample2.jpg
596
+// photos/2006/February/sample3.jpg
597
+// photos/2006/February/sample4.jpg
598
+//
599
+// Listing this bucket with delimiter set to "/" would yield the
600
+// following result:
601
+//
602
+// &ListResp{
603
+// Name: "sample-bucket",
604
+// MaxKeys: 1000,
605
+// Delimiter: "/",
606
+// Contents: []Key{
607
+// {Key: "index.html", "index2.html"},
608
+// },
609
+// CommonPrefixes: []string{
610
+// "photos/",
611
+// },
612
+// }
613
+//
614
+// Listing the same bucket with delimiter set to "/" and prefix set to
615
+// "photos/2006/" would yield the following result:
616
+//
617
+// &ListResp{
618
+// Name: "sample-bucket",
619
+// MaxKeys: 1000,
620
+// Delimiter: "/",
621
+// Prefix: "photos/2006/",
622
+// CommonPrefixes: []string{
623
+// "photos/2006/February/",
624
+// "photos/2006/January/",
625
+// },
626
+// }
627
+//
628
+// See http://goo.gl/YjQTc for details.
629
+func (b *Bucket) List(prefix, delim, marker string, max int) (result *ListResp, err error) {
630
+ params := map[string][]string{
631
+ "prefix": {prefix},
632
+ "delimiter": {delim},
633
+ "marker": {marker},
634
+ }
635
+ if max != 0 {
636
+ params["max-keys"] = []string{strconv.FormatInt(int64(max), 10)}
637
+ }
638
+ req := &request{
639
+ bucket: b.Name,
640
+ params: params,
641
+ }
642
+ result = &ListResp{}
643
+ for attempt := attempts.Start(); attempt.Next(); {
644
+ err = b.S3.query(req, result)
645
+ if !shouldRetry(err) {
646
+ break
647
+ }
648
+ }
649
+ if err != nil {
650
+ return nil, err
651
+ }
652
+ // if NextMarker is not returned, it should be set to the name of last key,
653
+ // so let's do it so that each caller doesn't have to
654
+ if result.IsTruncated && result.NextMarker == "" {
655
+ n := len(result.Contents)
656
+ if n > 0 {
657
+ result.NextMarker = result.Contents[n-1].Key
658
+ }
659
+ }
660
+ return result, nil
661
+}
662
+
663
+// The VersionsResp type holds the results of a list bucket Versions operation.
664
+type VersionsResp struct {
665
+ Name string
666
+ Prefix string
667
+ KeyMarker string
668
+ VersionIdMarker string
669
+ MaxKeys int
670
+ Delimiter string
671
+ IsTruncated bool
672
+ Versions []Version `xml:"Version"`
673
+ CommonPrefixes []string `xml:">Prefix"`
674
+}
675
+
676
+// The Version type represents an object version stored in an S3 bucket.
677
+type Version struct {
678
+ Key string
679
+ VersionId string
680
+ IsLatest bool
681
+ LastModified string
682
+ // ETag gives the hex-encoded MD5 sum of the contents,
683
+ // surrounded with double-quotes.
684
+ ETag string
685
+ Size int64
686
+ Owner Owner
687
+ StorageClass string
688
+}
689
+
690
+func (b *Bucket) Versions(prefix, delim, keyMarker string, versionIdMarker string, max int) (result *VersionsResp, err error) {
691
+ params := map[string][]string{
692
+ "versions": {""},
693
+ "prefix": {prefix},
694
+ "delimiter": {delim},
695
+ }
696
+
697
+ if len(versionIdMarker) != 0 {
698
+ params["version-id-marker"] = []string{versionIdMarker}
699
+ }
700
+ if len(keyMarker) != 0 {
701
+ params["key-marker"] = []string{keyMarker}
702
+ }
703
+
704
+ if max != 0 {
705
+ params["max-keys"] = []string{strconv.FormatInt(int64(max), 10)}
706
+ }
707
+ req := &request{
708
+ bucket: b.Name,
709
+ params: params,
710
+ }
711
+ result = &VersionsResp{}
712
+ for attempt := attempts.Start(); attempt.Next(); {
713
+ err = b.S3.query(req, result)
714
+ if !shouldRetry(err) {
715
+ break
716
+ }
717
+ }
718
+ if err != nil {
719
+ return nil, err
720
+ }
721
+ return result, nil
722
+}
723
+
724
+type GetLocationResp struct {
725
+ Location string `xml:",innerxml"`
726
+}
727
+
728
+func (b *Bucket) Location() (string, error) {
729
+ r, err := b.Get("/?location")
730
+ if err != nil {
731
+ return "", err
732
+ }
733
+
734
+ // Parse the XML response.
735
+ var resp GetLocationResp
736
+ if err = xml.Unmarshal(r, &resp); err != nil {
737
+ return "", err
738
+ }
739
+
740
+ if resp.Location == "" {
741
+ return "us-east-1", nil
742
+ } else {
743
+ return resp.Location, nil
744
+ }
745
+}
746
+
747
+// URL returns a non-signed URL that allows retriving the
748
+// object at path. It only works if the object is publicly
749
+// readable (see SignedURL).
750
+func (b *Bucket) URL(path string) string {
751
+ req := &request{
752
+ bucket: b.Name,
753
+ path: path,
754
+ }
755
+ err := b.S3.prepare(req)
756
+ if err != nil {
757
+ panic(err)
758
+ }
759
+ u, err := req.url()
760
+ if err != nil {
761
+ panic(err)
762
+ }
763
+ u.RawQuery = ""
764
+ return u.String()
765
+}
766
+
767
+// SignedURL returns a signed URL that allows anyone holding the URL
768
+// to retrieve the object at path. The signature is valid until expires.
769
+func (b *Bucket) SignedURL(path string, expires time.Time) string {
770
+ return b.SignedURLWithArgs(path, expires, nil, nil)
771
+}
772
+
773
+// SignedURLWithArgs returns a signed URL that allows anyone holding the URL
774
+// to retrieve the object at path. The signature is valid until expires.
775
+func (b *Bucket) SignedURLWithArgs(path string, expires time.Time, params url.Values, headers http.Header) string {
776
+ return b.SignedURLWithMethod("GET", path, expires, params, headers)
777
+}
778
+
779
+// SignedURLWithMethod returns a signed URL that allows anyone holding the URL
780
+// to either retrieve the object at path or make a HEAD request against it. The signature is valid until expires.
781
+func (b *Bucket) SignedURLWithMethod(method, path string, expires time.Time, params url.Values, headers http.Header) string {
782
+ var uv = url.Values{}
783
+
784
+ if params != nil {
785
+ uv = params
786
+ }
787
+
788
+ if b.S3.Signature == aws.V2Signature {
789
+ uv.Set("Expires", strconv.FormatInt(expires.Unix(), 10))
790
+ } else {
791
+ uv.Set("X-Amz-Expires", strconv.FormatInt(expires.Unix()-time.Now().Unix(), 10))
792
+ }
793
+
794
+ req := &request{
795
+ method: method,
796
+ bucket: b.Name,
797
+ path: path,
798
+ params: uv,
799
+ headers: headers,
800
+ }
801
+ err := b.S3.prepare(req)
802
+ if err != nil {
803
+ panic(err)
804
+ }
805
+ u, err := req.url()
806
+ if err != nil {
807
+ panic(err)
808
+ }
809
+ if b.S3.Auth.Token() != "" {
810
+ return u.String() + "&x-amz-security-token=" + url.QueryEscape(req.headers["X-Amz-Security-Token"][0])
811
+ } else {
812
+ return u.String()
813
+ }
814
+}
815
+
816
+// UploadSignedURL returns a signed URL that allows anyone holding the URL
817
+// to upload the object at path. The signature is valid until expires.
818
+// contenttype is a string like image/png
819
+// path is the resource name in s3 terminalogy like images/ali.png [obviously exclusing the bucket name itself]
820
+func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time.Time) string {
821
+ expire_date := expires.Unix()
822
+ if method != "POST" {
823
+ method = "PUT"
824
+ }
825
+
826
+ a := b.S3.Auth
827
+ tokenData := ""
828
+
829
+ if a.Token() != "" {
830
+ tokenData = "x-amz-security-token:" + a.Token() + "\n"
831
+ }
832
+
833
+ stringToSign := method + "\n\n" + content_type + "\n" + strconv.FormatInt(expire_date, 10) + "\n" + tokenData + "/" + b.Name + "/" + path
834
+ secretKey := a.SecretKey
835
+ accessId := a.AccessKey
836
+ mac := hmac.New(sha1.New, []byte(secretKey))
837
+ mac.Write([]byte(stringToSign))
838
+ macsum := mac.Sum(nil)
839
+ signature := base64.StdEncoding.EncodeToString([]byte(macsum))
840
+ signature = strings.TrimSpace(signature)
841
+
842
+ signedurl, err := url.Parse("https://" + b.Name + ".s3.amazonaws.com/")
843
+ if err != nil {
844
+ log.Println("ERROR sining url for S3 upload", err)
845
+ return ""
846
+ }
847
+ signedurl.Path += path
848
+ params := url.Values{}
849
+ params.Add("AWSAccessKeyId", accessId)
850
+ params.Add("Expires", strconv.FormatInt(expire_date, 10))
851
+ params.Add("Signature", signature)
852
+ if a.Token() != "" {
853
+ params.Add("x-amz-security-token", a.Token())
854
+ }
855
+
856
+ signedurl.RawQuery = params.Encode()
857
+ return signedurl.String()
858
+}
859
+
860
+// PostFormArgs returns the action and input fields needed to allow anonymous
861
+// uploads to a bucket within the expiration limit
862
+// Additional conditions can be specified with conds
863
+func (b *Bucket) PostFormArgsEx(path string, expires time.Time, redirect string, conds []string) (action string, fields map[string]string) {
864
+ conditions := make([]string, 0)
865
+ fields = map[string]string{
866
+ "AWSAccessKeyId": b.Auth.AccessKey,
867
+ "key": path,
868
+ }
869
+
870
+ if conds != nil {
871
+ conditions = append(conditions, conds...)
872
+ }
873
+
874
+ conditions = append(conditions, fmt.Sprintf("{\"key\": \"%s\"}", path))
875
+ conditions = append(conditions, fmt.Sprintf("{\"bucket\": \"%s\"}", b.Name))
876
+ if redirect != "" {
877
+ conditions = append(conditions, fmt.Sprintf("{\"success_action_redirect\": \"%s\"}", redirect))
878
+ fields["success_action_redirect"] = redirect
879
+ }
880
+
881
+ vExpiration := expires.Format("2006-01-02T15:04:05Z")
882
+ vConditions := strings.Join(conditions, ",")
883
+ policy := fmt.Sprintf("{\"expiration\": \"%s\", \"conditions\": [%s]}", vExpiration, vConditions)
884
+ policy64 := base64.StdEncoding.EncodeToString([]byte(policy))
885
+ fields["policy"] = policy64
886
+
887
+ signer := hmac.New(sha1.New, []byte(b.Auth.SecretKey))
888
+ signer.Write([]byte(policy64))
889
+ fields["signature"] = base64.StdEncoding.EncodeToString(signer.Sum(nil))
890
+
891
+ action = fmt.Sprintf("%s/%s/", b.S3.Region.S3Endpoint, b.Name)
892
+ return
893
+}
894
+
895
+// PostFormArgs returns the action and input fields needed to allow anonymous
896
+// uploads to a bucket within the expiration limit
897
+func (b *Bucket) PostFormArgs(path string, expires time.Time, redirect string) (action string, fields map[string]string) {
898
+ return b.PostFormArgsEx(path, expires, redirect, nil)
899
+}
900
+
901
+type request struct {
902
+ method string
903
+ bucket string
904
+ path string
905
+ params url.Values
906
+ headers http.Header
907
+ baseurl string
908
+ payload io.Reader
909
+ prepared bool
910
+}
911
+
912
+func (req *request) url() (*url.URL, error) {
913
+ u, err := url.Parse(req.baseurl)
914
+ if err != nil {
915
+ return nil, fmt.Errorf("bad S3 endpoint URL %q: %v", req.baseurl, err)
916
+ }
917
+ u.RawQuery = req.params.Encode()
918
+ u.Path = req.path
919
+ return u, nil
920
+}
921
+
922
+// query prepares and runs the req request.
923
+// If resp is not nil, the XML data contained in the response
924
+// body will be unmarshalled on it.
925
+func (s3 *S3) query(req *request, resp interface{}) error {
926
+ err := s3.prepare(req)
927
+ if err != nil {
928
+ return err
929
+ }
930
+ r, err := s3.run(req, resp)
931
+ if r != nil && r.Body != nil {
932
+ r.Body.Close()
933
+ }
934
+ return err
935
+}
936
+
937
+// queryV4Signprepares and runs the req request, signed with aws v4 signatures.
938
+// If resp is not nil, the XML data contained in the response
939
+// body will be unmarshalled on it.
940
+func (s3 *S3) queryV4Sign(req *request, resp interface{}) error {
941
+ if req.headers == nil {
942
+ req.headers = map[string][]string{}
943
+ }
944
+
945
+ err := s3.setBaseURL(req)
946
+ if err != nil {
947
+ return err
948
+ }
949
+
950
+ hreq, err := s3.setupHttpRequest(req)
951
+ if err != nil {
952
+ return err
953
+ }
954
+
955
+ // req.Host must be set for V4 signature calculation
956
+ hreq.Host = hreq.URL.Host
957
+
958
+ signer := aws.NewV4Signer(s3.Auth, "s3", s3.Region)
959
+ signer.IncludeXAmzContentSha256 = true
960
+ signer.Sign(hreq)
961
+
962
+ _, err = s3.doHttpRequest(hreq, resp)
963
+ return err
964
+}
965
+
966
+// Sets baseurl on req from bucket name and the region endpoint
967
+func (s3 *S3) setBaseURL(req *request) error {
968
+ if req.bucket == "" {
969
+ req.baseurl = s3.Region.S3Endpoint
970
+ } else {
971
+ req.baseurl = s3.Region.S3BucketEndpoint
972
+ if req.baseurl == "" {
973
+ // Use the path method to address the bucket.
974
+ req.baseurl = s3.Region.S3Endpoint
975
+ req.path = "/" + req.bucket + req.path
976
+ } else {
977
+ // Just in case, prevent injection.
978
+ if strings.IndexAny(req.bucket, "/:@") >= 0 {
979
+ return fmt.Errorf("bad S3 bucket: %q", req.bucket)
980
+ }
981
+ req.baseurl = strings.Replace(req.baseurl, "${bucket}", req.bucket, -1)
982
+ }
983
+ }
984
+
985
+ return nil
986
+}
987
+
988
+// partiallyEscapedPath partially escapes the S3 path allowing for all S3 REST API calls.
989
+//
990
+// Some commands including:
991
+// GET Bucket acl http://goo.gl/aoXflF
992
+// GET Bucket cors http://goo.gl/UlmBdx
993
+// GET Bucket lifecycle http://goo.gl/8Fme7M
994
+// GET Bucket policy http://goo.gl/ClXIo3
995
+// GET Bucket location http://goo.gl/5lh8RD
996
+// GET Bucket Logging http://goo.gl/sZ5ckF
997
+// GET Bucket notification http://goo.gl/qSSZKD
998
+// GET Bucket tagging http://goo.gl/QRvxnM
999
+// require the first character after the bucket name in the path to be a literal '?' and
1000
+// not the escaped hex representation '%3F'.
1001
+func partiallyEscapedPath(path string) string {
1002
+ pathEscapedAndSplit := strings.Split((&url.URL{Path: path}).String(), "/")
1003
+ if len(pathEscapedAndSplit) >= 3 {
1004
+ if len(pathEscapedAndSplit[2]) >= 3 {
1005
+ // Check for the one "?" that should not be escaped.
1006
+ if pathEscapedAndSplit[2][0:3] == "%3F" {
1007
+ pathEscapedAndSplit[2] = "?" + pathEscapedAndSplit[2][3:]
1008
+ }
1009
+ }
1010
+ }
1011
+ return strings.Replace(strings.Join(pathEscapedAndSplit, "/"), "+", "%2B", -1)
1012
+}
1013
+
1014
+// prepare sets up req to be delivered to S3.
1015
+func (s3 *S3) prepare(req *request) error {
1016
+ // Copy so they can be mutated without affecting on retries.
1017
+ params := make(url.Values)
1018
+ headers := make(http.Header)
1019
+ for k, v := range req.params {
1020
+ params[k] = v
1021
+ }
1022
+ for k, v := range req.headers {
1023
+ headers[k] = v
1024
+ }
1025
+ req.params = params
1026
+ req.headers = headers
1027
+
1028
+ if !req.prepared {
1029
+ req.prepared = true
1030
+ if req.method == "" {
1031
+ req.method = "GET"
1032
+ }
1033
+
1034
+ if !strings.HasPrefix(req.path, "/") {
1035
+ req.path = "/" + req.path
1036
+ }
1037
+
1038
+ err := s3.setBaseURL(req)
1039
+ if err != nil {
1040
+ return err
1041
+ }
1042
+ }
1043
+
1044
+ if s3.Auth.Token() != "" {
1045
+ req.headers["X-Amz-Security-Token"] = []string{s3.Auth.Token()}
1046
+ }
1047
+
1048
+ if s3.Signature == aws.V2Signature {
1049
+ // Always sign again as it's not clear how far the
1050
+ // server has handled a previous attempt.
1051
+ u, err := url.Parse(req.baseurl)
1052
+ if err != nil {
1053
+ return err
1054
+ }
1055
+
1056
+ signpathPatiallyEscaped := partiallyEscapedPath(req.path)
1057
+ req.headers["Host"] = []string{u.Host}
1058
+ req.headers["Date"] = []string{time.Now().In(time.UTC).Format(time.RFC1123)}
1059
+
1060
+ sign(s3.Auth, req.method, signpathPatiallyEscaped, req.params, req.headers)
1061
+ } else {
1062
+ hreq, err := s3.setupHttpRequest(req)
1063
+ if err != nil {
1064
+ return err
1065
+ }
1066
+
1067
+ hreq.Host = hreq.URL.Host
1068
+ signer := aws.NewV4Signer(s3.Auth, "s3", s3.Region)
1069
+ signer.IncludeXAmzContentSha256 = true
1070
+ signer.Sign(hreq)
1071
+
1072
+ req.payload = hreq.Body
1073
+ if _, ok := headers["Content-Length"]; ok {
1074
+ req.headers["Content-Length"] = headers["Content-Length"]
1075
+ }
1076
+ }
1077
+ return nil
1078
+}
1079
+
1080
+// Prepares an *http.Request for doHttpRequest
1081
+func (s3 *S3) setupHttpRequest(req *request) (*http.Request, error) {
1082
+ // Copy so that signing the http request will not mutate it
1083
+ headers := make(http.Header)
1084
+ for k, v := range req.headers {
1085
+ headers[k] = v
1086
+ }
1087
+ req.headers = headers
1088
+
1089
+ u, err := req.url()
1090
+ if err != nil {
1091
+ return nil, err
1092
+ }
1093
+ u.Opaque = fmt.Sprintf("//%s%s", u.Host, partiallyEscapedPath(u.Path))
1094
+
1095
+ hreq := http.Request{
1096
+ URL: u,
1097
+ Method: req.method,
1098
+ ProtoMajor: 1,
1099
+ ProtoMinor: 1,
1100
+ Close: true,
1101
+ Header: req.headers,
1102
+ Form: req.params,
1103
+ }
1104
+
1105
+ if v, ok := req.headers["Content-Length"]; ok {
1106
+ hreq.ContentLength, _ = strconv.ParseInt(v[0], 10, 64)
1107
+ delete(req.headers, "Content-Length")
1108
+ }
1109
+ if req.payload != nil {
1110
+ hreq.Body = ioutil.NopCloser(req.payload)
1111
+ }
1112
+
1113
+ return &hreq, nil
1114
+}
1115
+
1116
+// doHttpRequest sends hreq and returns the http response from the server.
1117
+// If resp is not nil, the XML data contained in the response
1118
+// body will be unmarshalled on it.
1119
+func (s3 *S3) doHttpRequest(hreq *http.Request, resp interface{}) (*http.Response, error) {
1120
+ c := http.Client{
1121
+ Transport: &http.Transport{
1122
+ Dial: func(netw, addr string) (c net.Conn, err error) {
1123
+ deadline := time.Now().Add(s3.ReadTimeout)
1124
+ if s3.ConnectTimeout > 0 {
1125
+ c, err = net.DialTimeout(netw, addr, s3.ConnectTimeout)
1126
+ } else {
1127
+ c, err = net.Dial(netw, addr)
1128
+ }
1129
+ if err != nil {
1130
+ return
1131
+ }
1132
+ if s3.ReadTimeout > 0 {
1133
+ err = c.SetDeadline(deadline)
1134
+ }
1135
+ return
1136
+ },
1137
+ Proxy: http.ProxyFromEnvironment,
1138
+ },
1139
+ }
1140
+
1141
+ hresp, err := c.Do(hreq)
1142
+ if err != nil {
1143
+ return nil, err
1144
+ }
1145
+ if debug {
1146
+ dump, _ := httputil.DumpResponse(hresp, true)
1147
+ log.Printf("} -> %s\n", dump)
1148
+ }
1149
+ if hresp.StatusCode != 200 && hresp.StatusCode != 204 && hresp.StatusCode != 206 {
1150
+ return nil, buildError(hresp)
1151
+ }
1152
+ if resp != nil {
1153
+ err = xml.NewDecoder(hresp.Body).Decode(resp)
1154
+ hresp.Body.Close()
1155
+
1156
+ if debug {
1157
+ log.Printf("goamz.s3> decoded xml into %#v", resp)
1158
+ }
1159
+
1160
+ }
1161
+ return hresp, err
1162
+}
1163
+
1164
+// run sends req and returns the http response from the server.
1165
+// If resp is not nil, the XML data contained in the response
1166
+// body will be unmarshalled on it.
1167
+func (s3 *S3) run(req *request, resp interface{}) (*http.Response, error) {
1168
+ if debug {
1169
+ log.Printf("Running S3 request: %#v", req)
1170
+ }
1171
+
1172
+ hreq, err := s3.setupHttpRequest(req)
1173
+ if err != nil {
1174
+ return nil, err
1175
+ }
1176
+
1177
+ return s3.doHttpRequest(hreq, resp)
1178
+}
1179
+
1180
+// Error represents an error in an operation with S3.
1181
+type Error struct {
1182
+ StatusCode int // HTTP status code (200, 403, ...)
1183
+ Code string // EC2 error code ("UnsupportedOperation", ...)
1184
+ Message string // The human-oriented error message
1185
+ BucketName string
1186
+ RequestId string
1187
+ HostId string
1188
+}
1189
+
1190
+func (e *Error) Error() string {
1191
+ return e.Message
1192
+}
1193
+
1194
+func buildError(r *http.Response) error {
1195
+ if debug {
1196
+ log.Printf("got error (status code %v)", r.StatusCode)
1197
+ data, err := ioutil.ReadAll(r.Body)
1198
+ if err != nil {
1199
+ log.Printf("\tread error: %v", err)
1200
+ } else {
1201
+ log.Printf("\tdata:\n%s\n\n", data)
1202
+ }
1203
+ r.Body = ioutil.NopCloser(bytes.NewBuffer(data))
1204
+ }
1205
+
1206
+ err := Error{}
1207
+ // TODO return error if Unmarshal fails?
1208
+ xml.NewDecoder(r.Body).Decode(&err)
1209
+ r.Body.Close()
1210
+ err.StatusCode = r.StatusCode
1211
+ if err.Message == "" {
1212
+ err.Message = r.Status
1213
+ }
1214
+ if debug {
1215
+ log.Printf("err: %#v\n", err)
1216
+ }
1217
+ return &err
1218
+}
1219
+
1220
+func shouldRetry(err error) bool {
1221
+ if err == nil {
1222
+ return false
1223
+ }
1224
+ switch err {
1225
+ case io.ErrUnexpectedEOF, io.EOF:
1226
+ return true
1227
+ }
1228
+ switch e := err.(type) {
1229
+ case *net.DNSError:
1230
+ return true
1231
+ case *net.OpError:
1232
+ switch e.Op {
1233
+ case "read", "write":
1234
+ return true
1235
+ }
1236
+ case *Error:
1237
+ switch e.Code {
1238
+ case "InternalError", "NoSuchUpload", "NoSuchBucket":
1239
+ return true
1240
+ }
1241
+ }
1242
+ return false
1243
+}
1244
+
1245
+func hasCode(err error, code string) bool {
1246
+ s3err, ok := err.(*Error)
1247
+ return ok && s3err.Code == code
1248
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3_test.go
new
+486
@@ -0,0 +1,486 @@
1
+package s3_test
2
+
3
+import (
4
+ "bytes"
5
+ "io/ioutil"
6
+ "net/http"
7
+ "testing"
8
+ "time"
9
+
10
+ "github.com/crowdmob/goamz/testutil"
11
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/aws"
12
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/s3"
13
+ "gopkg.in/check.v1"
14
+)
15
+
16
+func Test(t *testing.T) {
17
+ check.TestingT(t)
18
+}
19
+
20
+type S struct {
21
+ s3 *s3.S3
22
+}
23
+
24
+var _ = check.Suite(&S{})
25
+
26
+var testServer = testutil.NewHTTPServer()
27
+
28
+func (s *S) SetUpSuite(c *check.C) {
29
+ testServer.Start()
30
+ auth := aws.Auth{AccessKey: "abc", SecretKey: "123"}
31
+ s.s3 = s3.New(auth, aws.Region{Name: "faux-region-1", S3Endpoint: testServer.URL})
32
+}
33
+
34
+func (s *S) TearDownSuite(c *check.C) {
35
+ s3.SetAttemptStrategy(nil)
36
+}
37
+
38
+func (s *S) SetUpTest(c *check.C) {
39
+ attempts := aws.AttemptStrategy{
40
+ Total: 300 * time.Millisecond,
41
+ Delay: 100 * time.Millisecond,
42
+ }
43
+ s3.SetAttemptStrategy(&attempts)
44
+}
45
+
46
+func (s *S) TearDownTest(c *check.C) {
47
+ testServer.Flush()
48
+}
49
+
50
+func (s *S) DisableRetries() {
51
+ s3.SetAttemptStrategy(&aws.AttemptStrategy{})
52
+}
53
+
54
+// PutBucket docs: http://goo.gl/kBTCu
55
+
56
+func (s *S) TestPutBucket(c *check.C) {
57
+ testServer.Response(200, nil, "")
58
+
59
+ b := s.s3.Bucket("bucket")
60
+ err := b.PutBucket(s3.Private)
61
+ c.Assert(err, check.IsNil)
62
+
63
+ req := testServer.WaitRequest()
64
+ c.Assert(req.Method, check.Equals, "PUT")
65
+ c.Assert(req.URL.Path, check.Equals, "/bucket/")
66
+ c.Assert(req.Header["Date"], check.Not(check.Equals), "")
67
+}
68
+
69
+// PutBucketWebsite docs: http://goo.gl/TpRlUy
70
+
71
+func (s *S) TestPutBucketWebsite(c *check.C) {
72
+ testServer.Response(200, nil, "")
73
+
74
+ b := s.s3.Bucket("bucket")
75
+ config := s3.WebsiteConfiguration{
76
+ RedirectAllRequestsTo: &s3.RedirectAllRequestsTo{HostName: "example.com"},
77
+ }
78
+ err := b.PutBucketWebsite(config)
79
+ c.Assert(err, check.IsNil)
80
+
81
+ req := testServer.WaitRequest()
82
+ body, err := ioutil.ReadAll(req.Body)
83
+ req.Body.Close()
84
+ c.Assert(err, check.IsNil)
85
+ c.Assert(string(body), check.Equals, BucketWebsiteConfigurationDump)
86
+ c.Assert(req.Method, check.Equals, "PUT")
87
+ c.Assert(req.URL.Path, check.Equals, "/bucket/")
88
+ c.Assert(req.URL.RawQuery, check.Equals, "website=")
89
+ c.Assert(req.Header["Date"], check.Not(check.Equals), "")
90
+}
91
+
92
+// Head docs: http://bit.ly/17K1ylI
93
+
94
+func (s *S) TestHead(c *check.C) {
95
+ testServer.Response(200, nil, "content")
96
+
97
+ b := s.s3.Bucket("bucket")
98
+ resp, err := b.Head("name", nil)
99
+
100
+ req := testServer.WaitRequest()
101
+ c.Assert(req.Method, check.Equals, "HEAD")
102
+ c.Assert(req.URL.Path, check.Equals, "/bucket/name")
103
+ c.Assert(req.Header["Date"], check.Not(check.Equals), "")
104
+
105
+ c.Assert(err, check.IsNil)
106
+ c.Assert(resp.ContentLength, check.FitsTypeOf, int64(0))
107
+ c.Assert(resp, check.FitsTypeOf, &http.Response{})
108
+}
109
+
110
+// DeleteBucket docs: http://goo.gl/GoBrY
111
+
112
+func (s *S) TestDelBucket(c *check.C) {
113
+ testServer.Response(204, nil, "")
114
+
115
+ b := s.s3.Bucket("bucket")
116
+ err := b.DelBucket()
117
+ c.Assert(err, check.IsNil)
118
+
119
+ req := testServer.WaitRequest()
120
+ c.Assert(req.Method, check.Equals, "DELETE")
121
+ c.Assert(req.URL.Path, check.Equals, "/bucket/")
122
+ c.Assert(req.Header["Date"], check.Not(check.Equals), "")
123
+}
124
+
125
+// GetObject docs: http://goo.gl/isCO7
126
+
127
+func (s *S) TestGet(c *check.C) {
128
+ testServer.Response(200, nil, "content")
129
+
130
+ b := s.s3.Bucket("bucket")
131
+ data, err := b.Get("name")
132
+
133
+ req := testServer.WaitRequest()
134
+ c.Assert(req.Method, check.Equals, "GET")
135
+ c.Assert(req.URL.Path, check.Equals, "/bucket/name")
136
+ c.Assert(req.Header["Date"], check.Not(check.Equals), "")
137
+
138
+ c.Assert(err, check.IsNil)
139
+ c.Assert(string(data), check.Equals, "content")
140
+}
141
+
142
+func (s *S) TestGetWithPlus(c *check.C) {
143
+ testServer.Response(200, nil, "content")
144
+
145
+ b := s.s3.Bucket("bucket")
146
+ _, err := b.Get("has+plus")
147
+
148
+ req := testServer.WaitRequest()
149
+ c.Assert(err, check.IsNil)
150
+ c.Assert(req.RequestURI, check.Equals, "http://localhost:4444/bucket/has%2Bplus")
151
+}
152
+
153
+func (s *S) TestURL(c *check.C) {
154
+ testServer.Response(200, nil, "content")
155
+
156
+ b := s.s3.Bucket("bucket")
157
+ url := b.URL("name")
158
+ r, err := http.Get(url)
159
+ c.Assert(err, check.IsNil)
160
+ data, err := ioutil.ReadAll(r.Body)
161
+ r.Body.Close()
162
+ c.Assert(err, check.IsNil)
163
+ c.Assert(string(data), check.Equals, "content")
164
+
165
+ req := testServer.WaitRequest()
166
+ c.Assert(req.Method, check.Equals, "GET")
167
+ c.Assert(req.URL.Path, check.Equals, "/bucket/name")
168
+}
169
+
170
+func (s *S) TestGetReader(c *check.C) {
171
+ testServer.Response(200, nil, "content")
172
+
173
+ b := s.s3.Bucket("bucket")
174
+ rc, err := b.GetReader("name")
175
+ c.Assert(err, check.IsNil)
176
+ data, err := ioutil.ReadAll(rc)
177
+ rc.Close()
178
+ c.Assert(err, check.IsNil)
179
+ c.Assert(string(data), check.Equals, "content")
180
+
181
+ req := testServer.WaitRequest()
182
+ c.Assert(req.Method, check.Equals, "GET")
183
+ c.Assert(req.URL.Path, check.Equals, "/bucket/name")
184
+ c.Assert(req.Header["Date"], check.Not(check.Equals), "")
185
+}
186
+
187
+func (s *S) TestGetNotFound(c *check.C) {
188
+ for i := 0; i < 10; i++ {
189
+ testServer.Response(404, nil, GetObjectErrorDump)
190
+ }
191
+
192
+ b := s.s3.Bucket("non-existent-bucket")
193
+ data, err := b.Get("non-existent")
194
+
195
+ req := testServer.WaitRequest()
196
+ c.Assert(req.Method, check.Equals, "GET")
197
+ c.Assert(req.URL.Path, check.Equals, "/non-existent-bucket/non-existent")
198
+ c.Assert(req.Header["Date"], check.Not(check.Equals), "")
199
+
200
+ s3err, _ := err.(*s3.Error)
201
+ c.Assert(s3err, check.NotNil)
202
+ c.Assert(s3err.StatusCode, check.Equals, 404)
203
+ c.Assert(s3err.BucketName, check.Equals, "non-existent-bucket")
204
+ c.Assert(s3err.RequestId, check.Equals, "3F1B667FAD71C3D8")
205
+ c.Assert(s3err.HostId, check.Equals, "L4ee/zrm1irFXY5F45fKXIRdOf9ktsKY/8TDVawuMK2jWRb1RF84i1uBzkdNqS5D")
206
+ c.Assert(s3err.Code, check.Equals, "NoSuchBucket")
207
+ c.Assert(s3err.Message, check.Equals, "The specified bucket does not exist")
208
+ c.Assert(s3err.Error(), check.Equals, "The specified bucket does not exist")
209
+ c.Assert(data, check.IsNil)
210
+}
211
+
212
+// PutObject docs: http://goo.gl/FEBPD
213
+
214
+func (s *S) TestPutObject(c *check.C) {
215
+ testServer.Response(200, nil, "")
216
+ const DISPOSITION = "attachment; filename=\"0x1a2b3c.jpg\""
217
+
218
+ b := s.s3.Bucket("bucket")
219
+ err := b.Put("name", []byte("content"), "content-type", s3.Private, s3.Options{ContentDisposition: DISPOSITION})
220
+ c.Assert(err, check.IsNil)
221
+
222
+ req := testServer.WaitRequest()
223
+ c.Assert(req.Method, check.Equals, "PUT")
224
+ c.Assert(req.URL.Path, check.Equals, "/bucket/name")
225
+ c.Assert(req.Header["Date"], check.Not(check.DeepEquals), []string{""})
226
+ c.Assert(req.Header["Content-Type"], check.DeepEquals, []string{"content-type"})
227
+ c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"7"})
228
+ c.Assert(req.Header["Content-Disposition"], check.DeepEquals, []string{DISPOSITION})
229
+ //c.Assert(req.Header["Content-MD5"], gocheck.DeepEquals, "...")
230
+ c.Assert(req.Header["X-Amz-Acl"], check.DeepEquals, []string{"private"})
231
+}
232
+
233
+// PutCopy docs: http://goo.gl/mhEHtA
234
+func (s *S) TestPutCopy(c *check.C) {
235
+ testServer.Response(200, nil, PutCopyResultDump)
236
+
237
+ b := s.s3.Bucket("bucket")
238
+ res, err := b.PutCopy("name", s3.Private, s3.CopyOptions{},
239
+ // 0xFC is ü - 0xE9 is é
240
+ "source-bucket/\u00FCber-fil\u00E9.jpg")
241
+ c.Assert(err, check.IsNil)
242
+ c.Assert(res, check.DeepEquals, &s3.CopyObjectResult{
243
+ ETag: `"9b2cf535f27731c974343645a3985328"`,
244
+ LastModified: `2009-10-28T22:32:00`})
245
+
246
+ req := testServer.WaitRequest()
247
+ c.Assert(req.Method, check.Equals, "PUT")
248
+ c.Assert(req.URL.Path, check.Equals, "/bucket/name")
249
+ c.Assert(req.Header["Date"], check.Not(check.DeepEquals), []string{""})
250
+ c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"0"})
251
+ c.Assert(req.Header["X-Amz-Copy-Source"], check.DeepEquals, []string{`source-bucket%2F%C3%BCber-fil%C3%A9.jpg`})
252
+ c.Assert(req.Header["X-Amz-Acl"], check.DeepEquals, []string{"private"})
253
+}
254
+
255
+func (s *S) TestPutObjectReadTimeout(c *check.C) {
256
+ s.s3.ReadTimeout = 50 * time.Millisecond
257
+ defer func() {
258
+ s.s3.ReadTimeout = 0
259
+ }()
260
+
261
+ b := s.s3.Bucket("bucket")
262
+ err := b.Put("name", []byte("content"), "content-type", s3.Private, s3.Options{})
263
+
264
+ // Make sure that we get a timeout error.
265
+ c.Assert(err, check.NotNil)
266
+
267
+ // Set the response after the request times out so that the next request will work.
268
+ testServer.Response(200, nil, "")
269
+
270
+ // This time set the response within our timeout period so that we expect the call
271
+ // to return successfully.
272
+ go func() {
273
+ time.Sleep(25 * time.Millisecond)
274
+ testServer.Response(200, nil, "")
275
+ }()
276
+ err = b.Put("name", []byte("content"), "content-type", s3.Private, s3.Options{})
277
+ c.Assert(err, check.IsNil)
278
+}
279
+
280
+func (s *S) TestPutReader(c *check.C) {
281
+ testServer.Response(200, nil, "")
282
+
283
+ b := s.s3.Bucket("bucket")
284
+ buf := bytes.NewBufferString("content")
285
+ err := b.PutReader("name", buf, int64(buf.Len()), "content-type", s3.Private, s3.Options{})
286
+ c.Assert(err, check.IsNil)
287
+
288
+ req := testServer.WaitRequest()
289
+ c.Assert(req.Method, check.Equals, "PUT")
290
+ c.Assert(req.URL.Path, check.Equals, "/bucket/name")
291
+ c.Assert(req.Header["Date"], check.Not(check.DeepEquals), []string{""})
292
+ c.Assert(req.Header["Content-Type"], check.DeepEquals, []string{"content-type"})
293
+ c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"7"})
294
+ //c.Assert(req.Header["Content-MD5"], gocheck.Equals, "...")
295
+ c.Assert(req.Header["X-Amz-Acl"], check.DeepEquals, []string{"private"})
296
+}
297
+
298
+// DelObject docs: http://goo.gl/APeTt
299
+
300
+func (s *S) TestDelObject(c *check.C) {
301
+ testServer.Response(200, nil, "")
302
+
303
+ b := s.s3.Bucket("bucket")
304
+ err := b.Del("name")
305
+ c.Assert(err, check.IsNil)
306
+
307
+ req := testServer.WaitRequest()
308
+ c.Assert(req.Method, check.Equals, "DELETE")
309
+ c.Assert(req.URL.Path, check.Equals, "/bucket/name")
310
+ c.Assert(req.Header["Date"], check.Not(check.Equals), "")
311
+}
312
+
313
+func (s *S) TestDelMultiObjects(c *check.C) {
314
+ testServer.Response(200, nil, "")
315
+
316
+ b := s.s3.Bucket("bucket")
317
+ objects := []s3.Object{s3.Object{Key: "test"}}
318
+ err := b.DelMulti(s3.Delete{
319
+ Quiet: false,
320
+ Objects: objects,
321
+ })
322
+ c.Assert(err, check.IsNil)
323
+
324
+ req := testServer.WaitRequest()
325
+ c.Assert(req.Method, check.Equals, "POST")
326
+ c.Assert(req.URL.RawQuery, check.Equals, "delete=")
327
+ c.Assert(req.Header["Date"], check.Not(check.Equals), "")
328
+ c.Assert(req.Header["Content-MD5"], check.Not(check.Equals), "")
329
+ c.Assert(req.Header["Content-Type"], check.Not(check.Equals), "")
330
+ c.Assert(req.ContentLength, check.Not(check.Equals), "")
331
+}
332
+
333
+// Bucket List Objects docs: http://goo.gl/YjQTc
334
+
335
+func (s *S) TestList(c *check.C) {
336
+ testServer.Response(200, nil, GetListResultDump1)
337
+
338
+ b := s.s3.Bucket("quotes")
339
+
340
+ data, err := b.List("N", "", "", 0)
341
+ c.Assert(err, check.IsNil)
342
+
343
+ req := testServer.WaitRequest()
344
+ c.Assert(req.Method, check.Equals, "GET")
345
+ c.Assert(req.URL.Path, check.Equals, "/quotes/")
346
+ c.Assert(req.Header["Date"], check.Not(check.Equals), "")
347
+ c.Assert(req.Form["prefix"], check.DeepEquals, []string{"N"})
348
+ c.Assert(req.Form["delimiter"], check.DeepEquals, []string{""})
349
+ c.Assert(req.Form["marker"], check.DeepEquals, []string{""})
350
+ c.Assert(req.Form["max-keys"], check.DeepEquals, []string(nil))
351
+
352
+ c.Assert(data.Name, check.Equals, "quotes")
353
+ c.Assert(data.Prefix, check.Equals, "N")
354
+ c.Assert(data.IsTruncated, check.Equals, false)
355
+ c.Assert(len(data.Contents), check.Equals, 2)
356
+
357
+ c.Assert(data.Contents[0].Key, check.Equals, "Nelson")
358
+ c.Assert(data.Contents[0].LastModified, check.Equals, "2006-01-01T12:00:00.000Z")
359
+ c.Assert(data.Contents[0].ETag, check.Equals, `"828ef3fdfa96f00ad9f27c383fc9ac7f"`)
360
+ c.Assert(data.Contents[0].Size, check.Equals, int64(5))
361
+ c.Assert(data.Contents[0].StorageClass, check.Equals, "STANDARD")
362
+ c.Assert(data.Contents[0].Owner.ID, check.Equals, "bcaf161ca5fb16fd081034f")
363
+ c.Assert(data.Contents[0].Owner.DisplayName, check.Equals, "webfile")
364
+
365
+ c.Assert(data.Contents[1].Key, check.Equals, "Neo")
366
+ c.Assert(data.Contents[1].LastModified, check.Equals, "2006-01-01T12:00:00.000Z")
367
+ c.Assert(data.Contents[1].ETag, check.Equals, `"828ef3fdfa96f00ad9f27c383fc9ac7f"`)
368
+ c.Assert(data.Contents[1].Size, check.Equals, int64(4))
369
+ c.Assert(data.Contents[1].StorageClass, check.Equals, "STANDARD")
370
+ c.Assert(data.Contents[1].Owner.ID, check.Equals, "bcaf1ffd86a5fb16fd081034f")
371
+ c.Assert(data.Contents[1].Owner.DisplayName, check.Equals, "webfile")
372
+}
373
+
374
+func (s *S) TestListWithDelimiter(c *check.C) {
375
+ testServer.Response(200, nil, GetListResultDump2)
376
+
377
+ b := s.s3.Bucket("quotes")
378
+
379
+ data, err := b.List("photos/2006/", "/", "some-marker", 1000)
380
+ c.Assert(err, check.IsNil)
381
+
382
+ req := testServer.WaitRequest()
383
+ c.Assert(req.Method, check.Equals, "GET")
384
+ c.Assert(req.URL.Path, check.Equals, "/quotes/")
385
+ c.Assert(req.Header["Date"], check.Not(check.Equals), "")
386
+ c.Assert(req.Form["prefix"], check.DeepEquals, []string{"photos/2006/"})
387
+ c.Assert(req.Form["delimiter"], check.DeepEquals, []string{"/"})
388
+ c.Assert(req.Form["marker"], check.DeepEquals, []string{"some-marker"})
389
+ c.Assert(req.Form["max-keys"], check.DeepEquals, []string{"1000"})
390
+
391
+ c.Assert(data.Name, check.Equals, "example-bucket")
392
+ c.Assert(data.Prefix, check.Equals, "photos/2006/")
393
+ c.Assert(data.Delimiter, check.Equals, "/")
394
+ c.Assert(data.Marker, check.Equals, "some-marker")
395
+ c.Assert(data.IsTruncated, check.Equals, false)
396
+ c.Assert(len(data.Contents), check.Equals, 0)
397
+ c.Assert(data.CommonPrefixes, check.DeepEquals, []string{"photos/2006/feb/", "photos/2006/jan/"})
398
+}
399
+
400
+func (s *S) TestExists(c *check.C) {
401
+ testServer.Response(200, nil, "")
402
+
403
+ b := s.s3.Bucket("bucket")
404
+ result, err := b.Exists("name")
405
+
406
+ req := testServer.WaitRequest()
407
+
408
+ c.Assert(req.Method, check.Equals, "HEAD")
409
+
410
+ c.Assert(err, check.IsNil)
411
+ c.Assert(result, check.Equals, true)
412
+}
413
+
414
+func (s *S) TestExistsNotFound404(c *check.C) {
415
+ testServer.Response(404, nil, "")
416
+
417
+ b := s.s3.Bucket("bucket")
418
+ result, err := b.Exists("name")
419
+
420
+ req := testServer.WaitRequest()
421
+
422
+ c.Assert(req.Method, check.Equals, "HEAD")
423
+
424
+ c.Assert(err, check.IsNil)
425
+ c.Assert(result, check.Equals, false)
426
+}
427
+
428
+func (s *S) TestExistsNotFound403(c *check.C) {
429
+ testServer.Response(403, nil, "")
430
+
431
+ b := s.s3.Bucket("bucket")
432
+ result, err := b.Exists("name")
433
+
434
+ req := testServer.WaitRequest()
435
+
436
+ c.Assert(req.Method, check.Equals, "HEAD")
437
+
438
+ c.Assert(err, check.IsNil)
439
+ c.Assert(result, check.Equals, false)
440
+}
441
+
442
+func (s *S) TestGetService(c *check.C) {
443
+ testServer.Response(200, nil, GetServiceDump)
444
+
445
+ expected := s3.GetServiceResp{
446
+ Owner: s3.Owner{
447
+ ID: "bcaf1ffd86f461ca5fb16fd081034f",
448
+ DisplayName: "webfile",
449
+ },
450
+ Buckets: []s3.BucketInfo{
451
+ s3.BucketInfo{
452
+ Name: "quotes",
453
+ CreationDate: "2006-02-03T16:45:09.000Z",
454
+ },
455
+ s3.BucketInfo{
456
+ Name: "samples",
457
+ CreationDate: "2006-02-03T16:41:58.000Z",
458
+ },
459
+ },
460
+ }
461
+
462
+ received, err := s.s3.GetService()
463
+
464
+ c.Assert(err, check.IsNil)
465
+ c.Assert(*received, check.DeepEquals, expected)
466
+}
467
+
468
+func (s *S) TestLocation(c *check.C) {
469
+ testServer.Response(200, nil, GetLocationUsStandard)
470
+ expectedUsStandard := "us-east-1"
471
+
472
+ bucketUsStandard := s.s3.Bucket("us-east-1")
473
+ resultUsStandard, err := bucketUsStandard.Location()
474
+
475
+ c.Assert(err, check.IsNil)
476
+ c.Assert(resultUsStandard, check.Equals, expectedUsStandard)
477
+
478
+ testServer.Response(200, nil, GetLocationUsWest1)
479
+ expectedUsWest1 := "us-west-1"
480
+
481
+ bucketUsWest1 := s.s3.Bucket("us-west-1")
482
+ resultUsWest1, err := bucketUsWest1.Location()
483
+
484
+ c.Assert(err, check.IsNil)
485
+ c.Assert(resultUsWest1, check.Equals, expectedUsWest1)
486
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3i_test.go
new
+589
@@ -0,0 +1,589 @@
1
+package s3_test
2
+
3
+import (
4
+ "bytes"
5
+ "crypto/md5"
6
+ "fmt"
7
+ "github.com/crowdmob/goamz/testutil"
8
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/aws"
9
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/s3"
10
+ "gopkg.in/check.v1"
11
+ "io/ioutil"
12
+ "net"
13
+ "net/http"
14
+ "sort"
15
+ "strings"
16
+ "time"
17
+)
18
+
19
+// AmazonServer represents an Amazon S3 server.
20
+type AmazonServer struct {
21
+ auth aws.Auth
22
+}
23
+
24
+func (s *AmazonServer) SetUp(c *check.C) {
25
+ auth, err := aws.EnvAuth()
26
+ if err != nil {
27
+ c.Fatal(err.Error())
28
+ }
29
+ s.auth = auth
30
+}
31
+
32
+var _ = check.Suite(&AmazonClientSuite{Region: aws.USEast})
33
+var _ = check.Suite(&AmazonClientSuite{Region: aws.EUWest})
34
+var _ = check.Suite(&AmazonDomainClientSuite{Region: aws.USEast})
35
+
36
+// AmazonClientSuite tests the client against a live S3 server.
37
+type AmazonClientSuite struct {
38
+ aws.Region
39
+ srv AmazonServer
40
+ ClientTests
41
+}
42
+
43
+func (s *AmazonClientSuite) SetUpSuite(c *check.C) {
44
+ if !testutil.Amazon {
45
+ c.Skip("live tests against AWS disabled (no -amazon)")
46
+ }
47
+ s.srv.SetUp(c)
48
+ s.s3 = s3.New(s.srv.auth, s.Region)
49
+ // In case tests were interrupted in the middle before.
50
+ s.ClientTests.Cleanup()
51
+}
52
+
53
+func (s *AmazonClientSuite) TearDownTest(c *check.C) {
54
+ s.ClientTests.Cleanup()
55
+}
56
+
57
+// AmazonDomainClientSuite tests the client against a live S3
58
+// server using bucket names in the endpoint domain name rather
59
+// than the request path.
60
+type AmazonDomainClientSuite struct {
61
+ aws.Region
62
+ srv AmazonServer
63
+ ClientTests
64
+}
65
+
66
+func (s *AmazonDomainClientSuite) SetUpSuite(c *check.C) {
67
+ if !testutil.Amazon {
68
+ c.Skip("live tests against AWS disabled (no -amazon)")
69
+ }
70
+ s.srv.SetUp(c)
71
+ region := s.Region
72
+ region.S3BucketEndpoint = "https://${bucket}.s3.amazonaws.com"
73
+ s.s3 = s3.New(s.srv.auth, region)
74
+ s.ClientTests.Cleanup()
75
+}
76
+
77
+func (s *AmazonDomainClientSuite) TearDownTest(c *check.C) {
78
+ s.ClientTests.Cleanup()
79
+}
80
+
81
+// ClientTests defines integration tests designed to test the client.
82
+// It is not used as a test suite in itself, but embedded within
83
+// another type.
84
+type ClientTests struct {
85
+ s3 *s3.S3
86
+ authIsBroken bool
87
+}
88
+
89
+func (s *ClientTests) Cleanup() {
90
+ killBucket(testBucket(s.s3))
91
+}
92
+
93
+func testBucket(s *s3.S3) *s3.Bucket {
94
+ // Watch out! If this function is corrupted and made to match with something
95
+ // people own, killBucket will happily remove *everything* inside the bucket.
96
+ key := s.Auth.AccessKey
97
+ if len(key) >= 8 {
98
+ key = s.Auth.AccessKey[:8]
99
+ }
100
+ return s.Bucket(fmt.Sprintf("goamz-%s-%s", s.Region.Name, key))
101
+}
102
+
103
+var attempts = aws.AttemptStrategy{
104
+ Min: 5,
105
+ Total: 20 * time.Second,
106
+ Delay: 100 * time.Millisecond,
107
+}
108
+
109
+func killBucket(b *s3.Bucket) {
110
+ var err error
111
+ for attempt := attempts.Start(); attempt.Next(); {
112
+ err = b.DelBucket()
113
+ if err == nil {
114
+ return
115
+ }
116
+ if _, ok := err.(*net.DNSError); ok {
117
+ return
118
+ }
119
+ e, ok := err.(*s3.Error)
120
+ if ok && e.Code == "NoSuchBucket" {
121
+ return
122
+ }
123
+ if ok && e.Code == "BucketNotEmpty" {
124
+ // Errors are ignored here. Just retry.
125
+ resp, err := b.List("", "", "", 1000)
126
+ if err == nil {
127
+ for _, key := range resp.Contents {
128
+ _ = b.Del(key.Key)
129
+ }
130
+ }
131
+ multis, _, _ := b.ListMulti("", "")
132
+ for _, m := range multis {
133
+ _ = m.Abort()
134
+ }
135
+ }
136
+ }
137
+ message := "cannot delete test bucket"
138
+ if err != nil {
139
+ message += ": " + err.Error()
140
+ }
141
+ panic(message)
142
+}
143
+
144
+func get(url string) ([]byte, error) {
145
+ for attempt := attempts.Start(); attempt.Next(); {
146
+ resp, err := http.Get(url)
147
+ if err != nil {
148
+ if attempt.HasNext() {
149
+ continue
150
+ }
151
+ return nil, err
152
+ }
153
+ data, err := ioutil.ReadAll(resp.Body)
154
+ resp.Body.Close()
155
+ if err != nil {
156
+ if attempt.HasNext() {
157
+ continue
158
+ }
159
+ return nil, err
160
+ }
161
+ return data, err
162
+ }
163
+ panic("unreachable")
164
+}
165
+
166
+func (s *ClientTests) TestBasicFunctionality(c *check.C) {
167
+ b := testBucket(s.s3)
168
+ err := b.PutBucket(s3.PublicRead)
169
+ c.Assert(err, check.IsNil)
170
+
171
+ err = b.Put("name", []byte("yo!"), "text/plain", s3.PublicRead, s3.Options{})
172
+ c.Assert(err, check.IsNil)
173
+ defer b.Del("name")
174
+
175
+ data, err := b.Get("name")
176
+ c.Assert(err, check.IsNil)
177
+ c.Assert(string(data), check.Equals, "yo!")
178
+
179
+ data, err = get(b.URL("name"))
180
+ c.Assert(err, check.IsNil)
181
+ c.Assert(string(data), check.Equals, "yo!")
182
+
183
+ buf := bytes.NewBufferString("hey!")
184
+ err = b.PutReader("name2", buf, int64(buf.Len()), "text/plain", s3.Private, s3.Options{})
185
+ c.Assert(err, check.IsNil)
186
+ defer b.Del("name2")
187
+
188
+ rc, err := b.GetReader("name2")
189
+ c.Assert(err, check.IsNil)
190
+ data, err = ioutil.ReadAll(rc)
191
+ c.Check(err, check.IsNil)
192
+ c.Check(string(data), check.Equals, "hey!")
193
+ rc.Close()
194
+
195
+ data, err = get(b.SignedURL("name2", time.Now().Add(time.Hour)))
196
+ c.Assert(err, check.IsNil)
197
+ c.Assert(string(data), check.Equals, "hey!")
198
+
199
+ if !s.authIsBroken {
200
+ data, err = get(b.SignedURL("name2", time.Now().Add(-time.Hour)))
201
+ c.Assert(err, check.IsNil)
202
+ c.Assert(string(data), check.Matches, "(?s).*AccessDenied.*")
203
+ }
204
+
205
+ err = b.DelBucket()
206
+ c.Assert(err, check.NotNil)
207
+
208
+ s3err, ok := err.(*s3.Error)
209
+ c.Assert(ok, check.Equals, true)
210
+ c.Assert(s3err.Code, check.Equals, "BucketNotEmpty")
211
+ c.Assert(s3err.BucketName, check.Equals, b.Name)
212
+ c.Assert(s3err.Message, check.Equals, "The bucket you tried to delete is not empty")
213
+
214
+ err = b.Del("name")
215
+ c.Assert(err, check.IsNil)
216
+ err = b.Del("name2")
217
+ c.Assert(err, check.IsNil)
218
+
219
+ err = b.DelBucket()
220
+ c.Assert(err, check.IsNil)
221
+}
222
+
223
+func (s *ClientTests) TestGetNotFound(c *check.C) {
224
+ b := s.s3.Bucket("goamz-" + s.s3.Auth.AccessKey)
225
+ data, err := b.Get("non-existent")
226
+
227
+ s3err, _ := err.(*s3.Error)
228
+ c.Assert(s3err, check.NotNil)
229
+ c.Assert(s3err.StatusCode, check.Equals, 404)
230
+ c.Assert(s3err.Code, check.Equals, "NoSuchBucket")
231
+ c.Assert(s3err.Message, check.Equals, "The specified bucket does not exist")
232
+ c.Assert(data, check.IsNil)
233
+}
234
+
235
+// Communicate with all endpoints to see if they are alive.
236
+func (s *ClientTests) TestRegions(c *check.C) {
237
+ errs := make(chan error, len(aws.Regions))
238
+ for _, region := range aws.Regions {
239
+ go func(r aws.Region) {
240
+ s := s3.New(s.s3.Auth, r)
241
+ b := s.Bucket("goamz-" + s.Auth.AccessKey)
242
+ _, err := b.Get("non-existent")
243
+ errs <- err
244
+ }(region)
245
+ }
246
+ for _ = range aws.Regions {
247
+ err := <-errs
248
+ if err != nil {
249
+ s3_err, ok := err.(*s3.Error)
250
+ if ok {
251
+ c.Check(s3_err.Code, check.Matches, "NoSuchBucket")
252
+ } else if _, ok = err.(*net.DNSError); ok {
253
+ // Okay as well.
254
+ } else {
255
+ c.Errorf("Non-S3 error: %s", err)
256
+ }
257
+ } else {
258
+ c.Errorf("Test should have errored but it seems to have succeeded")
259
+ }
260
+ }
261
+}
262
+
263
+var objectNames = []string{
264
+ "index.html",
265
+ "index2.html",
266
+ "photos/2006/February/sample2.jpg",
267
+ "photos/2006/February/sample3.jpg",
268
+ "photos/2006/February/sample4.jpg",
269
+ "photos/2006/January/sample.jpg",
270
+ "test/bar",
271
+ "test/foo",
272
+}
273
+
274
+func keys(names ...string) []s3.Key {
275
+ ks := make([]s3.Key, len(names))
276
+ for i, name := range names {
277
+ ks[i].Key = name
278
+ }
279
+ return ks
280
+}
281
+
282
+// As the ListResp specifies all the parameters to the
283
+// request too, we use it to specify request parameters
284
+// and expected results. The Contents field is
285
+// used only for the key names inside it.
286
+var listTests = []s3.ListResp{
287
+ // normal list.
288
+ {
289
+ Contents: keys(objectNames...),
290
+ }, {
291
+ Marker: objectNames[0],
292
+ Contents: keys(objectNames[1:]...),
293
+ }, {
294
+ Marker: objectNames[0] + "a",
295
+ Contents: keys(objectNames[1:]...),
296
+ }, {
297
+ Marker: "z",
298
+ },
299
+
300
+ // limited results.
301
+ {
302
+ MaxKeys: 2,
303
+ Contents: keys(objectNames[0:2]...),
304
+ IsTruncated: true,
305
+ }, {
306
+ MaxKeys: 2,
307
+ Marker: objectNames[0],
308
+ Contents: keys(objectNames[1:3]...),
309
+ IsTruncated: true,
310
+ }, {
311
+ MaxKeys: 2,
312
+ Marker: objectNames[len(objectNames)-2],
313
+ Contents: keys(objectNames[len(objectNames)-1:]...),
314
+ },
315
+
316
+ // with delimiter
317
+ {
318
+ Delimiter: "/",
319
+ CommonPrefixes: []string{"photos/", "test/"},
320
+ Contents: keys("index.html", "index2.html"),
321
+ }, {
322
+ Delimiter: "/",
323
+ Prefix: "photos/2006/",
324
+ CommonPrefixes: []string{"photos/2006/February/", "photos/2006/January/"},
325
+ }, {
326
+ Delimiter: "/",
327
+ Prefix: "t",
328
+ CommonPrefixes: []string{"test/"},
329
+ }, {
330
+ Delimiter: "/",
331
+ MaxKeys: 1,
332
+ Contents: keys("index.html"),
333
+ IsTruncated: true,
334
+ }, {
335
+ Delimiter: "/",
336
+ MaxKeys: 1,
337
+ Marker: "index2.html",
338
+ CommonPrefixes: []string{"photos/"},
339
+ IsTruncated: true,
340
+ }, {
341
+ Delimiter: "/",
342
+ MaxKeys: 1,
343
+ Marker: "photos/",
344
+ CommonPrefixes: []string{"test/"},
345
+ IsTruncated: false,
346
+ }, {
347
+ Delimiter: "Feb",
348
+ CommonPrefixes: []string{"photos/2006/Feb"},
349
+ Contents: keys("index.html", "index2.html", "photos/2006/January/sample.jpg", "test/bar", "test/foo"),
350
+ },
351
+}
352
+
353
+func (s *ClientTests) TestDoublePutBucket(c *check.C) {
354
+ b := testBucket(s.s3)
355
+ err := b.PutBucket(s3.PublicRead)
356
+ c.Assert(err, check.IsNil)
357
+
358
+ err = b.PutBucket(s3.PublicRead)
359
+ if err != nil {
360
+ c.Assert(err, check.FitsTypeOf, new(s3.Error))
361
+ c.Assert(err.(*s3.Error).Code, check.Equals, "BucketAlreadyOwnedByYou")
362
+ }
363
+}
364
+
365
+func (s *ClientTests) TestBucketList(c *check.C) {
366
+ b := testBucket(s.s3)
367
+ err := b.PutBucket(s3.Private)
368
+ c.Assert(err, check.IsNil)
369
+
370
+ objData := make(map[string][]byte)
371
+ for i, path := range objectNames {
372
+ data := []byte(strings.Repeat("a", i))
373
+ err := b.Put(path, data, "text/plain", s3.Private, s3.Options{})
374
+ c.Assert(err, check.IsNil)
375
+ defer b.Del(path)
376
+ objData[path] = data
377
+ }
378
+
379
+ for i, t := range listTests {
380
+ c.Logf("test %d", i)
381
+ resp, err := b.List(t.Prefix, t.Delimiter, t.Marker, t.MaxKeys)
382
+ c.Assert(err, check.IsNil)
383
+ c.Check(resp.Name, check.Equals, b.Name)
384
+ c.Check(resp.Delimiter, check.Equals, t.Delimiter)
385
+ c.Check(resp.IsTruncated, check.Equals, t.IsTruncated)
386
+ c.Check(resp.CommonPrefixes, check.DeepEquals, t.CommonPrefixes)
387
+ checkContents(c, resp.Contents, objData, t.Contents)
388
+ }
389
+}
390
+
391
+func etag(data []byte) string {
392
+ sum := md5.New()
393
+ sum.Write(data)
394
+ return fmt.Sprintf(`"%x"`, sum.Sum(nil))
395
+}
396
+
397
+func checkContents(c *check.C, contents []s3.Key, data map[string][]byte, expected []s3.Key) {
398
+ c.Assert(contents, check.HasLen, len(expected))
399
+ for i, k := range contents {
400
+ c.Check(k.Key, check.Equals, expected[i].Key)
401
+ // TODO mtime
402
+ c.Check(k.Size, check.Equals, int64(len(data[k.Key])))
403
+ c.Check(k.ETag, check.Equals, etag(data[k.Key]))
404
+ }
405
+}
406
+
407
+func (s *ClientTests) TestMultiInitPutList(c *check.C) {
408
+ b := testBucket(s.s3)
409
+ err := b.PutBucket(s3.Private)
410
+ c.Assert(err, check.IsNil)
411
+
412
+ multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
413
+ c.Assert(err, check.IsNil)
414
+ c.Assert(multi.UploadId, check.Matches, ".+")
415
+ defer multi.Abort()
416
+
417
+ var sent []s3.Part
418
+
419
+ for i := 0; i < 5; i++ {
420
+ p, err := multi.PutPart(i+1, strings.NewReader(fmt.Sprintf("<part %d>", i+1)))
421
+ c.Assert(err, check.IsNil)
422
+ c.Assert(p.N, check.Equals, i+1)
423
+ c.Assert(p.Size, check.Equals, int64(8))
424
+ c.Assert(p.ETag, check.Matches, ".+")
425
+ sent = append(sent, p)
426
+ }
427
+
428
+ s3.SetListPartsMax(2)
429
+
430
+ parts, err := multi.ListParts()
431
+ c.Assert(err, check.IsNil)
432
+ c.Assert(parts, check.HasLen, len(sent))
433
+ for i := range parts {
434
+ c.Assert(parts[i].N, check.Equals, sent[i].N)
435
+ c.Assert(parts[i].Size, check.Equals, sent[i].Size)
436
+ c.Assert(parts[i].ETag, check.Equals, sent[i].ETag)
437
+ }
438
+
439
+ err = multi.Complete(parts)
440
+ s3err, failed := err.(*s3.Error)
441
+ c.Assert(failed, check.Equals, true)
442
+ c.Assert(s3err.Code, check.Equals, "EntityTooSmall")
443
+
444
+ err = multi.Abort()
445
+ c.Assert(err, check.IsNil)
446
+ _, err = multi.ListParts()
447
+ s3err, ok := err.(*s3.Error)
448
+ c.Assert(ok, check.Equals, true)
449
+ c.Assert(s3err.Code, check.Equals, "NoSuchUpload")
450
+}
451
+
452
+// This may take a minute or more due to the minimum size accepted S3
453
+// on multipart upload parts.
454
+func (s *ClientTests) TestMultiComplete(c *check.C) {
455
+ b := testBucket(s.s3)
456
+ err := b.PutBucket(s3.Private)
457
+ c.Assert(err, check.IsNil)
458
+
459
+ multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
460
+ c.Assert(err, check.IsNil)
461
+ c.Assert(multi.UploadId, check.Matches, ".+")
462
+ defer multi.Abort()
463
+
464
+ // Minimum size S3 accepts for all but the last part is 5MB.
465
+ data1 := make([]byte, 5*1024*1024)
466
+ data2 := []byte("<part 2>")
467
+
468
+ part1, err := multi.PutPart(1, bytes.NewReader(data1))
469
+ c.Assert(err, check.IsNil)
470
+ part2, err := multi.PutPart(2, bytes.NewReader(data2))
471
+ c.Assert(err, check.IsNil)
472
+
473
+ // Purposefully reversed. The order requirement must be handled.
474
+ err = multi.Complete([]s3.Part{part2, part1})
475
+ c.Assert(err, check.IsNil)
476
+
477
+ data, err := b.Get("multi")
478
+ c.Assert(err, check.IsNil)
479
+
480
+ c.Assert(len(data), check.Equals, len(data1)+len(data2))
481
+ for i := range data1 {
482
+ if data[i] != data1[i] {
483
+ c.Fatalf("uploaded object at byte %d: want %d, got %d", data1[i], data[i])
484
+ }
485
+ }
486
+ c.Assert(string(data[len(data1):]), check.Equals, string(data2))
487
+}
488
+
489
+type multiList []*s3.Multi
490
+
491
+func (l multiList) Len() int { return len(l) }
492
+func (l multiList) Less(i, j int) bool { return l[i].Key < l[j].Key }
493
+func (l multiList) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
494
+
495
+func (s *ClientTests) TestListMulti(c *check.C) {
496
+ b := testBucket(s.s3)
497
+ err := b.PutBucket(s3.Private)
498
+ c.Assert(err, check.IsNil)
499
+
500
+ // Ensure an empty state before testing its behavior.
501
+ multis, _, err := b.ListMulti("", "")
502
+ for _, m := range multis {
503
+ err := m.Abort()
504
+ c.Assert(err, check.IsNil)
505
+ }
506
+
507
+ keys := []string{
508
+ "a/multi2",
509
+ "a/multi3",
510
+ "b/multi4",
511
+ "multi1",
512
+ }
513
+ for _, key := range keys {
514
+ m, err := b.InitMulti(key, "", s3.Private, s3.Options{})
515
+ c.Assert(err, check.IsNil)
516
+ defer m.Abort()
517
+ }
518
+
519
+ // Amazon's implementation of the multiple-request listing for
520
+ // multipart uploads in progress seems broken in multiple ways.
521
+ // (next tokens are not provided, etc).
522
+ //s3.SetListMultiMax(2)
523
+
524
+ multis, prefixes, err := b.ListMulti("", "")
525
+ c.Assert(err, check.IsNil)
526
+ for attempt := attempts.Start(); attempt.Next() && len(multis) < len(keys); {
527
+ multis, prefixes, err = b.ListMulti("", "")
528
+ c.Assert(err, check.IsNil)
529
+ }
530
+ sort.Sort(multiList(multis))
531
+ c.Assert(prefixes, check.IsNil)
532
+ var gotKeys []string
533
+ for _, m := range multis {
534
+ gotKeys = append(gotKeys, m.Key)
535
+ }
536
+ c.Assert(gotKeys, check.DeepEquals, keys)
537
+ for _, m := range multis {
538
+ c.Assert(m.Bucket, check.Equals, b)
539
+ c.Assert(m.UploadId, check.Matches, ".+")
540
+ }
541
+
542
+ multis, prefixes, err = b.ListMulti("", "/")
543
+ for attempt := attempts.Start(); attempt.Next() && len(prefixes) < 2; {
544
+ multis, prefixes, err = b.ListMulti("", "")
545
+ c.Assert(err, check.IsNil)
546
+ }
547
+ c.Assert(err, check.IsNil)
548
+ c.Assert(prefixes, check.DeepEquals, []string{"a/", "b/"})
549
+ c.Assert(multis, check.HasLen, 1)
550
+ c.Assert(multis[0].Bucket, check.Equals, b)
551
+ c.Assert(multis[0].Key, check.Equals, "multi1")
552
+ c.Assert(multis[0].UploadId, check.Matches, ".+")
553
+
554
+ for attempt := attempts.Start(); attempt.Next() && len(multis) < 2; {
555
+ multis, prefixes, err = b.ListMulti("", "")
556
+ c.Assert(err, check.IsNil)
557
+ }
558
+ multis, prefixes, err = b.ListMulti("a/", "/")
559
+ c.Assert(err, check.IsNil)
560
+ c.Assert(prefixes, check.IsNil)
561
+ c.Assert(multis, check.HasLen, 2)
562
+ c.Assert(multis[0].Bucket, check.Equals, b)
563
+ c.Assert(multis[0].Key, check.Equals, "a/multi2")
564
+ c.Assert(multis[0].UploadId, check.Matches, ".+")
565
+ c.Assert(multis[1].Bucket, check.Equals, b)
566
+ c.Assert(multis[1].Key, check.Equals, "a/multi3")
567
+ c.Assert(multis[1].UploadId, check.Matches, ".+")
568
+}
569
+
570
+func (s *ClientTests) TestMultiPutAllZeroLength(c *check.C) {
571
+ b := testBucket(s.s3)
572
+ err := b.PutBucket(s3.Private)
573
+ c.Assert(err, check.IsNil)
574
+
575
+ multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
576
+ c.Assert(err, check.IsNil)
577
+ defer multi.Abort()
578
+
579
+ // This tests an edge case. Amazon requires at least one
580
+ // part for multiprat uploads to work, even the part is empty.
581
+ parts, err := multi.PutAll(strings.NewReader(""), 5*1024*1024)
582
+ c.Assert(err, check.IsNil)
583
+ c.Assert(parts, check.HasLen, 1)
584
+ c.Assert(parts[0].Size, check.Equals, int64(0))
585
+ c.Assert(parts[0].ETag, check.Equals, `"d41d8cd98f00b204e9800998ecf8427e"`)
586
+
587
+ err = multi.Complete(parts)
588
+ c.Assert(err, check.IsNil)
589
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3t_test.go
new
+79
@@ -0,0 +1,79 @@
1
+package s3_test
2
+
3
+import (
4
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/aws"
5
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/s3"
6
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3test"
7
+ "gopkg.in/check.v1"
8
+)
9
+
10
+type LocalServer struct {
11
+ auth aws.Auth
12
+ region aws.Region
13
+ srv *s3test.Server
14
+ config *s3test.Config
15
+}
16
+
17
+func (s *LocalServer) SetUp(c *check.C) {
18
+ srv, err := s3test.NewServer(s.config)
19
+ c.Assert(err, check.IsNil)
20
+ c.Assert(srv, check.NotNil)
21
+
22
+ s.srv = srv
23
+ s.region = aws.Region{
24
+ Name: "faux-region-1",
25
+ S3Endpoint: srv.URL(),
26
+ S3LocationConstraint: true, // s3test server requires a LocationConstraint
27
+ }
28
+}
29
+
30
+// LocalServerSuite defines tests that will run
31
+// against the local s3test server. It includes
32
+// selected tests from ClientTests;
33
+// when the s3test functionality is sufficient, it should
34
+// include all of them, and ClientTests can be simply embedded.
35
+type LocalServerSuite struct {
36
+ srv LocalServer
37
+ clientTests ClientTests
38
+}
39
+
40
+var (
41
+ // run tests twice, once in us-east-1 mode, once not.
42
+ _ = check.Suite(&LocalServerSuite{})
43
+ _ = check.Suite(&LocalServerSuite{
44
+ srv: LocalServer{
45
+ config: &s3test.Config{
46
+ Send409Conflict: true,
47
+ },
48
+ },
49
+ })
50
+)
51
+
52
+func (s *LocalServerSuite) SetUpSuite(c *check.C) {
53
+ s.srv.SetUp(c)
54
+ s.clientTests.s3 = s3.New(s.srv.auth, s.srv.region)
55
+
56
+ // TODO Sadly the fake server ignores auth completely right now. :-(
57
+ s.clientTests.authIsBroken = true
58
+ s.clientTests.Cleanup()
59
+}
60
+
61
+func (s *LocalServerSuite) TearDownTest(c *check.C) {
62
+ s.clientTests.Cleanup()
63
+}
64
+
65
+func (s *LocalServerSuite) TestBasicFunctionality(c *check.C) {
66
+ s.clientTests.TestBasicFunctionality(c)
67
+}
68
+
69
+func (s *LocalServerSuite) TestGetNotFound(c *check.C) {
70
+ s.clientTests.TestGetNotFound(c)
71
+}
72
+
73
+func (s *LocalServerSuite) TestBucketList(c *check.C) {
74
+ s.clientTests.TestBucketList(c)
75
+}
76
+
77
+func (s *LocalServerSuite) TestDoublePutBucket(c *check.C) {
78
+ s.clientTests.TestDoublePutBucket(c)
79
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3test/server.go
new
+629
@@ -0,0 +1,629 @@
1
+package s3test
2
+
3
+import (
4
+ "bytes"
5
+ "crypto/md5"
6
+ "encoding/base64"
7
+ "encoding/hex"
8
+ "encoding/xml"
9
+ "fmt"
10
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/s3"
11
+ "io"
12
+ "io/ioutil"
13
+ "log"
14
+ "net"
15
+ "net/http"
16
+ "net/url"
17
+ "regexp"
18
+ "sort"
19
+ "strconv"
20
+ "strings"
21
+ "sync"
22
+ "time"
23
+)
24
+
25
+const debug = false
26
+
27
+type s3Error struct {
28
+ statusCode int
29
+ XMLName struct{} `xml:"Error"`
30
+ Code string
31
+ Message string
32
+ BucketName string
33
+ RequestId string
34
+ HostId string
35
+}
36
+
37
+type action struct {
38
+ srv *Server
39
+ w http.ResponseWriter
40
+ req *http.Request
41
+ reqId string
42
+}
43
+
44
+// Config controls the internal behaviour of the Server. A nil config is the default
45
+// and behaves as if all configurations assume their default behaviour. Once passed
46
+// to NewServer, the configuration must not be modified.
47
+type Config struct {
48
+ // Send409Conflict controls how the Server will respond to calls to PUT on a
49
+ // previously existing bucket. The default is false, and corresponds to the
50
+ // us-east-1 s3 enpoint. Setting this value to true emulates the behaviour of
51
+ // all other regions.
52
+ // http://docs.amazonwebservices.com/AmazonS3/latest/API/ErrorResponses.html
53
+ Send409Conflict bool
54
+}
55
+
56
+func (c *Config) send409Conflict() bool {
57
+ if c != nil {
58
+ return c.Send409Conflict
59
+ }
60
+ return false
61
+}
62
+
63
+// Server is a fake S3 server for testing purposes.
64
+// All of the data for the server is kept in memory.
65
+type Server struct {
66
+ url string
67
+ reqId int
68
+ listener net.Listener
69
+ mu sync.Mutex
70
+ buckets map[string]*bucket
71
+ config *Config
72
+}
73
+
74
+type bucket struct {
75
+ name string
76
+ acl s3.ACL
77
+ ctime time.Time
78
+ objects map[string]*object
79
+}
80
+
81
+type object struct {
82
+ name string
83
+ mtime time.Time
84
+ meta http.Header // metadata to return with requests.
85
+ checksum []byte // also held as Content-MD5 in meta.
86
+ data []byte
87
+}
88
+
89
+// A resource encapsulates the subject of an HTTP request.
90
+// The resource referred to may or may not exist
91
+// when the request is made.
92
+type resource interface {
93
+ put(a *action) interface{}
94
+ get(a *action) interface{}
95
+ post(a *action) interface{}
96
+ delete(a *action) interface{}
97
+}
98
+
99
+func NewServer(config *Config) (*Server, error) {
100
+ l, err := net.Listen("tcp", "localhost:0")
101
+ if err != nil {
102
+ return nil, fmt.Errorf("cannot listen on localhost: %v", err)
103
+ }
104
+ srv := &Server{
105
+ listener: l,
106
+ url: "http://" + l.Addr().String(),
107
+ buckets: make(map[string]*bucket),
108
+ config: config,
109
+ }
110
+ go http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
111
+ srv.serveHTTP(w, req)
112
+ }))
113
+ return srv, nil
114
+}
115
+
116
+// Quit closes down the server.
117
+func (srv *Server) Quit() {
118
+ srv.listener.Close()
119
+}
120
+
121
+// URL returns a URL for the server.
122
+func (srv *Server) URL() string {
123
+ return srv.url
124
+}
125
+
126
+func fatalf(code int, codeStr string, errf string, a ...interface{}) {
127
+ panic(&s3Error{
128
+ statusCode: code,
129
+ Code: codeStr,
130
+ Message: fmt.Sprintf(errf, a...),
131
+ })
132
+}
133
+
134
+// serveHTTP serves the S3 protocol.
135
+func (srv *Server) serveHTTP(w http.ResponseWriter, req *http.Request) {
136
+ // ignore error from ParseForm as it's usually spurious.
137
+ req.ParseForm()
138
+
139
+ srv.mu.Lock()
140
+ defer srv.mu.Unlock()
141
+
142
+ if debug {
143
+ log.Printf("s3test %q %q", req.Method, req.URL)
144
+ }
145
+ a := &action{
146
+ srv: srv,
147
+ w: w,
148
+ req: req,
149
+ reqId: fmt.Sprintf("%09X", srv.reqId),
150
+ }
151
+ srv.reqId++
152
+
153
+ var r resource
154
+ defer func() {
155
+ switch err := recover().(type) {
156
+ case *s3Error:
157
+ switch r := r.(type) {
158
+ case objectResource:
159
+ err.BucketName = r.bucket.name
160
+ case bucketResource:
161
+ err.BucketName = r.name
162
+ }
163
+ err.RequestId = a.reqId
164
+ // TODO HostId
165
+ w.Header().Set("Content-Type", `xml version="1.0" encoding="UTF-8"`)
166
+ w.WriteHeader(err.statusCode)
167
+ xmlMarshal(w, err)
168
+ case nil:
169
+ default:
170
+ panic(err)
171
+ }
172
+ }()
173
+
174
+ r = srv.resourceForURL(req.URL)
175
+
176
+ var resp interface{}
177
+ switch req.Method {
178
+ case "PUT":
179
+ resp = r.put(a)
180
+ case "GET", "HEAD":
181
+ resp = r.get(a)
182
+ case "DELETE":
183
+ resp = r.delete(a)
184
+ case "POST":
185
+ resp = r.post(a)
186
+ default:
187
+ fatalf(400, "MethodNotAllowed", "unknown http request method %q", req.Method)
188
+ }
189
+ if resp != nil && req.Method != "HEAD" {
190
+ xmlMarshal(w, resp)
191
+ }
192
+}
193
+
194
+// xmlMarshal is the same as xml.Marshal except that
195
+// it panics on error. The marshalling should not fail,
196
+// but we want to know if it does.
197
+func xmlMarshal(w io.Writer, x interface{}) {
198
+ if err := xml.NewEncoder(w).Encode(x); err != nil {
199
+ panic(fmt.Errorf("error marshalling %#v: %v", x, err))
200
+ }
201
+}
202
+
203
+// In a fully implemented test server, each of these would have
204
+// its own resource type.
205
+var unimplementedBucketResourceNames = map[string]bool{
206
+ "acl": true,
207
+ "lifecycle": true,
208
+ "policy": true,
209
+ "location": true,
210
+ "logging": true,
211
+ "notification": true,
212
+ "versions": true,
213
+ "requestPayment": true,
214
+ "versioning": true,
215
+ "website": true,
216
+ "uploads": true,
217
+}
218
+
219
+var unimplementedObjectResourceNames = map[string]bool{
220
+ "uploadId": true,
221
+ "acl": true,
222
+ "torrent": true,
223
+ "uploads": true,
224
+}
225
+
226
+var pathRegexp = regexp.MustCompile("/(([^/]+)(/(.*))?)?")
227
+
228
+// resourceForURL returns a resource object for the given URL.
229
+func (srv *Server) resourceForURL(u *url.URL) (r resource) {
230
+ m := pathRegexp.FindStringSubmatch(u.Path)
231
+ if m == nil {
232
+ fatalf(404, "InvalidURI", "Couldn't parse the specified URI")
233
+ }
234
+ bucketName := m[2]
235
+ objectName := m[4]
236
+ if bucketName == "" {
237
+ return nullResource{} // root
238
+ }
239
+ b := bucketResource{
240
+ name: bucketName,
241
+ bucket: srv.buckets[bucketName],
242
+ }
243
+ q := u.Query()
244
+ if objectName == "" {
245
+ for name := range q {
246
+ if unimplementedBucketResourceNames[name] {
247
+ return nullResource{}
248
+ }
249
+ }
250
+ return b
251
+
252
+ }
253
+ if b.bucket == nil {
254
+ fatalf(404, "NoSuchBucket", "The specified bucket does not exist")
255
+ }
256
+ objr := objectResource{
257
+ name: objectName,
258
+ version: q.Get("versionId"),
259
+ bucket: b.bucket,
260
+ }
261
+ for name := range q {
262
+ if unimplementedObjectResourceNames[name] {
263
+ return nullResource{}
264
+ }
265
+ }
266
+ if obj := objr.bucket.objects[objr.name]; obj != nil {
267
+ objr.object = obj
268
+ }
269
+ return objr
270
+}
271
+
272
+// nullResource has error stubs for all resource methods.
273
+type nullResource struct{}
274
+
275
+func notAllowed() interface{} {
276
+ fatalf(400, "MethodNotAllowed", "The specified method is not allowed against this resource")
277
+ return nil
278
+}
279
+
280
+func (nullResource) put(a *action) interface{} { return notAllowed() }
281
+func (nullResource) get(a *action) interface{} { return notAllowed() }
282
+func (nullResource) post(a *action) interface{} { return notAllowed() }
283
+func (nullResource) delete(a *action) interface{} { return notAllowed() }
284
+
285
+const timeFormat = "2006-01-02T15:04:05.000Z07:00"
286
+
287
+type bucketResource struct {
288
+ name string
289
+ bucket *bucket // non-nil if the bucket already exists.
290
+}
291
+
292
+// GET on a bucket lists the objects in the bucket.
293
+// http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTBucketGET.html
294
+func (r bucketResource) get(a *action) interface{} {
295
+ if r.bucket == nil {
296
+ fatalf(404, "NoSuchBucket", "The specified bucket does not exist")
297
+ }
298
+ delimiter := a.req.Form.Get("delimiter")
299
+ marker := a.req.Form.Get("marker")
300
+ maxKeys := -1
301
+ if s := a.req.Form.Get("max-keys"); s != "" {
302
+ i, err := strconv.Atoi(s)
303
+ if err != nil || i < 0 {
304
+ fatalf(400, "invalid value for max-keys: %q", s)
305
+ }
306
+ maxKeys = i
307
+ }
308
+ prefix := a.req.Form.Get("prefix")
309
+ a.w.Header().Set("Content-Type", "application/xml")
310
+
311
+ if a.req.Method == "HEAD" {
312
+ return nil
313
+ }
314
+
315
+ var objs orderedObjects
316
+
317
+ // first get all matching objects and arrange them in alphabetical order.
318
+ for name, obj := range r.bucket.objects {
319
+ if strings.HasPrefix(name, prefix) {
320
+ objs = append(objs, obj)
321
+ }
322
+ }
323
+ sort.Sort(objs)
324
+
325
+ if maxKeys <= 0 {
326
+ maxKeys = 1000
327
+ }
328
+ resp := &s3.ListResp{
329
+ Name: r.bucket.name,
330
+ Prefix: prefix,
331
+ Delimiter: delimiter,
332
+ Marker: marker,
333
+ MaxKeys: maxKeys,
334
+ }
335
+
336
+ var prefixes []string
337
+ for _, obj := range objs {
338
+ if !strings.HasPrefix(obj.name, prefix) {
339
+ continue
340
+ }
341
+ name := obj.name
342
+ isPrefix := false
343
+ if delimiter != "" {
344
+ if i := strings.Index(obj.name[len(prefix):], delimiter); i >= 0 {
345
+ name = obj.name[:len(prefix)+i+len(delimiter)]
346
+ if prefixes != nil && prefixes[len(prefixes)-1] == name {
347
+ continue
348
+ }
349
+ isPrefix = true
350
+ }
351
+ }
352
+ if name <= marker {
353
+ continue
354
+ }
355
+ if len(resp.Contents)+len(prefixes) >= maxKeys {
356
+ resp.IsTruncated = true
357
+ break
358
+ }
359
+ if isPrefix {
360
+ prefixes = append(prefixes, name)
361
+ } else {
362
+ // Contents contains only keys not found in CommonPrefixes
363
+ resp.Contents = append(resp.Contents, obj.s3Key())
364
+ }
365
+ }
366
+ resp.CommonPrefixes = prefixes
367
+ return resp
368
+}
369
+
370
+// orderedObjects holds a slice of objects that can be sorted
371
+// by name.
372
+type orderedObjects []*object
373
+
374
+func (s orderedObjects) Len() int {
375
+ return len(s)
376
+}
377
+func (s orderedObjects) Swap(i, j int) {
378
+ s[i], s[j] = s[j], s[i]
379
+}
380
+func (s orderedObjects) Less(i, j int) bool {
381
+ return s[i].name < s[j].name
382
+}
383
+
384
+func (obj *object) s3Key() s3.Key {
385
+ return s3.Key{
386
+ Key: obj.name,
387
+ LastModified: obj.mtime.Format(timeFormat),
388
+ Size: int64(len(obj.data)),
389
+ ETag: fmt.Sprintf(`"%x"`, obj.checksum),
390
+ // TODO StorageClass
391
+ // TODO Owner
392
+ }
393
+}
394
+
395
+// DELETE on a bucket deletes the bucket if it's not empty.
396
+func (r bucketResource) delete(a *action) interface{} {
397
+ b := r.bucket
398
+ if b == nil {
399
+ fatalf(404, "NoSuchBucket", "The specified bucket does not exist")
400
+ }
401
+ if len(b.objects) > 0 {
402
+ fatalf(400, "BucketNotEmpty", "The bucket you tried to delete is not empty")
403
+ }
404
+ delete(a.srv.buckets, b.name)
405
+ return nil
406
+}
407
+
408
+// PUT on a bucket creates the bucket.
409
+// http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTBucketPUT.html
410
+func (r bucketResource) put(a *action) interface{} {
411
+ var created bool
412
+ if r.bucket == nil {
413
+ if !validBucketName(r.name) {
414
+ fatalf(400, "InvalidBucketName", "The specified bucket is not valid")
415
+ }
416
+ if loc := locationConstraint(a); loc == "" {
417
+ fatalf(400, "InvalidRequets", "The unspecified location constraint is incompatible for the region specific endpoint this request was sent to.")
418
+ }
419
+ // TODO validate acl
420
+ r.bucket = &bucket{
421
+ name: r.name,
422
+ // TODO default acl
423
+ objects: make(map[string]*object),
424
+ }
425
+ a.srv.buckets[r.name] = r.bucket
426
+ created = true
427
+ }
428
+ if !created && a.srv.config.send409Conflict() {
429
+ fatalf(409, "BucketAlreadyOwnedByYou", "Your previous request to create the named bucket succeeded and you already own it.")
430
+ }
431
+ r.bucket.acl = s3.ACL(a.req.Header.Get("x-amz-acl"))
432
+ return nil
433
+}
434
+
435
+func (bucketResource) post(a *action) interface{} {
436
+ fatalf(400, "Method", "bucket POST method not available")
437
+ return nil
438
+}
439
+
440
+// validBucketName returns whether name is a valid bucket name.
441
+// Here are the rules, from:
442
+// http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/BucketRestrictions.html
443
+//
444
+// Can contain lowercase letters, numbers, periods (.), underscores (_),
445
+// and dashes (-). You can use uppercase letters for buckets only in the
446
+// US Standard region.
447
+//
448
+// Must start with a number or letter
449
+//
450
+// Must be between 3 and 255 characters long
451
+//
452
+// There's one extra rule (Must not be formatted as an IP address (e.g., 192.168.5.4)
453
+// but the real S3 server does not seem to check that rule, so we will not
454
+// check it either.
455
+//
456
+func validBucketName(name string) bool {
457
+ if len(name) < 3 || len(name) > 255 {
458
+ return false
459
+ }
460
+ r := name[0]
461
+ if !(r >= '0' && r <= '9' || r >= 'a' && r <= 'z') {
462
+ return false
463
+ }
464
+ for _, r := range name {
465
+ switch {
466
+ case r >= '0' && r <= '9':
467
+ case r >= 'a' && r <= 'z':
468
+ case r == '_' || r == '-':
469
+ case r == '.':
470
+ default:
471
+ return false
472
+ }
473
+ }
474
+ return true
475
+}
476
+
477
+var responseParams = map[string]bool{
478
+ "content-type": true,
479
+ "content-language": true,
480
+ "expires": true,
481
+ "cache-control": true,
482
+ "content-disposition": true,
483
+ "content-encoding": true,
484
+}
485
+
486
+type objectResource struct {
487
+ name string
488
+ version string
489
+ bucket *bucket // always non-nil.
490
+ object *object // may be nil.
491
+}
492
+
493
+// GET on an object gets the contents of the object.
494
+// http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTObjectGET.html
495
+func (objr objectResource) get(a *action) interface{} {
496
+ obj := objr.object
497
+ if obj == nil {
498
+ fatalf(404, "NoSuchKey", "The specified key does not exist.")
499
+ }
500
+ h := a.w.Header()
501
+ // add metadata
502
+ for name, d := range obj.meta {
503
+ h[name] = d
504
+ }
505
+ // override header values in response to request parameters.
506
+ for name, vals := range a.req.Form {
507
+ if strings.HasPrefix(name, "response-") {
508
+ name = name[len("response-"):]
509
+ if !responseParams[name] {
510
+ continue
511
+ }
512
+ h.Set(name, vals[0])
513
+ }
514
+ }
515
+ if r := a.req.Header.Get("Range"); r != "" {
516
+ fatalf(400, "NotImplemented", "range unimplemented")
517
+ }
518
+ // TODO Last-Modified-Since
519
+ // TODO If-Modified-Since
520
+ // TODO If-Unmodified-Since
521
+ // TODO If-Match
522
+ // TODO If-None-Match
523
+ // TODO Connection: close ??
524
+ // TODO x-amz-request-id
525
+ h.Set("Content-Length", fmt.Sprint(len(obj.data)))
526
+ h.Set("ETag", hex.EncodeToString(obj.checksum))
527
+ h.Set("Last-Modified", obj.mtime.Format(time.RFC1123))
528
+ if a.req.Method == "HEAD" {
529
+ return nil
530
+ }
531
+ // TODO avoid holding the lock when writing data.
532
+ _, err := a.w.Write(obj.data)
533
+ if err != nil {
534
+ // we can't do much except just log the fact.
535
+ log.Printf("error writing data: %v", err)
536
+ }
537
+ return nil
538
+}
539
+
540
+var metaHeaders = map[string]bool{
541
+ "Content-MD5": true,
542
+ "x-amz-acl": true,
543
+ "Content-Type": true,
544
+ "Content-Encoding": true,
545
+ "Content-Disposition": true,
546
+}
547
+
548
+// PUT on an object creates the object.
549
+func (objr objectResource) put(a *action) interface{} {
550
+ // TODO Cache-Control header
551
+ // TODO Expires header
552
+ // TODO x-amz-server-side-encryption
553
+ // TODO x-amz-storage-class
554
+
555
+ // TODO is this correct, or should we erase all previous metadata?
556
+ obj := objr.object
557
+ if obj == nil {
558
+ obj = &object{
559
+ name: objr.name,
560
+ meta: make(http.Header),
561
+ }
562
+ }
563
+
564
+ var expectHash []byte
565
+ if c := a.req.Header.Get("Content-MD5"); c != "" {
566
+ var err error
567
+ expectHash, err = base64.StdEncoding.DecodeString(c)
568
+ if err != nil || len(expectHash) != md5.Size {
569
+ fatalf(400, "InvalidDigest", "The Content-MD5 you specified was invalid")
570
+ }
571
+ }
572
+ sum := md5.New()
573
+ // TODO avoid holding lock while reading data.
574
+ data, err := ioutil.ReadAll(io.TeeReader(a.req.Body, sum))
575
+ if err != nil {
576
+ fatalf(400, "TODO", "read error")
577
+ }
578
+ gotHash := sum.Sum(nil)
579
+ if expectHash != nil && bytes.Compare(gotHash, expectHash) != 0 {
580
+ fatalf(400, "BadDigest", "The Content-MD5 you specified did not match what we received")
581
+ }
582
+ if a.req.ContentLength >= 0 && int64(len(data)) != a.req.ContentLength {
583
+ fatalf(400, "IncompleteBody", "You did not provide the number of bytes specified by the Content-Length HTTP header")
584
+ }
585
+
586
+ // PUT request has been successful - save data and metadata
587
+ for key, values := range a.req.Header {
588
+ key = http.CanonicalHeaderKey(key)
589
+ if metaHeaders[key] || strings.HasPrefix(key, "X-Amz-Meta-") {
590
+ obj.meta[key] = values
591
+ }
592
+ }
593
+ obj.data = data
594
+ obj.checksum = gotHash
595
+ obj.mtime = time.Now()
596
+ objr.bucket.objects[objr.name] = obj
597
+ return nil
598
+}
599
+
600
+func (objr objectResource) delete(a *action) interface{} {
601
+ delete(objr.bucket.objects, objr.name)
602
+ return nil
603
+}
604
+
605
+func (objr objectResource) post(a *action) interface{} {
606
+ fatalf(400, "MethodNotAllowed", "The specified method is not allowed against this resource")
607
+ return nil
608
+}
609
+
610
+type CreateBucketConfiguration struct {
611
+ LocationConstraint string
612
+}
613
+
614
+// locationConstraint parses the <CreateBucketConfiguration /> request body (if present).
615
+// If there is no body, an empty string will be returned.
616
+func locationConstraint(a *action) string {
617
+ var body bytes.Buffer
618
+ if _, err := io.Copy(&body, a.req.Body); err != nil {
619
+ fatalf(400, "InvalidRequest", err.Error())
620
+ }
621
+ if body.Len() == 0 {
622
+ return ""
623
+ }
624
+ var loc CreateBucketConfiguration
625
+ if err := xml.NewDecoder(&body).Decode(&loc); err != nil {
626
+ fatalf(400, "InvalidRequest", err.Error())
627
+ }
628
+ return loc.LocationConstraint
629
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/s3/sign.go
new
+120
@@ -0,0 +1,120 @@
1
+package s3
2
+
3
+import (
4
+ "crypto/hmac"
5
+ "crypto/sha1"
6
+ "encoding/base64"
7
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/aws"
8
+ "log"
9
+ "sort"
10
+ "strings"
11
+)
12
+
13
+var b64 = base64.StdEncoding
14
+
15
+// ----------------------------------------------------------------------------
16
+// S3 signing (http://goo.gl/G1LrK)
17
+
18
+var s3ParamsToSign = map[string]bool{
19
+ "acl": true,
20
+ "location": true,
21
+ "logging": true,
22
+ "notification": true,
23
+ "partNumber": true,
24
+ "policy": true,
25
+ "requestPayment": true,
26
+ "torrent": true,
27
+ "uploadId": true,
28
+ "uploads": true,
29
+ "versionId": true,
30
+ "versioning": true,
31
+ "versions": true,
32
+ "response-content-type": true,
33
+ "response-content-language": true,
34
+ "response-expires": true,
35
+ "response-cache-control": true,
36
+ "response-content-disposition": true,
37
+ "response-content-encoding": true,
38
+ "website": true,
39
+ "delete": true,
40
+}
41
+
42
+func sign(auth aws.Auth, method, canonicalPath string, params, headers map[string][]string) {
43
+ var md5, ctype, date, xamz string
44
+ var xamzDate bool
45
+ var keys, sarray []string
46
+ xheaders := make(map[string]string)
47
+ for k, v := range headers {
48
+ k = strings.ToLower(k)
49
+ switch k {
50
+ case "content-md5":
51
+ md5 = v[0]
52
+ case "content-type":
53
+ ctype = v[0]
54
+ case "date":
55
+ if !xamzDate {
56
+ date = v[0]
57
+ }
58
+ default:
59
+ if strings.HasPrefix(k, "x-amz-") {
60
+ keys = append(keys, k)
61
+ xheaders[k] = strings.Join(v, ",")
62
+ if k == "x-amz-date" {
63
+ xamzDate = true
64
+ date = ""
65
+ }
66
+ }
67
+ }
68
+ }
69
+ if len(keys) > 0 {
70
+ sort.StringSlice(keys).Sort()
71
+ for i := range keys {
72
+ key := keys[i]
73
+ value := xheaders[key]
74
+ sarray = append(sarray, key+":"+value)
75
+ }
76
+ xamz = strings.Join(sarray, "\n") + "\n"
77
+ }
78
+
79
+ expires := false
80
+ if v, ok := params["Expires"]; ok {
81
+ // Query string request authentication alternative.
82
+ expires = true
83
+ date = v[0]
84
+ params["AWSAccessKeyId"] = []string{auth.AccessKey}
85
+ }
86
+
87
+ sarray = sarray[0:0]
88
+ for k, v := range params {
89
+ if s3ParamsToSign[k] {
90
+ for _, vi := range v {
91
+ if vi == "" {
92
+ sarray = append(sarray, k)
93
+ } else {
94
+ // "When signing you do not encode these values."
95
+ sarray = append(sarray, k+"="+vi)
96
+ }
97
+ }
98
+ }
99
+ }
100
+ if len(sarray) > 0 {
101
+ sort.StringSlice(sarray).Sort()
102
+ canonicalPath = canonicalPath + "?" + strings.Join(sarray, "&")
103
+ }
104
+
105
+ payload := method + "\n" + md5 + "\n" + ctype + "\n" + date + "\n" + xamz + canonicalPath
106
+ hash := hmac.New(sha1.New, []byte(auth.SecretKey))
107
+ hash.Write([]byte(payload))
108
+ signature := make([]byte, b64.EncodedLen(hash.Size()))
109
+ b64.Encode(signature, hash.Sum(nil))
110
+
111
+ if expires {
112
+ params["Signature"] = []string{string(signature)}
113
+ } else {
114
+ headers["Authorization"] = []string{"AWS " + auth.AccessKey + ":" + string(signature)}
115
+ }
116
+ if debug {
117
+ log.Printf("Signature payload: %q", payload)
118
+ log.Printf("Signature: %q", signature)
119
+ }
120
+}
Godeps/_workspace/src/github.com/crowdmob/goamz/s3/sign_test.go
new
+148
@@ -0,0 +1,148 @@
1
+package s3_test
2
+
3
+import (
4
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/aws"
5
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/s3"
6
+ "gopkg.in/check.v1"
7
+)
8
+
9
+// S3 ReST authentication docs: http://goo.gl/G1LrK
10
+
11
+var testAuth = aws.Auth{AccessKey: "0PN5J17HBGZHT7JJ3X82", SecretKey: "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o"}
12
+
13
+func (s *S) TestSignExampleObjectGet(c *check.C) {
14
+ method := "GET"
15
+ path := "/johnsmith/photos/puppy.jpg"
16
+ headers := map[string][]string{
17
+ "Host": {"johnsmith.s3.amazonaws.com"},
18
+ "Date": {"Tue, 27 Mar 2007 19:36:42 +0000"},
19
+ }
20
+ s3.Sign(testAuth, method, path, nil, headers)
21
+ expected := "AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA="
22
+ c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
23
+}
24
+
25
+func (s *S) TestSignExampleObjectPut(c *check.C) {
26
+ method := "PUT"
27
+ path := "/johnsmith/photos/puppy.jpg"
28
+ headers := map[string][]string{
29
+ "Host": {"johnsmith.s3.amazonaws.com"},
30
+ "Date": {"Tue, 27 Mar 2007 21:15:45 +0000"},
31
+ "Content-Type": {"image/jpeg"},
32
+ "Content-Length": {"94328"},
33
+ }
34
+ s3.Sign(testAuth, method, path, nil, headers)
35
+ expected := "AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ="
36
+ c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
37
+}
38
+
39
+func (s *S) TestSignExampleList(c *check.C) {
40
+ method := "GET"
41
+ path := "/johnsmith/"
42
+ params := map[string][]string{
43
+ "prefix": {"photos"},
44
+ "max-keys": {"50"},
45
+ "marker": {"puppy"},
46
+ }
47
+ headers := map[string][]string{
48
+ "Host": {"johnsmith.s3.amazonaws.com"},
49
+ "Date": {"Tue, 27 Mar 2007 19:42:41 +0000"},
50
+ "User-Agent": {"Mozilla/5.0"},
51
+ }
52
+ s3.Sign(testAuth, method, path, params, headers)
53
+ expected := "AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4="
54
+ c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
55
+}
56
+
57
+func (s *S) TestSignExampleFetch(c *check.C) {
58
+ method := "GET"
59
+ path := "/johnsmith/"
60
+ params := map[string][]string{
61
+ "acl": {""},
62
+ }
63
+ headers := map[string][]string{
64
+ "Host": {"johnsmith.s3.amazonaws.com"},
65
+ "Date": {"Tue, 27 Mar 2007 19:44:46 +0000"},
66
+ }
67
+ s3.Sign(testAuth, method, path, params, headers)
68
+ expected := "AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g="
69
+ c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
70
+}
71
+
72
+func (s *S) TestSignExampleDelete(c *check.C) {
73
+ method := "DELETE"
74
+ path := "/johnsmith/photos/puppy.jpg"
75
+ params := map[string][]string{}
76
+ headers := map[string][]string{
77
+ "Host": {"s3.amazonaws.com"},
78
+ "Date": {"Tue, 27 Mar 2007 21:20:27 +0000"},
79
+ "User-Agent": {"dotnet"},
80
+ "x-amz-date": {"Tue, 27 Mar 2007 21:20:26 +0000"},
81
+ }
82
+ s3.Sign(testAuth, method, path, params, headers)
83
+ expected := "AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk="
84
+ c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
85
+}
86
+
87
+func (s *S) TestSignExampleUpload(c *check.C) {
88
+ method := "PUT"
89
+ path := "/static.johnsmith.net/db-backup.dat.gz"
90
+ params := map[string][]string{}
91
+ headers := map[string][]string{
92
+ "Host": {"static.johnsmith.net:8080"},
93
+ "Date": {"Tue, 27 Mar 2007 21:06:08 +0000"},
94
+ "User-Agent": {"curl/7.15.5"},
95
+ "x-amz-acl": {"public-read"},
96
+ "content-type": {"application/x-download"},
97
+ "Content-MD5": {"4gJE4saaMU4BqNR0kLY+lw=="},
98
+ "X-Amz-Meta-ReviewedBy": {"joe@johnsmith.net,jane@johnsmith.net"},
99
+ "X-Amz-Meta-FileChecksum": {"0x02661779"},
100
+ "X-Amz-Meta-ChecksumAlgorithm": {"crc32"},
101
+ "Content-Disposition": {"attachment; filename=database.dat"},
102
+ "Content-Encoding": {"gzip"},
103
+ "Content-Length": {"5913339"},
104
+ }
105
+ s3.Sign(testAuth, method, path, params, headers)
106
+ expected := "AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI="
107
+ c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
108
+}
109
+
110
+func (s *S) TestSignExampleListAllMyBuckets(c *check.C) {
111
+ method := "GET"
112
+ path := "/"
113
+ headers := map[string][]string{
114
+ "Host": {"s3.amazonaws.com"},
115
+ "Date": {"Wed, 28 Mar 2007 01:29:59 +0000"},
116
+ }
117
+ s3.Sign(testAuth, method, path, nil, headers)
118
+ expected := "AWS 0PN5J17HBGZHT7JJ3X82:Db+gepJSUbZKwpx1FR0DLtEYoZA="
119
+ c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
120
+}
121
+
122
+func (s *S) TestSignExampleUnicodeKeys(c *check.C) {
123
+ method := "GET"
124
+ path := "/dictionary/fran%C3%A7ais/pr%c3%a9f%c3%a8re"
125
+ headers := map[string][]string{
126
+ "Host": {"s3.amazonaws.com"},
127
+ "Date": {"Wed, 28 Mar 2007 01:49:49 +0000"},
128
+ }
129
+ s3.Sign(testAuth, method, path, nil, headers)
130
+ expected := "AWS 0PN5J17HBGZHT7JJ3X82:dxhSBHoI6eVSPcXJqEghlUzZMnY="
131
+ c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
132
+}
133
+
134
+func (s *S) TestSignExampleCustomSSE(c *check.C) {
135
+ method := "GET"
136
+ path := "/secret/config"
137
+ params := map[string][]string{}
138
+ headers := map[string][]string{
139
+ "Host": {"secret.johnsmith.net:8080"},
140
+ "Date": {"Tue, 27 Mar 2007 21:06:08 +0000"},
141
+ "x-amz-server-side-encryption-customer-key": {"MWJhakVna1dQT1B0SDFMeGtVVnRQRTFGaU1ldFJrU0I="},
142
+ "x-amz-server-side-encryption-customer-key-MD5": {"glIqxpqQ4a9aoK/iLttKzQ=="},
143
+ "x-amz-server-side-encryption-customer-algorithm": {"AES256"},
144
+ }
145
+ s3.Sign(testAuth, method, path, params, headers)
146
+ expected := "AWS 0PN5J17HBGZHT7JJ3X82:Xq6PWmIo0aOWq+LDjCEiCGgbmHE="
147
+ c.Assert(headers["Authorization"], check.DeepEquals, []string{expected})
148
+}