I'm trying to use the AWS v2 SDK for Go to list all objects in a given bucket on DigitalOcean Spaces. Their documentation gives examples of how to use the v1 SDK to do this, but my app uses v2. I know I could technically use both, but I'd rather not if possible.
Here's what I've got so far:
package main
import (
"context"
"fmt"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func main() {
customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
URL: "https://sfo2.digitaloceanspaces.com",
}, nil
})
cfg, err := config.LoadDefaultConfig(
context.TODO(),
config.WithRegion("us-east-1"),
config.WithEndpointResolverWithOptions(customResolver),
config.WithCredentialsProvider(aws.AnonymousCredentials{}),
)
if err != nil {
fmt.Println(err)
}
s3Client := s3.NewFromConfig(cfg)
var continuationToken *string
continuationToken = nil
for {
output, err := s3Client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
Bucket: aws.String("stats"),
ContinuationToken: continuationToken},
)
if err != nil {
fmt.Println(err)
}
for _, obj := range output.Contents {
fmt.Println(obj)
}
if output.IsTruncated == false {
break
}
continuationToken = output.ContinuationToken
}
}
This is the error I'm getting:
operation error S3: ListObjectsV2, https response error StatusCode: 400, RequestID: tx0000000000000051339d4-00620701db-2174fe1c-sfo2a, HostID: 2174fe1c-sfo2a-sfo, api error InvalidArgument: UnknownError
The error seems to indicate there's something wrong with my request but I don't know what.
For pagination i think you need to do it via a pagination function like this
// Create the Paginator for the ListObjectsV2 operation.
p := s3.NewListObjectsV2Paginator(client, params, func(o *s3.ListObjectsV2PaginatorOptions) {
if v := int32(maxKeys); v != 0 {
o.Limit = v
}
})
Here's a fully working example I'm using to read from a digital ocean spaces bucket
package s3
import (
"context"
"os"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func read(ctx context.Context) error {
// Define the parameters for the session you want to create.
spacesKey := os.Getenv("SPACES_KEY")
spacesSecret := os.Getenv("SPACES_SECRET")
creds := credentials.NewStaticCredentialsProvider(spacesKey, spacesSecret, "")
customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
URL: "https://sfo3.digitaloceanspaces.com",
}, nil
})
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion("us-east-1"),
config.WithCredentialsProvider(creds),
config.WithEndpointResolverWithOptions(customResolver))
if err != nil {
return err
}
// Create an Amazon S3 service client
awsS3Client := s3.NewFromConfig(cfg)
input := &s3.GetObjectInput{
Bucket: aws.String("zeus-fyi"),
Key: aws.String("test.txt"),
}
downloader := manager.NewDownloader(awsS3Client)
newFile, err := os.Create("./local-test.txt")
if err != nil {
return err
}
defer newFile.Close()
_, err = downloader.Download(ctx, newFile, input)
if err != nil {
return err
}
return err
}