goamazon-s3aws-sdk-godigital-ocean-spaces

How can I use the AWS SDK v2 for Go with DigitalOcean Spaces?


I'm trying to use the AWS v2 SDK for Go to list all objects in a given bucket on DigitalOcean Spaces. Their documentation gives examples of how to use the v1 SDK to do this, but my app uses v2. I know I could technically use both, but I'd rather not if possible.

Here's what I've got so far:

package main

import (
    "context"
    "fmt"
    "github.com/aws/aws-sdk-go-v2/aws"
    "github.com/aws/aws-sdk-go-v2/config"
    "github.com/aws/aws-sdk-go-v2/service/s3"
)

func main() {

    customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
        return aws.Endpoint{
            URL: "https://sfo2.digitaloceanspaces.com",
        }, nil
    })
    cfg, err := config.LoadDefaultConfig(
        context.TODO(),
        config.WithRegion("us-east-1"),
        config.WithEndpointResolverWithOptions(customResolver),
        config.WithCredentialsProvider(aws.AnonymousCredentials{}),
    )
    if err != nil {
        fmt.Println(err)
    }

    s3Client := s3.NewFromConfig(cfg)

    var continuationToken *string
    continuationToken = nil

    for {
        output, err := s3Client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
            Bucket:            aws.String("stats"),
            ContinuationToken: continuationToken},
        )
        if err != nil {
            fmt.Println(err)
        }

        for _, obj := range output.Contents {
            fmt.Println(obj)
        }

        if output.IsTruncated == false {
            break
        }

        continuationToken = output.ContinuationToken
    }
}

This is the error I'm getting:

operation error S3: ListObjectsV2, https response error StatusCode: 400, RequestID: tx0000000000000051339d4-00620701db-2174fe1c-sfo2a, HostID: 2174fe1c-sfo2a-sfo, api error InvalidArgument: UnknownError

The error seems to indicate there's something wrong with my request but I don't know what.


Solution

  • For pagination i think you need to do it via a pagination function like this

        // Create the Paginator for the ListObjectsV2 operation.
        p := s3.NewListObjectsV2Paginator(client, params, func(o *s3.ListObjectsV2PaginatorOptions) {
            if v := int32(maxKeys); v != 0 {
                o.Limit = v
            }
        })
    

    Here's a fully working example I'm using to read from a digital ocean spaces bucket

    package s3
    
    import (
        "context"
        "os"
    
        "github.com/aws/aws-sdk-go-v2/aws"
        "github.com/aws/aws-sdk-go-v2/config"
        "github.com/aws/aws-sdk-go-v2/credentials"
        "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
        "github.com/aws/aws-sdk-go-v2/service/s3"
    )
    
    func read(ctx context.Context) error {
        // Define the parameters for the session you want to create.
    
        spacesKey := os.Getenv("SPACES_KEY")
        spacesSecret := os.Getenv("SPACES_SECRET")
    
        creds := credentials.NewStaticCredentialsProvider(spacesKey, spacesSecret, "")
    
        customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
            return aws.Endpoint{
                URL: "https://sfo3.digitaloceanspaces.com",
            }, nil
        })
        cfg, err := config.LoadDefaultConfig(ctx,
            config.WithRegion("us-east-1"),
            config.WithCredentialsProvider(creds),
            config.WithEndpointResolverWithOptions(customResolver))
        if err != nil {
            return err
        }
        // Create an Amazon S3 service client
        awsS3Client := s3.NewFromConfig(cfg)
        input := &s3.GetObjectInput{
            Bucket: aws.String("zeus-fyi"),
            Key:    aws.String("test.txt"),
        }
        downloader := manager.NewDownloader(awsS3Client)
        newFile, err := os.Create("./local-test.txt")
        if err != nil {
            return err
        }
        defer newFile.Close()
        _, err = downloader.Download(ctx, newFile, input)
        if err != nil {
            return err
        }
        return err
    }