c++amazon-s3multipartaws-sdk-cpp

Multipart Upload S3 using AWS C++ SDK


I am trying to upload a file to S3 using multipart upload feature in AWS C++ SDK. I could find examples for JAVA, .NET, PHP, RUBY and Rest API, but didnt find any lead on how to do it in C++. Can you please provide me a direction to achieve the same.


Solution

  • transfer manager needs the file to be stored on disk and have a filename. That's not good for streaming. Here's the code template that I used to do prototyping.

    #include <aws/core/Aws.h>
    #include <aws/s3/S3Client.h>
    #include <aws/s3/model/CreateMultipartUploadRequest.h>
    #include <aws/core/utils/HashingUtils.h>
    #include <aws/s3/model/CompleteMultipartUploadRequest.h>
    #include <aws/s3/model/GetObjectRequest.h>
    #include <aws/s3/model/UploadPartRequest.h>
    
    #include <sstream>
    #include <iostream>
    #include <string>
    
    using namespace Aws::S3::Model;
    
    int main(int argc, char *argv[]) {
        // new api
        Aws::SDKOptions options;
        Aws::InitAPI(options);
    
        // use default credential provider chains
        Aws::Client::ClientConfiguration clientConfiguration;
        clientConfiguration.region = "<your-region>";
        clientConfiguration.endpointOverride = "<endpoint-override>";
        Aws::S3::S3Client s3_client(clientConfiguration);
    
        std::string bucket = "<bucket>";
        std::string key = "<key>";
    
        // initiate the process
        Aws::S3::Model::CreateMultipartUploadRequest create_request;
        create_request.SetBucket(bucket.c_str());
        create_request.SetKey(key.c_str());
        create_request.SetContentType("text/plain");
    
        auto createMultipartUploadOutcome =
          s3_client.CreateMultipartUpload(create_request);
    
        std::string upload_id = createMultipartUploadOutcome.GetResult().GetUploadId();
        std::cout << "multiplarts upload id is:" << upload_id << std::endl;
    
        // start upload
        Aws::S3::Model::UploadPartRequest my_request;
        my_request.SetBucket(bucket.c_str());
        my_request.SetKey(key.c_str());
        my_request.SetPartNumber(1);
        my_request.SetUploadId(upload_id.c_str());
    
        Aws::StringStream ss;
        // just have a small chunk of data to verify everything works
        ss << "to upload";
    
        std::shared_ptr<Aws::StringStream> stream_ptr =
          Aws::MakeShared<Aws::StringStream>("WriteStream::Upload" /* log id */, ss.str());
    
        my_request.SetBody(stream_ptr);
    
        Aws::Utils::ByteBuffer part_md5(
            Aws::Utils::HashingUtils::CalculateMD5(*stream_ptr));
        my_request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5));
    
        auto start_pos = stream_ptr->tellg();
        stream_ptr->seekg(0LL, stream_ptr->end);
        my_request.SetContentLength(static_cast<long>(stream_ptr->tellg()));
        stream_ptr->seekg(start_pos);
    
        auto uploadPartOutcomeCallable1 = s3_client.UploadPartCallable(my_request);
    
        // finish upload
        Aws::S3::Model::CompleteMultipartUploadRequest completeMultipartUploadRequest;
        completeMultipartUploadRequest.SetBucket(bucket.c_str());
        completeMultipartUploadRequest.SetKey(key.c_str());
        completeMultipartUploadRequest.SetUploadId(upload_id.c_str());
    
        UploadPartOutcome uploadPartOutcome1 = uploadPartOutcomeCallable1.get();
        CompletedPart completedPart1;
        completedPart1.SetPartNumber(1);
        auto etag = uploadPartOutcome1.GetResult().GetETag();
        // if etag must not be empty
        assert(etag.empty());
        completedPart1.SetETag(etag);
    
        completeMultipartUploadRequest.SetBucket(bucket.c_str());
        completeMultipartUploadRequest.SetKey(key.c_str());
        completeMultipartUploadRequest.SetUploadId(upload_id.c_str());
    
        CompletedMultipartUpload completedMultipartUpload;
            completedMultipartUpload.AddParts(completedPart1);
    
        completeMultipartUploadRequest.WithMultipartUpload(completedMultipartUpload);
    
        auto completeMultipartUploadOutcome =
          s3_client.CompleteMultipartUpload(completeMultipartUploadRequest);
    
        if (!completeMultipartUploadOutcome.IsSuccess()) {
          auto error = completeMultipartUploadOutcome.GetError();
          std::stringstream ss;
          ss << error << error.GetExceptionName() << ": " << error.GetMessage() << std::endl;
          return -1;
        }
    }