c++protocol-buffers

Protobuf with GZIP compression


I am having a problem in trying to serialize an array of unsigned char into file with GZIP compression using protobuf while playing with the library.

I think the problem might have to do with some of my syntax or misuse of API. I have also tried std::fstream. FYI, Windows 8.1 & VS2013 is the building environment.

scene.proto

syntax = "proto3";

package Recipe;

message Scene
{
    repeated int32 imageData = 1 [packed=true];
}

source.cpp

#include <iostream>
#include <fstream>
#include <ostream>
#include <istream>
#include <string>
#include <cstdint>
#include "Scene.pb.h"
#include <google\protobuf\io\zero_copy_stream_impl.h>
#include <google\protobuf\io\gzip_stream.h>

int const _MIN = 0;
int const _MAX = 255;
unsigned int const _SIZE = 65200000;
unsigned int const _COMPRESSION_LEVEL = 10;

void randWithinUnsignedCharSize(uint8_t * buffer, unsigned int size)
{
    for (size_t i = 0; i < size; ++i)
    {
        buffer[i] = _MIN + (rand() % static_cast<int>(_MAX - _MIN + 1));
    }
}

using namespace google::protobuf::io;

int main()
{
    GOOGLE_PROTOBUF_VERIFY_VERSION;

    Recipe::Scene * scene = new Recipe::Scene();
    uint8_t * imageData = new uint8_t[_SIZE];
    randWithinUnsignedCharSize(imageData, _SIZE);

    for (size_t i = 0; i < _SIZE; i++)
    {
        scene->add_imagedata(imageData[i]);
    }
    std::cout << "scene->imagedata_size() " << scene->imagedata_size() << std::endl;
    {
        std::ofstream output("scene.art", std::ofstream::out | std::ofstream::trunc | std::ofstream::binary);
        OstreamOutputStream outputFileStream(&output);
        GzipOutputStream::Options options;
        options.format = GzipOutputStream::GZIP;
        options.compression_level = _COMPRESSION_LEVEL;

        GzipOutputStream gzipOutputStream(&outputFileStream, options);

        if (!scene->SerializeToZeroCopyStream(&gzipOutputStream)) {
            std::cerr << "Failed to write scene." << std::endl;
            return -1;
        }
    }
    Recipe::Scene * scene1 = new Recipe::Scene();
    {
        std::ifstream input("scene.art", std::ifstream::in | std::ifstream::binary);
        IstreamInputStream inputFileStream(&input);
        GzipInputStream gzipInputStream(&inputFileStream);

        if (!scene1->ParseFromZeroCopyStream(&gzipInputStream)) {
            std::cerr << "Failed to parse scene." << std::endl;
            return -1;
        }
    }
    std::cout << "scene1->imagedata_size() " << scene1->imagedata_size() <<std::endl;
    google::protobuf::ShutdownProtobufLibrary();
    return 0;
}

Solution

  • You seem to have a typo in your code. Compression level is according to documentation in range 0-9. You set incorrectly compression level to 10.

    Your example is working for me when corrected to:

    unsigned int const _COMPRESSION_LEVEL = 9;