Hello, please help with the following problem.
I use CodedOutputStream to write a message's length and message itself to a
file.
When I later read from generated file, it turns out that file contains less
data.
I provide source code for simple test: write 1000 messages to a file and
then read this file.
Output is the following:
[aagenosov@black build]$ bin/test_coded_stream
written 1000 records, total bytes: 16384
test successfull
total length: 13890, consumed bytes: 13890, messages count: 594
I tried to forcedly flush buffers, but there was no effect.
--
You received this message because you are subscribed to the Google Groups
"Protocol Buffers" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to protobuf+unsubscr...@googlegroups.com.
To post to this group, send email to protobuf@googlegroups.com.
Visit this group at http://groups.google.com/group/protobuf.
For more options, visit https://groups.google.com/d/optout.
#include <iostream>
#include <sstream>
#include <fstream>
#include <string>
#include <memory>
#include <fcntl.h>
#include <errno.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>
static const uint32_t records_limit = 1000;
int main()
{
using namespace google::protobuf::io;
const char * file_path = "/tmp/coded_stream_test.log";
{
int file_handle = open(file_path,
O_WRONLY | O_CREAT,
S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH
);
if (file_handle == -1)
{
std::cerr << "can't create file " << file_path << ", errno: " << errno << std::endl;
return -1;
}
std::auto_ptr<FileOutputStream> file_stream;
std::auto_ptr<CodedOutputStream> coded_output;
file_stream.reset(new FileOutputStream(file_handle));
file_stream->SetCloseOnDelete(true);
coded_output.reset(new CodedOutputStream(file_stream.get()));
uint32_t idx = 0;
for (; idx < records_limit; ++idx)
{
std::ostringstream data;
data << "iteration " << idx;
const std::string & str = data.str();
int coded_size = CodedOutputStream::VarintSize32(str.length()) + str.length();
uint8_t * buffer = coded_output->GetDirectBufferForNBytesAndAdvance(coded_size);
if (buffer != NULL)
{
buffer = CodedOutputStream::WriteVarint32ToArray(str.length(), buffer);
buffer = CodedOutputStream::WriteRawToArray(str.c_str(), str.length(), buffer);
}
else
{
coded_output->WriteVarint32(str.length());
coded_output->WriteString(str);
}
if (coded_output->HadError())
{
std::cerr << "error on iteration " << idx << std::endl;
break;
}
}
/*coded_output.reset();
if (!file_stream->Close())
{
std::cerr << "error " << file_stream->GetErrno() << " on close file" << std::endl;
}*/
std::cout << "written " << idx << " records, total bytes: " << file_stream->ByteCount() << std::endl;
}
std::fstream input_file(file_path, std::ios_base::in | std::ios_base::binary);
input_file.seekg(0, input_file.end);
size_t total_length = input_file.tellg();
input_file.seekg(0, input_file.beg);
std::auto_ptr<ZeroCopyInputStream> raw_input(new IstreamInputStream(&input_file));
std::auto_ptr<CodedInputStream> coded_input(new CodedInputStream(raw_input.get()));
uint32_t message_count = 0;
int error = 0;
while (!coded_input->ExpectAtEnd() && static_cast<size_t>(raw_input->ByteCount()) != total_length)
{
uint32_t msg_len;
if (!coded_input->ReadVarint32(&msg_len))
{
std::cerr << "failed to get length of next message" << std::endl;
error = -1;
break;
}
CodedInputStream::Limit limit = coded_input->PushLimit(msg_len);
std::string buf;
if (!coded_input->ReadString(&buf, msg_len))
{
std::cerr << "failed to parse message" << std::endl;
error = -1;
break;
}
message_count++;
//std::cout << buf << std::endl;
coded_input->PopLimit(limit);
}
if (error == 0)
{
std::cout << "test successfull" << std::endl;
}
std::cout << "total length: " << total_length
<< ", consumed bytes: " << raw_input->ByteCount()
<< ", messages count: " << message_count
<< std::endl;
input_file.close();
return error;
}
