compressZlib: don't use a SharedBuffer but a raw u8 * pointer
Remove usage of the SharedBuffer in zlib compression which has two problems: * We copied the whole memory block to compress it (not good with mapblocks) * We copied sometimes strings to SharedBuffer to SharedBuffer (2nd time) Use this method in MapNode::serializeBulk + optimize serialization but merging 3 identical loops in a single loopmaster
parent
61e4877190
commit
c27504a322
|
@ -658,7 +658,7 @@ void MapNode::serializeBulk(std::ostream &os, int version,
|
||||||
const MapNode *nodes, u32 nodecount,
|
const MapNode *nodes, u32 nodecount,
|
||||||
u8 content_width, u8 params_width, bool compressed)
|
u8 content_width, u8 params_width, bool compressed)
|
||||||
{
|
{
|
||||||
if(!ser_ver_supported(version))
|
if (!ser_ver_supported(version))
|
||||||
throw VersionMismatchException("ERROR: MapNode format not supported");
|
throw VersionMismatchException("ERROR: MapNode format not supported");
|
||||||
|
|
||||||
sanity_check(content_width == 2);
|
sanity_check(content_width == 2);
|
||||||
|
@ -666,38 +666,33 @@ void MapNode::serializeBulk(std::ostream &os, int version,
|
||||||
|
|
||||||
// Can't do this anymore; we have 16-bit dynamically allocated node IDs
|
// Can't do this anymore; we have 16-bit dynamically allocated node IDs
|
||||||
// in memory; conversion just won't work in this direction.
|
// in memory; conversion just won't work in this direction.
|
||||||
if(version < 24)
|
if (version < 24)
|
||||||
throw SerializationError("MapNode::serializeBulk: serialization to "
|
throw SerializationError("MapNode::serializeBulk: serialization to "
|
||||||
"version < 24 not possible");
|
"version < 24 not possible");
|
||||||
|
|
||||||
SharedBuffer<u8> databuf(nodecount * (content_width + params_width));
|
size_t databuf_size = nodecount * (content_width + params_width);
|
||||||
|
u8 *databuf = new u8[databuf_size];
|
||||||
|
|
||||||
|
u32 start1 = content_width * nodecount;
|
||||||
|
u32 start2 = (content_width + 1) * nodecount;
|
||||||
|
|
||||||
// Serialize content
|
// Serialize content
|
||||||
for(u32 i=0; i<nodecount; i++)
|
for (u32 i = 0; i < nodecount; i++) {
|
||||||
writeU16(&databuf[i*2], nodes[i].param0);
|
writeU16(&databuf[i * 2], nodes[i].param0);
|
||||||
|
|
||||||
// Serialize param1
|
|
||||||
u32 start1 = content_width * nodecount;
|
|
||||||
for(u32 i=0; i<nodecount; i++)
|
|
||||||
writeU8(&databuf[start1 + i], nodes[i].param1);
|
writeU8(&databuf[start1 + i], nodes[i].param1);
|
||||||
|
|
||||||
// Serialize param2
|
|
||||||
u32 start2 = (content_width + 1) * nodecount;
|
|
||||||
for(u32 i=0; i<nodecount; i++)
|
|
||||||
writeU8(&databuf[start2 + i], nodes[i].param2);
|
writeU8(&databuf[start2 + i], nodes[i].param2);
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
Compress data to output stream
|
Compress data to output stream
|
||||||
*/
|
*/
|
||||||
|
|
||||||
if(compressed)
|
if (compressed)
|
||||||
{
|
compressZlib(databuf, databuf_size, os);
|
||||||
compressZlib(databuf, os);
|
|
||||||
}
|
|
||||||
else
|
else
|
||||||
{
|
os.write((const char*) &databuf[0], databuf_size);
|
||||||
os.write((const char*) &databuf[0], databuf.getSize());
|
|
||||||
}
|
delete [] databuf;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Deserialize bulk node data
|
// Deserialize bulk node data
|
||||||
|
|
|
@ -27,7 +27,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
|
||||||
|
|
||||||
/* report a zlib or i/o error */
|
/* report a zlib or i/o error */
|
||||||
void zerr(int ret)
|
void zerr(int ret)
|
||||||
{
|
{
|
||||||
dstream<<"zerr: ";
|
dstream<<"zerr: ";
|
||||||
switch (ret) {
|
switch (ret) {
|
||||||
case Z_ERRNO:
|
case Z_ERRNO:
|
||||||
|
@ -53,7 +53,7 @@ void zerr(int ret)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void compressZlib(SharedBuffer<u8> data, std::ostream &os, int level)
|
void compressZlib(const u8 *data, size_t data_size, std::ostream &os, int level)
|
||||||
{
|
{
|
||||||
z_stream z;
|
z_stream z;
|
||||||
const s32 bufsize = 16384;
|
const s32 bufsize = 16384;
|
||||||
|
@ -68,16 +68,16 @@ void compressZlib(SharedBuffer<u8> data, std::ostream &os, int level)
|
||||||
ret = deflateInit(&z, level);
|
ret = deflateInit(&z, level);
|
||||||
if(ret != Z_OK)
|
if(ret != Z_OK)
|
||||||
throw SerializationError("compressZlib: deflateInit failed");
|
throw SerializationError("compressZlib: deflateInit failed");
|
||||||
|
|
||||||
// Point zlib to our input buffer
|
// Point zlib to our input buffer
|
||||||
z.next_in = (Bytef*)&data[0];
|
z.next_in = (Bytef*)&data[0];
|
||||||
z.avail_in = data.getSize();
|
z.avail_in = data_size;
|
||||||
// And get all output
|
// And get all output
|
||||||
for(;;)
|
for(;;)
|
||||||
{
|
{
|
||||||
z.next_out = (Bytef*)output_buffer;
|
z.next_out = (Bytef*)output_buffer;
|
||||||
z.avail_out = bufsize;
|
z.avail_out = bufsize;
|
||||||
|
|
||||||
status = deflate(&z, Z_FINISH);
|
status = deflate(&z, Z_FINISH);
|
||||||
if(status == Z_NEED_DICT || status == Z_DATA_ERROR
|
if(status == Z_NEED_DICT || status == Z_DATA_ERROR
|
||||||
|| status == Z_MEM_ERROR)
|
|| status == Z_MEM_ERROR)
|
||||||
|
@ -98,8 +98,7 @@ void compressZlib(SharedBuffer<u8> data, std::ostream &os, int level)
|
||||||
|
|
||||||
void compressZlib(const std::string &data, std::ostream &os, int level)
|
void compressZlib(const std::string &data, std::ostream &os, int level)
|
||||||
{
|
{
|
||||||
SharedBuffer<u8> databuf((u8*)data.c_str(), data.size());
|
compressZlib((u8*)data.c_str(), data.size(), os, level);
|
||||||
compressZlib(databuf, os, level);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void decompressZlib(std::istream &is, std::ostream &os)
|
void decompressZlib(std::istream &is, std::ostream &os)
|
||||||
|
@ -120,9 +119,9 @@ void decompressZlib(std::istream &is, std::ostream &os)
|
||||||
ret = inflateInit(&z);
|
ret = inflateInit(&z);
|
||||||
if(ret != Z_OK)
|
if(ret != Z_OK)
|
||||||
throw SerializationError("dcompressZlib: inflateInit failed");
|
throw SerializationError("dcompressZlib: inflateInit failed");
|
||||||
|
|
||||||
z.avail_in = 0;
|
z.avail_in = 0;
|
||||||
|
|
||||||
//dstream<<"initial fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;
|
//dstream<<"initial fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;
|
||||||
|
|
||||||
for(;;)
|
for(;;)
|
||||||
|
@ -143,7 +142,7 @@ void decompressZlib(std::istream &is, std::ostream &os)
|
||||||
//dstream<<"z.avail_in == 0"<<std::endl;
|
//dstream<<"z.avail_in == 0"<<std::endl;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
//dstream<<"1 z.avail_in="<<z.avail_in<<std::endl;
|
//dstream<<"1 z.avail_in="<<z.avail_in<<std::endl;
|
||||||
status = inflate(&z, Z_NO_FLUSH);
|
status = inflate(&z, Z_NO_FLUSH);
|
||||||
//dstream<<"2 z.avail_in="<<z.avail_in<<std::endl;
|
//dstream<<"2 z.avail_in="<<z.avail_in<<std::endl;
|
||||||
|
@ -163,7 +162,7 @@ void decompressZlib(std::istream &is, std::ostream &os)
|
||||||
if(status == Z_STREAM_END)
|
if(status == Z_STREAM_END)
|
||||||
{
|
{
|
||||||
//dstream<<"Z_STREAM_END"<<std::endl;
|
//dstream<<"Z_STREAM_END"<<std::endl;
|
||||||
|
|
||||||
//dstream<<"z.avail_in="<<z.avail_in<<std::endl;
|
//dstream<<"z.avail_in="<<z.avail_in<<std::endl;
|
||||||
//dstream<<"fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;
|
//dstream<<"fail="<<is.fail()<<" bad="<<is.bad()<<std::endl;
|
||||||
// Unget all the data that inflate didn't take
|
// Unget all the data that inflate didn't take
|
||||||
|
@ -178,7 +177,7 @@ void decompressZlib(std::istream &is, std::ostream &os)
|
||||||
throw SerializationError("decompressZlib: unget failed");
|
throw SerializationError("decompressZlib: unget failed");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -186,23 +185,23 @@ void decompressZlib(std::istream &is, std::ostream &os)
|
||||||
inflateEnd(&z);
|
inflateEnd(&z);
|
||||||
}
|
}
|
||||||
|
|
||||||
void compress(SharedBuffer<u8> data, std::ostream &os, u8 version)
|
void compress(const SharedBuffer<u8> &data, std::ostream &os, u8 version)
|
||||||
{
|
{
|
||||||
if(version >= 11)
|
if(version >= 11)
|
||||||
{
|
{
|
||||||
compressZlib(data, os);
|
compressZlib(*data ,data.getSize(), os);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if(data.getSize() == 0)
|
if(data.getSize() == 0)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
// Write length (u32)
|
// Write length (u32)
|
||||||
|
|
||||||
u8 tmp[4];
|
u8 tmp[4];
|
||||||
writeU32(tmp, data.getSize());
|
writeU32(tmp, data.getSize());
|
||||||
os.write((char*)tmp, 4);
|
os.write((char*)tmp, 4);
|
||||||
|
|
||||||
// We will be writing 8-bit pairs of more_count and byte
|
// We will be writing 8-bit pairs of more_count and byte
|
||||||
u8 more_count = 0;
|
u8 more_count = 0;
|
||||||
u8 current_byte = data[0];
|
u8 current_byte = data[0];
|
||||||
|
@ -242,7 +241,7 @@ void decompress(std::istream &is, std::ostream &os, u8 version)
|
||||||
u8 tmp[4];
|
u8 tmp[4];
|
||||||
is.read((char*)tmp, 4);
|
is.read((char*)tmp, 4);
|
||||||
u32 len = readU32(tmp);
|
u32 len = readU32(tmp);
|
||||||
|
|
||||||
// We will be reading 8-bit pairs of more_count and byte
|
// We will be reading 8-bit pairs of more_count and byte
|
||||||
u32 count = 0;
|
u32 count = 0;
|
||||||
for(;;)
|
for(;;)
|
||||||
|
@ -251,7 +250,7 @@ void decompress(std::istream &is, std::ostream &os, u8 version)
|
||||||
u8 byte=0;
|
u8 byte=0;
|
||||||
|
|
||||||
is.read((char*)&more_count, 1);
|
is.read((char*)&more_count, 1);
|
||||||
|
|
||||||
is.read((char*)&byte, 1);
|
is.read((char*)&byte, 1);
|
||||||
|
|
||||||
if(is.eof())
|
if(is.eof())
|
||||||
|
|
|
@ -86,12 +86,12 @@ inline bool ser_ver_supported(s32 v) {
|
||||||
Misc. serialization functions
|
Misc. serialization functions
|
||||||
*/
|
*/
|
||||||
|
|
||||||
void compressZlib(SharedBuffer<u8> data, std::ostream &os, int level = -1);
|
void compressZlib(const u8 *data, size_t data_size, std::ostream &os, int level = -1);
|
||||||
void compressZlib(const std::string &data, std::ostream &os, int level = -1);
|
void compressZlib(const std::string &data, std::ostream &os, int level = -1);
|
||||||
void decompressZlib(std::istream &is, std::ostream &os);
|
void decompressZlib(std::istream &is, std::ostream &os);
|
||||||
|
|
||||||
// These choose between zlib and a self-made one according to version
|
// These choose between zlib and a self-made one according to version
|
||||||
void compress(SharedBuffer<u8> data, std::ostream &os, u8 version);
|
void compress(const SharedBuffer<u8> &data, std::ostream &os, u8 version);
|
||||||
//void compress(const std::string &data, std::ostream &os, u8 version);
|
//void compress(const std::string &data, std::ostream &os, u8 version);
|
||||||
void decompress(std::istream &is, std::ostream &os, u8 version);
|
void decompress(std::istream &is, std::ostream &os, u8 version);
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue