LBRYCommunity/lbrycrd
1
0
Fork 0
Code Issues 29 Pull requests 1 Projects Releases 32 Packages Wiki Activity

src/serialize.h: base serialization level endianness neutrality

Browse source 
Serialization type-safety and endianness compatibility.
This commit is contained in:
Wladimir J. van der Laan 2014-12-19 11:41:50 +01:00
parent 4e853aa163
commit 01f9c3449a
2 changed files with 147 additions and 74 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

197
src/serialize.h
View file

@ -18,6 +18,8 @@
#include <utility> #include <utility>
#include <vector> #include <vector>
#include "compat/endian.h"
class CScript; class CScript;
static const unsigned int MAX_SIZE = 0x02000000; static const unsigned int MAX_SIZE = 0x02000000;
@ -71,6 +73,79 @@ inline const T* end_ptr(const std::vector<T,TAl>& v)
return v.empty() ? NULL : (&v[0] + v.size()); return v.empty() ? NULL : (&v[0] + v.size());
} }
/*
* Lowest-level serialization and conversion.
* @note Sizes of these types are verified in the tests
*/
template<typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj)
{
s.write((char*)&obj, 1);
}
template<typename Stream> inline void ser_writedata16(Stream &s, uint16_t obj)
{
obj = htole16(obj);
s.write((char*)&obj, 2);
}
template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj)
{
obj = htole32(obj);
s.write((char*)&obj, 4);
}
template<typename Stream> inline void ser_writedata64(Stream &s, uint64_t obj)
{
obj = htole64(obj);
s.write((char*)&obj, 8);
}
template<typename Stream> inline uint8_t ser_readdata8(Stream &s)
{
uint8_t obj;
s.read((char*)&obj, 1);
return obj;
}
template<typename Stream> inline uint16_t ser_readdata16(Stream &s)
{
uint16_t obj;
s.read((char*)&obj, 2);
return le16toh(obj);
}
template<typename Stream> inline uint32_t ser_readdata32(Stream &s)
{
uint32_t obj;
s.read((char*)&obj, 4);
return le32toh(obj);
}
template<typename Stream> inline uint64_t ser_readdata64(Stream &s)
{
uint64_t obj;
s.read((char*)&obj, 8);
return le64toh(obj);
}
inline uint64_t ser_double_to_uint64(double x)
{
union { double x; uint64_t y; } tmp;
tmp.x = x;
return tmp.y;
}
inline uint32_t ser_float_to_uint32(float x)
{
union { float x; uint32_t y; } tmp;
tmp.x = x;
return tmp.y;
}
inline double ser_uint64_to_double(uint64_t y)
{
union { double x; uint64_t y; } tmp;
tmp.y = y;
return tmp.x;
}
inline float ser_uint32_to_float(uint32_t y)
{
union { float x; uint32_t y; } tmp;
tmp.y = y;
return tmp.x;
}
///////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////
// //
// Templates for serializing to anything that looks like a stream, // Templates for serializing to anything that looks like a stream,
@ -108,59 +183,48 @@ enum
SerializationOp(s, CSerActionUnserialize(), nType, nVersion); \ SerializationOp(s, CSerActionUnserialize(), nType, nVersion); \
} }
/* /*
* Basic Types * Basic Types
*/ */
#define WRITEDATA(s, obj) s.write((char*)&(obj), sizeof(obj)) inline unsigned int GetSerializeSize(char a, int, int=0) { return 1; }
#define READDATA(s, obj) s.read((char*)&(obj), sizeof(obj)) inline unsigned int GetSerializeSize(int8_t a, int, int=0) { return 1; }
inline unsigned int GetSerializeSize(uint8_t a, int, int=0) { return 1; }
inline unsigned int GetSerializeSize(int16_t a, int, int=0) { return 2; }
inline unsigned int GetSerializeSize(uint16_t a, int, int=0) { return 2; }
inline unsigned int GetSerializeSize(int32_t a, int, int=0) { return 4; }
inline unsigned int GetSerializeSize(uint32_t a, int, int=0) { return 4; }
inline unsigned int GetSerializeSize(int64_t a, int, int=0) { return 8; }
inline unsigned int GetSerializeSize(uint64_t a, int, int=0) { return 8; }
inline unsigned int GetSerializeSize(float a, int, int=0) { return 4; }
inline unsigned int GetSerializeSize(double a, int, int=0) { return 8; }
inline unsigned int GetSerializeSize(char a, int, int=0) { return sizeof(a); } template<typename Stream> inline void Serialize(Stream& s, char a, int, int=0) { ser_writedata8(s, a); } // TODO Get rid of bare char
inline unsigned int GetSerializeSize(signed char a, int, int=0) { return sizeof(a); } template<typename Stream> inline void Serialize(Stream& s, int8_t a, int, int=0) { ser_writedata8(s, a); }
inline unsigned int GetSerializeSize(unsigned char a, int, int=0) { return sizeof(a); } template<typename Stream> inline void Serialize(Stream& s, uint8_t a, int, int=0) { ser_writedata8(s, a); }
inline unsigned int GetSerializeSize(signed short a, int, int=0) { return sizeof(a); } template<typename Stream> inline void Serialize(Stream& s, int16_t a, int, int=0) { ser_writedata16(s, a); }
inline unsigned int GetSerializeSize(unsigned short a, int, int=0) { return sizeof(a); } template<typename Stream> inline void Serialize(Stream& s, uint16_t a, int, int=0) { ser_writedata16(s, a); }
inline unsigned int GetSerializeSize(signed int a, int, int=0) { return sizeof(a); } template<typename Stream> inline void Serialize(Stream& s, int32_t a, int, int=0) { ser_writedata32(s, a); }
inline unsigned int GetSerializeSize(unsigned int a, int, int=0) { return sizeof(a); } template<typename Stream> inline void Serialize(Stream& s, uint32_t a, int, int=0) { ser_writedata32(s, a); }
inline unsigned int GetSerializeSize(signed long a, int, int=0) { return sizeof(a); } template<typename Stream> inline void Serialize(Stream& s, int64_t a, int, int=0) { ser_writedata64(s, a); }
inline unsigned int GetSerializeSize(unsigned long a, int, int=0) { return sizeof(a); } template<typename Stream> inline void Serialize(Stream& s, uint64_t a, int, int=0) { ser_writedata64(s, a); }
inline unsigned int GetSerializeSize(signed long long a, int, int=0) { return sizeof(a); } template<typename Stream> inline void Serialize(Stream& s, float a, int, int=0) { ser_writedata32(s, ser_float_to_uint32(a)); }
inline unsigned int GetSerializeSize(unsigned long long a, int, int=0) { return sizeof(a); } template<typename Stream> inline void Serialize(Stream& s, double a, int, int=0) { ser_writedata64(s, ser_double_to_uint64(a)); }
inline unsigned int GetSerializeSize(float a, int, int=0) { return sizeof(a); }
inline unsigned int GetSerializeSize(double a, int, int=0) { return sizeof(a); }
template<typename Stream> inline void Serialize(Stream& s, char a, int, int=0) { WRITEDATA(s, a); } template<typename Stream> inline void Unserialize(Stream& s, char& a, int, int=0) { a = ser_readdata8(s); } // TODO Get rid of bare char
template<typename Stream> inline void Serialize(Stream& s, signed char a, int, int=0) { WRITEDATA(s, a); } template<typename Stream> inline void Unserialize(Stream& s, int8_t& a, int, int=0) { a = ser_readdata8(s); }
template<typename Stream> inline void Serialize(Stream& s, unsigned char a, int, int=0) { WRITEDATA(s, a); } template<typename Stream> inline void Unserialize(Stream& s, uint8_t& a, int, int=0) { a = ser_readdata8(s); }
template<typename Stream> inline void Serialize(Stream& s, signed short a, int, int=0) { WRITEDATA(s, a); } template<typename Stream> inline void Unserialize(Stream& s, int16_t& a, int, int=0) { a = ser_readdata16(s); }
template<typename Stream> inline void Serialize(Stream& s, unsigned short a, int, int=0) { WRITEDATA(s, a); } template<typename Stream> inline void Unserialize(Stream& s, uint16_t& a, int, int=0) { a = ser_readdata16(s); }
template<typename Stream> inline void Serialize(Stream& s, signed int a, int, int=0) { WRITEDATA(s, a); } template<typename Stream> inline void Unserialize(Stream& s, int32_t& a, int, int=0) { a = ser_readdata32(s); }
template<typename Stream> inline void Serialize(Stream& s, unsigned int a, int, int=0) { WRITEDATA(s, a); } template<typename Stream> inline void Unserialize(Stream& s, uint32_t& a, int, int=0) { a = ser_readdata32(s); }
template<typename Stream> inline void Serialize(Stream& s, signed long a, int, int=0) { WRITEDATA(s, a); } template<typename Stream> inline void Unserialize(Stream& s, int64_t& a, int, int=0) { a = ser_readdata64(s); }
template<typename Stream> inline void Serialize(Stream& s, unsigned long a, int, int=0) { WRITEDATA(s, a); } template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a, int, int=0) { a = ser_readdata64(s); }
template<typename Stream> inline void Serialize(Stream& s, signed long long a, int, int=0) { WRITEDATA(s, a); } template<typename Stream> inline void Unserialize(Stream& s, float& a, int, int=0) { a = ser_uint32_to_float(ser_readdata32(s)); }
template<typename Stream> inline void Serialize(Stream& s, unsigned long long a, int, int=0) { WRITEDATA(s, a); } template<typename Stream> inline void Unserialize(Stream& s, double& a, int, int=0) { a = ser_uint64_to_double(ser_readdata64(s)); }
template<typename Stream> inline void Serialize(Stream& s, float a, int, int=0) { WRITEDATA(s, a); }
template<typename Stream> inline void Serialize(Stream& s, double a, int, int=0) { WRITEDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, char& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, signed char& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, unsigned char& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, signed short& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, unsigned short& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, signed int& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, unsigned int& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, signed long& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, unsigned long& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, signed long long& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, unsigned long long& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, float& a, int, int=0) { READDATA(s, a); }
template<typename Stream> inline void Unserialize(Stream& s, double& a, int, int=0) { READDATA(s, a); }
inline unsigned int GetSerializeSize(bool a, int, int=0) { return sizeof(char); } inline unsigned int GetSerializeSize(bool a, int, int=0) { return sizeof(char); }
template<typename Stream> inline void Serialize(Stream& s, bool a, int, int=0) { char f=a; WRITEDATA(s, f); } template<typename Stream> inline void Serialize(Stream& s, bool a, int, int=0) { char f=a; ser_writedata8(s, f); }
template<typename Stream> inline void Unserialize(Stream& s, bool& a, int, int=0) { char f; READDATA(s, f); a=f; } template<typename Stream> inline void Unserialize(Stream& s, bool& a, int, int=0) { char f=ser_readdata8(s); a=f; }
@ -187,29 +251,22 @@ void WriteCompactSize(Stream& os, uint64_t nSize)
{ {
if (nSize < 253) if (nSize < 253)
{ {
unsigned char chSize = nSize; ser_writedata8(os, nSize);
WRITEDATA(os, chSize);
} }
else if (nSize <= std::numeric_limits<unsigned short>::max()) else if (nSize <= std::numeric_limits<unsigned short>::max())
{ {
unsigned char chSize = 253; ser_writedata8(os, 253);
unsigned short xSize = nSize; ser_writedata16(os, nSize);
WRITEDATA(os, chSize);
WRITEDATA(os, xSize);
} }
else if (nSize <= std::numeric_limits<unsigned int>::max()) else if (nSize <= std::numeric_limits<unsigned int>::max())
{ {
unsigned char chSize = 254; ser_writedata8(os, 254);
unsigned int xSize = nSize; ser_writedata32(os, nSize);
WRITEDATA(os, chSize);
WRITEDATA(os, xSize);
} }
else else
{ {
unsigned char chSize = 255; ser_writedata8(os, 255);
uint64_t xSize = nSize; ser_writedata64(os, nSize);
WRITEDATA(os, chSize);
WRITEDATA(os, xSize);
} }
return; return;
} }
@ -217,8 +274,7 @@ void WriteCompactSize(Stream& os, uint64_t nSize)
template<typename Stream> template<typename Stream>
uint64_t ReadCompactSize(Stream& is) uint64_t ReadCompactSize(Stream& is)
{ {
unsigned char chSize; uint8_t chSize = ser_readdata8(is);
READDATA(is, chSize);
uint64_t nSizeRet = 0; uint64_t nSizeRet = 0;
if (chSize < 253) if (chSize < 253)
{ {
@ -226,25 +282,19 @@ uint64_t ReadCompactSize(Stream& is)
} }
else if (chSize == 253) else if (chSize == 253)
{ {
unsigned short xSize; nSizeRet = ser_readdata16(is);
READDATA(is, xSize);
nSizeRet = xSize;
if (nSizeRet < 253) if (nSizeRet < 253)
throw std::ios_base::failure("non-canonical ReadCompactSize()"); throw std::ios_base::failure("non-canonical ReadCompactSize()");
} }
else if (chSize == 254) else if (chSize == 254)
{ {
unsigned int xSize; nSizeRet = ser_readdata32(is);
READDATA(is, xSize);
nSizeRet = xSize;
if (nSizeRet < 0x10000u) if (nSizeRet < 0x10000u)
throw std::ios_base::failure("non-canonical ReadCompactSize()"); throw std::ios_base::failure("non-canonical ReadCompactSize()");
} }
else else
{ {
uint64_t xSize; nSizeRet = ser_readdata64(is);
READDATA(is, xSize);
nSizeRet = xSize;
if (nSizeRet < 0x100000000ULL) if (nSizeRet < 0x100000000ULL)
throw std::ios_base::failure("non-canonical ReadCompactSize()"); throw std::ios_base::failure("non-canonical ReadCompactSize()");
} }
@ -303,7 +353,7 @@ void WriteVarInt(Stream& os, I n)
len++; len++;
} }
do { do {
WRITEDATA(os, tmp[len]); ser_writedata8(os, tmp[len]);
} while(len--); } while(len--);
} }
@ -312,8 +362,7 @@ I ReadVarInt(Stream& is)
{ {
I n = 0; I n = 0;
while(true) { while(true) {
unsigned char chData; unsigned char chData = ser_readdata8(is);
READDATA(is, chData);
n = (n << 7) | (chData & 0x7F); n = (n << 7) | (chData & 0x7F);
if (chData & 0x80) if (chData & 0x80)
n++; n++;

24
src/test/serialize_tests.cpp
View file

@ -13,6 +13,30 @@ using namespace std;
BOOST_AUTO_TEST_SUITE(serialize_tests) BOOST_AUTO_TEST_SUITE(serialize_tests)
BOOST_AUTO_TEST_CASE(sizes)
{
BOOST_CHECK_EQUAL(sizeof(char), GetSerializeSize(char(0), 0));
BOOST_CHECK_EQUAL(sizeof(int8_t), GetSerializeSize(int8_t(0), 0));
BOOST_CHECK_EQUAL(sizeof(uint8_t), GetSerializeSize(uint8_t(0), 0));
BOOST_CHECK_EQUAL(sizeof(int16_t), GetSerializeSize(int16_t(0), 0));
BOOST_CHECK_EQUAL(sizeof(uint16_t), GetSerializeSize(uint16_t(0), 0));
BOOST_CHECK_EQUAL(sizeof(int32_t), GetSerializeSize(int32_t(0), 0));
BOOST_CHECK_EQUAL(sizeof(uint32_t), GetSerializeSize(uint32_t(0), 0));
BOOST_CHECK_EQUAL(sizeof(int64_t), GetSerializeSize(int64_t(0), 0));
BOOST_CHECK_EQUAL(sizeof(uint64_t), GetSerializeSize(uint64_t(0), 0));
BOOST_CHECK_EQUAL(sizeof(float), GetSerializeSize(float(0), 0));
BOOST_CHECK_EQUAL(sizeof(double), GetSerializeSize(double(0), 0));
// Bool is serialized as char
BOOST_CHECK_EQUAL(sizeof(char), GetSerializeSize(bool(0), 0));
}
BOOST_AUTO_TEST_CASE(floats)
{
// TODO ser_uint32_to_float, ser_uint64_to_double
// TODO ser_float_to_uint32, ser_double_to_uint64
}
BOOST_AUTO_TEST_CASE(varints) BOOST_AUTO_TEST_CASE(varints)
{ {
// encode // encode