diff options
| -rw-r--r-- | docs/reference.rst | 2 | ||||
| -rw-r--r-- | examples/using_double_on_avr/Makefile | 24 | ||||
| -rw-r--r-- | examples/using_double_on_avr/README.txt | 25 | ||||
| -rw-r--r-- | examples/using_double_on_avr/decode_double.c | 33 | ||||
| -rw-r--r-- | examples/using_double_on_avr/double_conversion.c | 123 | ||||
| -rw-r--r-- | examples/using_double_on_avr/double_conversion.h | 26 | ||||
| -rw-r--r-- | examples/using_double_on_avr/doubleproto.proto | 15 | ||||
| -rw-r--r-- | examples/using_double_on_avr/encode_double.c | 25 | ||||
| -rw-r--r-- | examples/using_double_on_avr/test_conversions.c | 56 | ||||
| -rw-r--r-- | pb.h | 4 | ||||
| -rw-r--r-- | pb_decode.c | 76 | ||||
| -rw-r--r-- | pb_decode.h | 5 | ||||
| -rw-r--r-- | pb_encode.c | 56 | ||||
| -rw-r--r-- | pb_encode.h | 6 | ||||
| -rw-r--r-- | tests/float_double_conversion/SConscript | 18 | ||||
| -rw-r--r-- | tests/float_double_conversion/doublemsg.proto | 5 | ||||
| -rw-r--r-- | tests/float_double_conversion/float_double_conversion.c | 79 |
17 files changed, 251 insertions, 327 deletions
diff --git a/docs/reference.rst b/docs/reference.rst index 89d1f0f..39bc611 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -59,6 +59,8 @@ PB_ENCODE_ARRAYS_UNPACKED Don't encode scalar arrays as packed. This is only to be used when the decoder on the receiving side cannot process packed scalar arrays. Such example is older protobuf.js. +PB_CONVERT_DOUBLE_FLOAT Convert doubles to floats for platforms that do + not support 64-bit doubles. Mainly AVR. ============================ ================================================ The PB_MAX_REQUIRED_FIELDS, PB_FIELD_16BIT and PB_FIELD_32BIT settings allow diff --git a/examples/using_double_on_avr/Makefile b/examples/using_double_on_avr/Makefile deleted file mode 100644 index 874a64b..0000000 --- a/examples/using_double_on_avr/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -# Include the nanopb provided Makefile rules -include ../../extra/nanopb.mk - -# Compiler flags to enable all warnings & debug info -CFLAGS = -Wall -Werror -g -O0 -CFLAGS += -I$(NANOPB_DIR) - -all: run_tests - -.SUFFIXES: - -clean: - rm -f test_conversions encode_double decode_double doubleproto.pb.c doubleproto.pb.h - -test_conversions: test_conversions.c double_conversion.c - $(CC) $(CFLAGS) -o $@ $^ - -%: %.c double_conversion.c doubleproto.pb.c - $(CC) $(CFLAGS) -o $@ $^ $(NANOPB_CORE) - -run_tests: test_conversions encode_double decode_double - ./test_conversions - ./encode_double | ./decode_double - diff --git a/examples/using_double_on_avr/README.txt b/examples/using_double_on_avr/README.txt deleted file mode 100644 index d9fcdfc..0000000 --- a/examples/using_double_on_avr/README.txt +++ /dev/null @@ -1,25 +0,0 @@ -Nanopb example "using_double_on_avr" -==================================== - -Some processors/compilers, such as AVR-GCC, do not support the double -datatype. Instead, they have sizeof(double) == 4. Because protocol -binary format uses the double encoding directly, this causes trouble -if the protocol in .proto requires double fields. - -This directory contains a solution to this problem. It uses uint64_t -to store the raw wire values, because its size is correct on all -platforms. The file double_conversion.c provides functions that -convert these values to/from floats, without relying on compiler -support. - -To use this method, you need to make some modifications to your code: - -1) Change all 'double' fields into 'fixed64' in the .proto. - -2) Whenever writing to a 'double' field, use float_to_double(). - -3) Whenever reading a 'double' field, use double_to_float(). - -The conversion routines are as accurate as the float datatype can -be. Furthermore, they should handle all special values (NaN, inf, denormalized -numbers) correctly. There are testcases in test_conversions.c. diff --git a/examples/using_double_on_avr/decode_double.c b/examples/using_double_on_avr/decode_double.c deleted file mode 100644 index 5802eca..0000000 --- a/examples/using_double_on_avr/decode_double.c +++ /dev/null @@ -1,33 +0,0 @@ -/* Decodes a double value into a float variable. - * Used to read double values with AVR code, which doesn't support double directly. - */ - -#include <stdio.h> -#include <pb_decode.h> -#include "double_conversion.h" -#include "doubleproto.pb.h" - -int main() -{ - uint8_t buffer[32]; - size_t count = fread(buffer, 1, sizeof(buffer), stdin); - pb_istream_t stream = pb_istream_from_buffer(buffer, count); - - AVRDoubleMessage message; - pb_decode(&stream, AVRDoubleMessage_fields, &message); - - float v1 = double_to_float(message.field1); - float v2 = double_to_float(message.field2); - - printf("Values: %f %f\n", v1, v2); - - if (v1 == 1234.5678f && - v2 == 0.00001f) - { - return 0; - } - else - { - return 1; - } -} diff --git a/examples/using_double_on_avr/double_conversion.c b/examples/using_double_on_avr/double_conversion.c deleted file mode 100644 index cf79b9a..0000000 --- a/examples/using_double_on_avr/double_conversion.c +++ /dev/null @@ -1,123 +0,0 @@ -/* Conversion routines for platforms that do not support 'double' directly. */ - -#include "double_conversion.h" -#include <math.h> - -typedef union { - float f; - uint32_t i; -} conversion_t; - -/* Note: IEE 754 standard specifies float formats as follows: - * Single precision: sign, 8-bit exp, 23-bit frac. - * Double precision: sign, 11-bit exp, 52-bit frac. - */ - -uint64_t float_to_double(float value) -{ - conversion_t in; - in.f = value; - uint8_t sign; - int16_t exponent; - uint64_t mantissa; - - /* Decompose input value */ - sign = (in.i >> 31) & 1; - exponent = ((in.i >> 23) & 0xFF) - 127; - mantissa = in.i & 0x7FFFFF; - - if (exponent == 128) - { - /* Special value (NaN etc.) */ - exponent = 1024; - } - else if (exponent == -127) - { - if (!mantissa) - { - /* Zero */ - exponent = -1023; - } - else - { - /* Denormalized */ - mantissa <<= 1; - while (!(mantissa & 0x800000)) - { - mantissa <<= 1; - exponent--; - } - mantissa &= 0x7FFFFF; - } - } - - /* Combine fields */ - mantissa <<= 29; - mantissa |= (uint64_t)(exponent + 1023) << 52; - mantissa |= (uint64_t)sign << 63; - - return mantissa; -} - -float double_to_float(uint64_t value) -{ - uint8_t sign; - int16_t exponent; - uint32_t mantissa; - conversion_t out; - - /* Decompose input value */ - sign = (value >> 63) & 1; - exponent = ((value >> 52) & 0x7FF) - 1023; - mantissa = (value >> 28) & 0xFFFFFF; /* Highest 24 bits */ - - /* Figure if value is in range representable by floats. */ - if (exponent == 1024) - { - /* Special value */ - exponent = 128; - } - else if (exponent > 127) - { - /* Too large */ - if (sign) - return -INFINITY; - else - return INFINITY; - } - else if (exponent < -150) - { - /* Too small */ - if (sign) - return -0.0f; - else - return 0.0f; - } - else if (exponent < -126) - { - /* Denormalized */ - mantissa |= 0x1000000; - mantissa >>= (-126 - exponent); - exponent = -127; - } - - /* Round off mantissa */ - mantissa = (mantissa + 1) >> 1; - - /* Check if mantissa went over 2.0 */ - if (mantissa & 0x800000) - { - exponent += 1; - mantissa &= 0x7FFFFF; - mantissa >>= 1; - } - - /* Combine fields */ - out.i = mantissa; - out.i |= (uint32_t)(exponent + 127) << 23; - out.i |= (uint32_t)sign << 31; - - return out.f; -} - - diff --git a/examples/using_double_on_avr/double_conversion.h b/examples/using_double_on_avr/double_conversion.h deleted file mode 100644 index 62b6a8a..0000000 --- a/examples/using_double_on_avr/double_conversion.h +++ /dev/null @@ -1,26 +0,0 @@ -/* AVR-GCC does not have real double datatype. Instead its double - * is equal to float, i.e. 32 bit value. If you need to communicate - * with other systems that use double in their .proto files, you - * need to do some conversion. - * - * These functions use bitwise operations to mangle floats into doubles - * and then store them in uint64_t datatype. - */ - -#ifndef DOUBLE_CONVERSION -#define DOUBLE_CONVERSION - -#include <stdint.h> - -/* Convert native 4-byte float into a 8-byte double. */ -extern uint64_t float_to_double(float value); - -/* Convert 8-byte double into native 4-byte float. - * Values are rounded to nearest, 0.5 away from zero. - * Overflowing values are converted to Inf or -Inf. - */ -extern float double_to_float(uint64_t value); - - -#endif - diff --git a/examples/using_double_on_avr/doubleproto.proto b/examples/using_double_on_avr/doubleproto.proto deleted file mode 100644 index 72d3f9c..0000000 --- a/examples/using_double_on_avr/doubleproto.proto +++ /dev/null @@ -1,15 +0,0 @@ -// A message containing doubles, as used by other applications. -syntax = "proto2"; - -message DoubleMessage { - required double field1 = 1; - required double field2 = 2; -} - -// A message containing doubles, but redefined using uint64_t. -// For use in AVR code. -message AVRDoubleMessage { - required fixed64 field1 = 1; - required fixed64 field2 = 2; -} - diff --git a/examples/using_double_on_avr/encode_double.c b/examples/using_double_on_avr/encode_double.c deleted file mode 100644 index cd532d4..0000000 --- a/examples/using_double_on_avr/encode_double.c +++ /dev/null @@ -1,25 +0,0 @@ -/* Encodes a float value into a double on the wire. - * Used to emit doubles from AVR code, which doesn't support double directly. - */ - -#include <stdio.h> -#include <pb_encode.h> -#include "double_conversion.h" -#include "doubleproto.pb.h" - -int main() -{ - AVRDoubleMessage message = { - float_to_double(1234.5678f), - float_to_double(0.00001f) - }; - - uint8_t buffer[32]; - pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); - - pb_encode(&stream, AVRDoubleMessage_fields, &message); - fwrite(buffer, 1, stream.bytes_written, stdout); - - return 0; -} - diff --git a/examples/using_double_on_avr/test_conversions.c b/examples/using_double_on_avr/test_conversions.c deleted file mode 100644 index 22620a6..0000000 --- a/examples/using_double_on_avr/test_conversions.c +++ /dev/null @@ -1,56 +0,0 @@ -#include "double_conversion.h" -#include <math.h> -#include <stdio.h> - -static const double testvalues[] = { - 0.0, -0.0, 0.1, -0.1, - M_PI, -M_PI, 123456.789, -123456.789, - INFINITY, -INFINITY, NAN, INFINITY - INFINITY, - 1e38, -1e38, 1e39, -1e39, - 1e-38, -1e-38, 1e-39, -1e-39, - 3.14159e-37,-3.14159e-37, 3.14159e-43, -3.14159e-43, - 1e-60, -1e-60, 1e-45, -1e-45, - 0.99999999999999, -0.99999999999999, 127.999999999999, -127.999999999999 -}; - -#define TESTVALUES_COUNT (sizeof(testvalues)/sizeof(testvalues[0])) - -int main() -{ - int status = 0; - int i; - for (i = 0; i < TESTVALUES_COUNT; i++) - { - double orig = testvalues[i]; - float expected_float = (float)orig; - double expected_double = (double)expected_float; - - float got_float = double_to_float(*(uint64_t*)&orig); - uint64_t got_double = float_to_double(got_float); - - uint32_t e1 = *(uint32_t*)&expected_float; - uint32_t g1 = *(uint32_t*)&got_float; - uint64_t e2 = *(uint64_t*)&expected_double; - uint64_t g2 = got_double; - - if (g1 != e1) - { - printf("%3d double_to_float fail: %08x != %08x\n", i, g1, e1); - status = 1; - } - - if (g2 != e2) - { - printf("%3d float_to_double fail: %016llx != %016llx\n", i, - (unsigned long long)g2, - (unsigned long long)e2); - status = 1; - } - } - - return status; -} - - - - @@ -45,6 +45,10 @@ * Such example is older protobuf.js. */ /* #define PB_ENCODE_ARRAYS_UNPACKED 1 */ +/* Enable conversion of doubles to floats for platforms that do not + * support 64-bit doubles. Most commonly AVR. */ +/* #define PB_CONVERT_DOUBLE_FLOAT 1 */ + /****************************************************************** * You usually don't need to change anything below this line. * * Feel free to look around and use the defined macros, though. * diff --git a/pb_decode.c b/pb_decode.c index 42e9210..f47018e 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -1415,6 +1415,13 @@ static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_iter_ static bool checkreturn pb_dec_fixed(pb_istream_t *stream, const pb_field_iter_t *field) { +#ifdef PB_CONVERT_DOUBLE_FLOAT + if (field->data_size == sizeof(float) && PB_LTYPE(field->type) == PB_LTYPE_FIXED64) + { + return pb_decode_double_as_float(stream, (float*)field->pData); + } +#endif + if (field->data_size == sizeof(uint32_t)) { return pb_decode_fixed32(stream, field->pData); @@ -1551,3 +1558,72 @@ static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, const pb return pb_read(stream, (pb_byte_t*)field->pData, field->data_size); } +#ifdef PB_CONVERT_DOUBLE_FLOAT +bool pb_decode_double_as_float(pb_istream_t *stream, float *dest) +{ + uint8_t sign; + int exponent; + uint32_t mantissa; + uint64_t value; + union { float f; uint32_t i; } out; + + if (!pb_decode_fixed64(stream, &value)) + return false; + + /* Decompose input value */ + sign = (uint8_t)((value >> 63) & 1); + exponent = (int)(((value >> 52) & 0x7FF) - 1023); + mantissa = (value >> 28) & 0xFFFFFF; /* Highest 24 bits */ + + /* Figure if value is in range representable by floats. */ + if (exponent == 1024) + { + /* Special value */ + exponent = 128; + } + else if (exponent > 127) + { + /* Too large */ + if (sign) + *dest = -1.0f/0.0f; /* -INFINITY */ + else + *dest = 1.0f/0.0f; /* +INFINITY */ + return true; + } + else if (exponent < -150) + { + /* Too small */ + if (sign) + *dest = -0.0f; + else + *dest = 0.0f; + return true; + } + else if (exponent < -126) + { + /* Denormalized */ + mantissa |= 0x1000000; + mantissa >>= (-126 - exponent); + exponent = -127; + } + + /* Round off mantissa */ + mantissa = (mantissa + 1) >> 1; + + /* Check if mantissa went over 2.0 */ + if (mantissa & 0x800000) + { + exponent += 1; + mantissa &= 0x7FFFFF; + mantissa >>= 1; + } + + /* Combine fields */ + out.i = mantissa; + out.i |= (uint32_t)(exponent + 127) << 23; + out.i |= (uint32_t)sign << 31; + + *dest = out.f; + return true; +} +#endif diff --git a/pb_decode.h b/pb_decode.h index 0498a83..b64d95a 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -177,6 +177,11 @@ bool pb_decode_fixed32(pb_istream_t *stream, void *dest); bool pb_decode_fixed64(pb_istream_t *stream, void *dest); #endif +#ifdef PB_CONVERT_DOUBLE_FLOAT +/* Decode a double value into float variable. */ +bool pb_decode_double_as_float(pb_istream_t *stream, float *dest); +#endif + /* Make a limited-length substream for reading a PB_WT_STRING field. */ bool pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream); bool pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream); diff --git a/pb_encode.c b/pb_encode.c index 8856981..1dccba3 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -795,6 +795,13 @@ static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_iter_ static bool checkreturn pb_enc_fixed(pb_ostream_t *stream, const pb_field_iter_t *field) { +#ifdef PB_CONVERT_DOUBLE_FLOAT + if (field->data_size == sizeof(float) && PB_LTYPE(field->type) == PB_LTYPE_FIXED64) + { + return pb_encode_float_as_double(stream, *(float*)field->pData); + } +#endif + if (field->data_size == sizeof(uint32_t)) { return pb_encode_fixed32(stream, field->pData); @@ -892,3 +899,52 @@ static bool checkreturn pb_enc_fixed_length_bytes(pb_ostream_t *stream, const pb { return pb_encode_string(stream, (const pb_byte_t*)field->pData, field->data_size); } + +#ifdef PB_CONVERT_DOUBLE_FLOAT +bool pb_encode_float_as_double(pb_ostream_t *stream, float value) +{ + union { float f; uint32_t i; } in; + uint8_t sign; + int exponent; + uint64_t mantissa; + + in.f = value; + + /* Decompose input value */ + sign = (uint8_t)((in.i >> 31) & 1); + exponent = (int)(((in.i >> 23) & 0xFF) - 127); + mantissa = in.i & 0x7FFFFF; + + if (exponent == 128) + { + /* Special value (NaN etc.) */ + exponent = 1024; + } + else if (exponent == -127) + { + if (!mantissa) + { + /* Zero */ + exponent = -1023; + } + else + { + /* Denormalized */ + mantissa <<= 1; + while (!(mantissa & 0x800000)) + { + mantissa <<= 1; + exponent--; + } + mantissa &= 0x7FFFFF; + } + } + + /* Combine fields */ + mantissa <<= 29; + mantissa |= (uint64_t)(exponent + 1023) << 52; + mantissa |= (uint64_t)sign << 63; + + return pb_encode_fixed64(stream, &mantissa); +} +#endif diff --git a/pb_encode.h b/pb_encode.h index 66f6b2f..88e246a 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -165,6 +165,12 @@ bool pb_encode_fixed32(pb_ostream_t *stream, const void *value); bool pb_encode_fixed64(pb_ostream_t *stream, const void *value); #endif +#ifdef PB_CONVERT_DOUBLE_FLOAT +/* Encode a float value so that it appears like a double in the encoded + * message. */ +bool pb_encode_float_as_double(pb_ostream_t *stream, float value); +#endif + /* Encode a submessage field. * You need to pass the pb_field_t array and pointer to struct, just like * with pb_encode(). This internally encodes the submessage twice, first to diff --git a/tests/float_double_conversion/SConscript b/tests/float_double_conversion/SConscript new file mode 100644 index 0000000..f64a318 --- /dev/null +++ b/tests/float_double_conversion/SConscript @@ -0,0 +1,18 @@ +Import("env") + +env.NanopbProto("doublemsg") + +# Define the compilation options +opts = env.Clone() +opts.Append(CPPDEFINES = {'PB_CONVERT_DOUBLE_FLOAT': 1}) + +# Build new version of core +strict = opts.Clone() +strict.Append(CFLAGS = strict['CORECFLAGS']) +strict.Object("pb_decode_fldbl.o", "$NANOPB/pb_decode.c") +strict.Object("pb_encode_fldbl.o", "$NANOPB/pb_encode.c") +strict.Object("pb_common_fldbl.o", "$NANOPB/pb_common.c") + +# Build and run test +test = opts.Program(["float_double_conversion.c", "doublemsg.pb.c", "pb_encode_fldbl.o", "pb_decode_fldbl.o", "pb_common_fldbl.o"]) +env.RunTest(test) diff --git a/tests/float_double_conversion/doublemsg.proto b/tests/float_double_conversion/doublemsg.proto new file mode 100644 index 0000000..d022692 --- /dev/null +++ b/tests/float_double_conversion/doublemsg.proto @@ -0,0 +1,5 @@ +syntax = "proto2"; + +message DoubleMsg { + required double value = 1; +} diff --git a/tests/float_double_conversion/float_double_conversion.c b/tests/float_double_conversion/float_double_conversion.c new file mode 100644 index 0000000..eb3b45a --- /dev/null +++ b/tests/float_double_conversion/float_double_conversion.c @@ -0,0 +1,79 @@ +#define _USE_MATH_DEFINES 1 +#undef __STRICT_ANSI__ +#include <math.h> +#include <stdio.h> +#include <string.h> +#include <pb_decode.h> +#include <pb_encode.h> +#include "doublemsg.pb.h" +#include "unittests.h" + +/* This message mimics how DoubleMsg would appear on e.g. AVR. */ +typedef struct { + float value; +} FloatMsg; +PB_BIND(DoubleMsg, FloatMsg, AUTO) + +static const double testvalues[] = { + 0.0, -0.0, 0.1, -0.1, + M_PI, -M_PI, 123456.789, -123456.789, + INFINITY, -INFINITY, NAN, INFINITY - INFINITY, + 1e38, -1e38, 1e39, -1e39, + 1e-38, -1e-38, 1e-39, -1e-39, + 3.14159e-37,-3.14159e-37, 3.14159e-43, -3.14159e-43, + 1e-60, -1e-60, 1e-45, -1e-45, + 0.99999999999999, -0.99999999999999, 127.999999999999, -127.999999999999 +}; + +#define TESTVALUES_COUNT (sizeof(testvalues)/sizeof(testvalues[0])) + +int main() +{ + uint8_t buf[16]; + size_t msglen; + int status = 0; + int i; + for (i = 0; i < TESTVALUES_COUNT; i++) + { + double orig_double = testvalues[i]; + float expected_float = (float)orig_double; + double expected_double = (double)expected_float; + + printf("\n---- Testcase: %f ----\n", expected_float); + + { + /* Encode the original double */ + pb_ostream_t stream = pb_ostream_from_buffer(buf, sizeof(buf)); + DoubleMsg msg = { 0.0 }; + msg.value = orig_double; + TEST(pb_encode(&stream, &DoubleMsg_msg, &msg)); + msglen = stream.bytes_written; + TEST(msglen == 9); + } + + { + /* Decode as float */ + pb_ostream_t ostream; + pb_istream_t stream = pb_istream_from_buffer(buf, msglen); + FloatMsg msg = { 0.0f }; + TEST(pb_decode(&stream, &FloatMsg_msg, &msg)); + TEST(memcmp(&msg.value, &expected_float, sizeof(float)) == 0); + + /* Re-encode */ + ostream = pb_ostream_from_buffer(buf, sizeof(buf)); + TEST(pb_encode(&ostream, &FloatMsg_msg, &msg)); + msglen = ostream.bytes_written; + TEST(msglen == 9); + } + + { + /* Decode as double */ + pb_istream_t stream = pb_istream_from_buffer(buf, msglen); + DoubleMsg msg = { 0.0 }; + TEST(pb_decode(&stream, &DoubleMsg_msg, &msg)); + TEST(memcmp(&msg.value, &expected_double, sizeof(double)) == 0); + } + } + + return status; +} |