stdlib: Make strtod/strtof set ERANGE consistently for underflow.

The C standard says that errno may acquire the value ERANGE if the
result from strtod underflows. According to IEEE 754, underflow occurs
whenever the value cannot be represented in normalized form.

Newlib is inconsistent in this, setting errno to ERANGE only if the
value underflows to zero, but not for denorm values, and never for hex
format floats.

This patch attempts to consistently set errno to ERANGE for all
'underflow' conditions, which is to say all values which are not
exactly zero and which cannot be represented in normalized form.

This matches glibc behavior, as well as the Linux, Mac OS X, OpenBSD,
FreeBSD and SunOS strtod man pages.

Signed-off-by: Keith Packard <keithp@keithp.com>

1 files changed, 29 insertions, 2 deletions

diff --git a/newlib/libc/stdlib/strtod.c b/newlib/libc/stdlib/strtod.c
index 8bb75ef0a..019416ca7 100644
--- a/newlib/libc/stdlib/strtod.c
+++ b/newlib/libc/stdlib/strtod.c
@@ -326,6 +326,11 @@ _strtod_l (struct _reent *ptr, const char *__restrict s00, char **__restrict se,
 					Bfree(ptr,bb);
 					}
 				ULtod(rv.i, bits, exp, i);
+#ifndef NO_ERRNO
+                                /* try to avoid the bug of testing an 8087 register value */
+                                if ((dword0(rv)&Exp_mask) == 0)
+                                    errno = ERANGE;
+#endif
 			  }}
 			goto ret;
 		  }
@@ -1238,7 +1243,7 @@ _strtod_l (struct _reent *ptr, const char *__restrict s00, char **__restrict se,
 		dval(rv) *= dval(rv0);
 #ifndef NO_ERRNO
 		/* try to avoid the bug of testing an 8087 register value */
-		if (dword0(rv) == 0 && dword1(rv) == 0)
+		if ((dword0(rv) & Exp_mask) == 0)
 			ptr->_errno = ERANGE;
 #endif
 		}
@@ -1298,6 +1303,28 @@ strtof_l (const char *__restrict s00, char **__restrict se, locale_t loc)
   return retval;
 }
 
+/*
+ * These two functions are not quite correct as they return true for
+ * zero, however they are 'good enough' for the test in strtof below
+ * as we only need to know whether the double test is false when
+ * the float test is true.
+ */
+static inline int
+isdenorm(double d)
+{
+    U u;
+    dval(u) = d;
+    return (dword0(u) & Exp_mask) == 0;
+}
+
+static inline int
+isdenormf(float f)
+{
+    union { float f; __uint32_t i; } u;
+    u.f = f;
+    return (u.i & 0x7f800000) == 0;
+}
+
 float
 strtof (const char *__restrict s00,
 	char **__restrict se)
@@ -1307,7 +1334,7 @@ strtof (const char *__restrict s00,
     return signbit (val) ? -nanf ("") : nanf ("");
   float retval = (float) val;
 #ifndef NO_ERRNO
-  if (isinf (retval) && !isinf (val))
+  if ((isinf (retval) && !isinf (val)) || (isdenormf(retval) && !isdenorm(val)))
     _REENT->_errno = ERANGE;
 #endif
   return retval;