Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/moses-smt/mosesdecoder.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to 'util/file_piece.cc')
-rw-r--r--util/file_piece.cc82
1 files changed, 50 insertions, 32 deletions
diff --git a/util/file_piece.cc b/util/file_piece.cc
index c808e7d90..92edef27d 100644
--- a/util/file_piece.cc
+++ b/util/file_piece.cc
@@ -11,19 +11,22 @@
#include <unistd.h>
#endif
+#include <cassert>
+#include <cerrno>
+#include <cmath>
+#include <cstdlib>
#include <iostream>
-#include <string>
#include <limits>
-#include <cassert>
+#include <string>
+
#include <fcntl.h>
-#include <cstdlib>
#include <sys/types.h>
#include <sys/stat.h>
namespace util {
ParseNumberException::ParseNumberException(StringPiece value) throw() {
- *this << "Could not parse \"" << value << "\" into a number";
+ *this << "Could not parse \"" << value << "\" into a ";
}
// Sigh this is the only way I could come up with to do a _const_ bool. It has ' ', '\f', '\n', '\r', '\t', and '\v' (same as isspace on C locale).
@@ -62,12 +65,17 @@ FilePiece::FilePiece(std::istream &stream, const char *name, std::size_t min_buf
FilePiece::~FilePiece() {}
-StringPiece FilePiece::ReadLine(char delim) {
+StringPiece FilePiece::ReadLine(char delim, bool strip_cr) {
std::size_t skip = 0;
while (true) {
for (const char *i = position_ + skip; i < position_end_; ++i) {
if (*i == delim) {
- StringPiece ret(position_, i - position_);
+ // End of line.
+ // Take 1 byte off the end if it's an unwanted carriage return.
+ const std::size_t subtract_cr = (
+ (strip_cr && i > position_ && *(i - 1) == '\r') ?
+ 1 : 0);
+ StringPiece ret(position_, i - position_ - subtract_cr);
position_ = i + 1;
return ret;
}
@@ -83,9 +91,9 @@ StringPiece FilePiece::ReadLine(char delim) {
}
}
-bool FilePiece::ReadLineOrEOF(StringPiece &to, char delim) {
+bool FilePiece::ReadLineOrEOF(StringPiece &to, char delim, bool strip_cr) {
try {
- to = ReadLine(delim);
+ to = ReadLine(delim, strip_cr);
} catch (const util::EndOfFileException &e) { return false; }
return true;
}
@@ -145,49 +153,59 @@ static const double_conversion::StringToDoubleConverter kConverter(
"inf",
"NaN");
-void ParseNumber(const char *begin, const char *&end, float &out) {
+StringPiece FirstToken(StringPiece str) {
+ const char *i;
+ for (i = str.data(); i != str.data() + str.size(); ++i) {
+ if (kSpaces[(unsigned char)*i]) break;
+ }
+ return StringPiece(str.data(), i - str.data());
+}
+
+const char *ParseNumber(StringPiece str, float &out) {
int count;
- out = kConverter.StringToFloat(begin, end - begin, &count);
- end = begin + count;
+ out = kConverter.StringToFloat(str.data(), str.size(), &count);
+ UTIL_THROW_IF_ARG(isnan(out) && str != "NaN" && str != "nan", ParseNumberException, (FirstToken(str)), "float");
+ return str.data() + count;
}
-void ParseNumber(const char *begin, const char *&end, double &out) {
+const char *ParseNumber(StringPiece str, double &out) {
int count;
- out = kConverter.StringToDouble(begin, end - begin, &count);
- end = begin + count;
+ out = kConverter.StringToDouble(str.data(), str.size(), &count);
+ UTIL_THROW_IF_ARG(isnan(out) && str != "NaN" && str != "nan", ParseNumberException, (FirstToken(str)), "double");
+ return str.data() + count;
}
-void ParseNumber(const char *begin, const char *&end, long int &out) {
- char *silly_end;
- out = strtol(begin, &silly_end, 10);
- end = silly_end;
+const char *ParseNumber(StringPiece str, long int &out) {
+ char *end;
+ errno = 0;
+ out = strtol(str.data(), &end, 10);
+ UTIL_THROW_IF_ARG(errno || (end == str.data()), ParseNumberException, (FirstToken(str)), "long int");
+ return end;
}
-void ParseNumber(const char *begin, const char *&end, unsigned long int &out) {
- char *silly_end;
- out = strtoul(begin, &silly_end, 10);
- end = silly_end;
+const char *ParseNumber(StringPiece str, unsigned long int &out) {
+ char *end;
+ errno = 0;
+ out = strtoul(str.data(), &end, 10);
+ UTIL_THROW_IF_ARG(errno || (end == str.data()), ParseNumberException, (FirstToken(str)), "unsigned long int");
+ return end;
}
} // namespace
template <class T> T FilePiece::ReadNumber() {
SkipSpaces();
while (last_space_ < position_) {
- if (at_end_) {
+ if (UTIL_UNLIKELY(at_end_)) {
// Hallucinate a null off the end of the file.
std::string buffer(position_, position_end_);
- const char *buf = buffer.c_str();
- const char *end = buf + buffer.size();
T ret;
- ParseNumber(buf, end, ret);
- if (buf == end) throw ParseNumberException(buffer);
- position_ += end - buf;
+ // Has to be null-terminated.
+ const char *begin = buffer.c_str();
+ const char *end = ParseNumber(StringPiece(begin, buffer.size()), ret);
+ position_ += end - begin;
return ret;
}
Shift();
}
- const char *end = last_space_;
T ret;
- ParseNumber(position_, end, ret);
- if (end == position_) throw ParseNumberException(ReadDelimited());
- position_ = end;
+ position_ = ParseNumber(StringPiece(position_, last_space_ - position_), ret);
return ret;
}