Updated kenlm 96ef3f2c11.

Invalidates old gcc and 32-bit formats, replacing these with one consistent format: 64-bit new gcc.  Backwards compatible with these files.

1 files changed, 47 insertions, 37 deletions

diff --git a/util/sorted_uniform_test.cc b/util/sorted_uniform_test.cc
index 4aa4c8aad..ac7a0bfc5 100644
--- a/util/sorted_uniform_test.cc
+++ b/util/sorted_uniform_test.cc
@@ -1,7 +1,5 @@
 #include "util/sorted_uniform.hh"
 
-#include "util/key_value_packing.hh"
-
 #include <boost/random/mersenne_twister.hpp>
 #include <boost/random/uniform_int.hpp>
 #include <boost/random/variate_generator.hpp>
@@ -17,74 +15,86 @@
 namespace util {
 namespace {
 
-template <class Map, class Key, class Value> void Check(const Map &map, const boost::unordered_map<Key, Value> &reference, const Key key) {
+template <class KeyT, class ValueT> struct Entry {
+  typedef KeyT Key;
+  typedef ValueT Value;
+
+  Key key;
+  Value value;
+
+  Key GetKey() const {
+    return key;
+  }
+
+  Value GetValue() const {
+    return value;
+  }
+
+  bool operator<(const Entry<Key,Value> &other) const {
+    return key < other.key;
+  }
+};
+
+template <class KeyT> struct Accessor {
+  typedef KeyT Key;
+  template <class Value> Key operator()(const Entry<Key, Value> *entry) const {
+    return entry->GetKey();
+  }
+};
+
+template <class Key, class Value> void Check(const Entry<Key, Value> *begin, const Entry<Key, Value> *end, const boost::unordered_map<Key, Value> &reference, const Key key) {
   typename boost::unordered_map<Key, Value>::const_iterator ref = reference.find(key);
-  typename Map::ConstIterator i = typename Map::ConstIterator();
+  typedef const Entry<Key, Value> *It;
+  // g++ can't tell that require will crash and burn.
+  It i = NULL;
+  bool ret = SortedUniformFind<It, Accessor<Key>, Pivot64>(Accessor<Key>(), begin, end, key, i);
   if (ref == reference.end()) {
-    BOOST_CHECK(!map.Find(key, i));
+    BOOST_CHECK(!ret);
   } else {
-    // g++ can't tell that require will crash and burn.
-    BOOST_REQUIRE(map.Find(key, i));
+    BOOST_REQUIRE(ret);
     BOOST_CHECK_EQUAL(ref->second, i->GetValue());
   }
 }
 
-typedef SortedUniformMap<AlignedPacking<uint64_t, uint32_t> > TestMap;
-
 BOOST_AUTO_TEST_CASE(empty) {
-  char buf[TestMap::Size(0)];
-  TestMap map(buf, TestMap::Size(0));
-  map.FinishedInserting();
-  TestMap::ConstIterator i;
-  BOOST_CHECK(!map.Find(42, i));
-}
-
-BOOST_AUTO_TEST_CASE(one) {
-  char buf[TestMap::Size(1)];
-  TestMap map(buf, sizeof(buf));
-  Entry<uint64_t, uint32_t> e;
-  e.Set(42,2);
-  map.Insert(e);
-  map.FinishedInserting();
-  TestMap::ConstIterator i = TestMap::ConstIterator();
-  BOOST_REQUIRE(map.Find(42, i));
-  BOOST_CHECK(i == map.begin());
-  BOOST_CHECK(!map.Find(43, i));
-  BOOST_CHECK(!map.Find(41, i));
+  typedef const Entry<uint64_t, float> T;
+  const T *i;
+  bool ret = SortedUniformFind<const T*, Accessor<uint64_t>, Pivot64>(Accessor<uint64_t>(), (const T*)NULL, (const T*)NULL, (uint64_t)10, i);
+  BOOST_CHECK(!ret);
 }
 
 template <class Key> void RandomTest(Key upper, size_t entries, size_t queries) {
   typedef unsigned char Value;
-  typedef SortedUniformMap<AlignedPacking<Key, unsigned char> > Map;
-  boost::scoped_array<char> buffer(new char[Map::Size(entries)]);
-  Map map(buffer.get(), entries);
   boost::mt19937 rng;
   boost::uniform_int<Key> range_key(0, upper);
   boost::uniform_int<Value> range_value(0, 255);
   boost::variate_generator<boost::mt19937&, boost::uniform_int<Key> > gen_key(rng, range_key);
   boost::variate_generator<boost::mt19937&, boost::uniform_int<unsigned char> > gen_value(rng, range_value);
 
+  typedef Entry<Key, Value> Ent;
+  std::vector<Ent> backing;
   boost::unordered_map<Key, unsigned char> reference;
-  Entry<Key, unsigned char> ent;
+  Ent ent;
   for (size_t i = 0; i < entries; ++i) {
     Key key = gen_key();
     unsigned char value = gen_value();
     if (reference.insert(std::make_pair(key, value)).second) {
-      ent.Set(key, value);
-      map.Insert(Entry<Key, unsigned char>(ent));
+      ent.key = key;
+      ent.value = value;
+      backing.push_back(ent);
     }
   }
-  map.FinishedInserting();
+  std::sort(backing.begin(), backing.end());
 
   // Random queries.  
   for (size_t i = 0; i < queries; ++i) {
     const Key key = gen_key();
-    Check<Map, Key, unsigned char>(map, reference, key);
+    Check<Key, unsigned char>(&*backing.begin(), &*backing.end(), reference, key);
   }
 
   typename boost::unordered_map<Key, unsigned char>::const_iterator it = reference.begin();
   for (size_t i = 0; (i < queries) && (it != reference.end()); ++i, ++it) {
-    Check<Map, Key, unsigned char>(map, reference, it->second);
+    Check<Key, unsigned char>(&*backing.begin(), &*backing.end(), reference, it->second);
   }
 }