build word2vec by default

14 files changed, 5 insertions, 798 deletions

diff --git a/Jamroot b/Jamroot
index 1c850861f..efd34fdc7 100644
--- a/Jamroot
+++ b/Jamroot
@@ -212,6 +212,7 @@ biconcor
 mira//mira 
 contrib/server//mosesserver 
 contrib/bleu-champ//programs 
+contrib/word2vec//word2vec 
 mm
 ;
 
diff --git a/contrib/word2vec/Jamfile b/contrib/word2vec/Jamfile
new file mode 100644
index 000000000..d3c3efd76
--- /dev/null
+++ b/contrib/word2vec/Jamfile
@@ -0,0 +1,4 @@
+
+exe word2vec : word2vec.c ;
+
+install legacy : word2vec : <location>. ;
diff --git a/contrib/word2vec/README.txt b/contrib/word2vec/README.txt
deleted file mode 100644
index 2bcd9da44..000000000
--- a/contrib/word2vec/README.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-Tools for computing distributed representtion of words
-------------------------------------------------------
-
-We provide an implementation of the Continuous Bag-of-Words (CBOW) and the Skip-gram model (SG), as well as several demo scripts.
-
-Given a text corpus, the word2vec tool learns a vector for every word in the vocabulary using the Continuous
-Bag-of-Words or the Skip-Gram neural network architectures. The user should to specify the following:
- - desired vector dimensionality
- - the size of the context window for either the Skip-Gram or the Continuous Bag-of-Words model
- - training algorithm: hierarchical softmax and / or negative sampling
- - threshold for downsampling the frequent words 
- - number of threads to use
- - the format of the output word vector file (text or binary)
-
-Usually, the other hyper-parameters such as the learning rate do not need to be tuned for different training sets. 
-
-The script demo-word.sh downloads a small (100MB) text corpus from the web, and trains a small word vector model. After the training
-is finished, the user can interactively explore the similarity of the words.
-
-More information about the scripts is provided at https://code.google.com/p/word2vec/
-
diff --git a/contrib/word2vec/compute-accuracy.c b/contrib/word2vec/compute-accuracy.c
deleted file mode 100644
index d83fcbb9a..000000000
--- a/contrib/word2vec/compute-accuracy.c
+++ /dev/null
@@ -1,137 +0,0 @@
-//  Copyright 2013 Google Inc. All Rights Reserved.
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
-#include <malloc.h>
-#include <ctype.h>
-
-const long long max_size = 2000;         // max length of strings
-const long long N = 1;                   // number of closest words
-const long long max_w = 50;              // max length of vocabulary entries
-
-int main(int argc, char **argv)
-{
-  FILE *f;
-  char st1[max_size], st2[max_size], st3[max_size], st4[max_size], bestw[N][max_size], file_name[max_size], ch;
-  float dist, len, bestd[N], vec[max_size];
-  long long words, size, a, b, c, d, b1, b2, b3, threshold = 0;
-  float *M;
-  char *vocab;
-  int TCN, CCN = 0, TACN = 0, CACN = 0, SECN = 0, SYCN = 0, SEAC = 0, SYAC = 0, QID = 0, TQ = 0, TQS = 0;
-  if (argc < 2) {
-    printf("Usage: ./compute-accuracy <FILE> <threshold>\nwhere FILE contains word projections, and threshold is used to reduce vocabulary of the model for fast approximate evaluation (0 = off, otherwise typical value is 30000)\n");
-    return 0;
-  }
-  strcpy(file_name, argv[1]);
-  if (argc > 2) threshold = atoi(argv[2]);
-  f = fopen(file_name, "rb");
-  if (f == NULL) {
-    printf("Input file not found\n");
-    return -1;
-  }
-  fscanf(f, "%lld", &words);
-  if (threshold) if (words > threshold) words = threshold;
-  fscanf(f, "%lld", &size);
-  vocab = (char *)malloc(words * max_w * sizeof(char));
-  M = (float *)malloc(words * size * sizeof(float));
-  if (M == NULL) {
-    printf("Cannot allocate memory: %lld MB\n", words * size * sizeof(float) / 1048576);
-    return -1;
-  }
-  for (b = 0; b < words; b++) {
-    fscanf(f, "%s%c", &vocab[b * max_w], &ch);
-    for (a = 0; a < max_w; a++) vocab[b * max_w + a] = toupper(vocab[b * max_w + a]);
-    for (a = 0; a < size; a++) fread(&M[a + b * size], sizeof(float), 1, f);
-    len = 0;
-    for (a = 0; a < size; a++) len += M[a + b * size] * M[a + b * size];
-    len = sqrt(len);
-    for (a = 0; a < size; a++) M[a + b * size] /= len;
-  }
-  fclose(f);
-  TCN = 0;
-  while (1) {
-    for (a = 0; a < N; a++) bestd[a] = 0;
-    for (a = 0; a < N; a++) bestw[a][0] = 0;
-    scanf("%s", st1);
-    for (a = 0; a < strlen(st1); a++) st1[a] = toupper(st1[a]);
-    if ((!strcmp(st1, ":")) || (!strcmp(st1, "EXIT")) || feof(stdin)) {
-      if (TCN == 0) TCN = 1;
-      if (QID != 0) {
-        printf("ACCURACY TOP1: %.2f %%  (%d / %d)\n", CCN / (float)TCN * 100, CCN, TCN);
-        printf("Total accuracy: %.2f %%   Semantic accuracy: %.2f %%   Syntactic accuracy: %.2f %% \n", CACN / (float)TACN * 100, SEAC / (float)SECN * 100, SYAC / (float)SYCN * 100);
-      }
-      QID++;
-      scanf("%s", st1);
-      if (feof(stdin)) break;
-      printf("%s:\n", st1);
-      TCN = 0;
-      CCN = 0;
-      continue;
-    }
-    if (!strcmp(st1, "EXIT")) break;
-    scanf("%s", st2);
-    for (a = 0; a < strlen(st2); a++) st2[a] = toupper(st2[a]);
-    scanf("%s", st3);
-    for (a = 0; a<strlen(st3); a++) st3[a] = toupper(st3[a]);
-    scanf("%s", st4);
-    for (a = 0; a < strlen(st4); a++) st4[a] = toupper(st4[a]);
-    for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st1)) break;
-    b1 = b;
-    for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st2)) break;
-    b2 = b;
-    for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st3)) break;
-    b3 = b;
-    for (a = 0; a < N; a++) bestd[a] = 0;
-    for (a = 0; a < N; a++) bestw[a][0] = 0;
-    TQ++;
-    if (b1 == words) continue;
-    if (b2 == words) continue;
-    if (b3 == words) continue;
-    for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st4)) break;
-    if (b == words) continue;
-    for (a = 0; a < size; a++) vec[a] = (M[a + b2 * size] - M[a + b1 * size]) + M[a + b3 * size];
-    TQS++;
-    for (c = 0; c < words; c++) {
-      if (c == b1) continue;
-      if (c == b2) continue;
-      if (c == b3) continue;
-      dist = 0;
-      for (a = 0; a < size; a++) dist += vec[a] * M[a + c * size];
-      for (a = 0; a < N; a++) {
-        if (dist > bestd[a]) {
-          for (d = N - 1; d > a; d--) {
-            bestd[d] = bestd[d - 1];
-            strcpy(bestw[d], bestw[d - 1]);
-          }
-          bestd[a] = dist;
-          strcpy(bestw[a], &vocab[c * max_w]);
-          break;
-        }
-      }
-    }
-    if (!strcmp(st4, bestw[0])) {
-      CCN++;
-      CACN++;
-      if (QID <= 5) SEAC++; else SYAC++;
-    }
-    if (QID <= 5) SECN++; else SYCN++;
-    TCN++;
-    TACN++;
-  }
-  printf("Questions seen / total: %d %d   %.2f %% \n", TQS, TQ, TQS/(float)TQ*100);
-  return 0;
-}
diff --git a/contrib/word2vec/demo-analogy.sh b/contrib/word2vec/demo-analogy.sh
deleted file mode 100755
index 3d1fcf7f9..000000000
--- a/contrib/word2vec/demo-analogy.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-make
-if [ ! -e text8 ]; then
-  wget http://mattmahoney.net/dc/text8.zip -O text8.gz
-  gzip -d text8.gz -f
-fi
-echo -----------------------------------------------------------------------------------------------------
-echo Note that for the word analogy to perform well, the models should be trained on much larger data sets
-echo Example input: paris france berlin
-echo -----------------------------------------------------------------------------------------------------
-time ./word2vec -train text8 -output vectors.bin -cbow 0 -size 200 -window 5 -negative 0 -hs 1 -sample 1e-3 -threads 12 -binary 1
-./word-analogy vectors.bin
diff --git a/contrib/word2vec/demo-classes.sh b/contrib/word2vec/demo-classes.sh
deleted file mode 100755
index b0b9d991c..000000000
--- a/contrib/word2vec/demo-classes.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-make
-if [ ! -e text8 ]; then
-  wget http://mattmahoney.net/dc/text8.zip -O text8.gz
-  gzip -d text8.gz -f
-fi
-time ./word2vec -train text8 -output classes.txt -cbow 0 -size 200 -window 5 -negative 0 -hs 1 -sample 1e-3 -threads 12 -classes 500
-sort classes.txt -k 2 -n > classes.sorted.txt
-echo The word classes were saved to file classes.sorted.txt
diff --git a/contrib/word2vec/demo-phrase-accuracy.sh b/contrib/word2vec/demo-phrase-accuracy.sh
deleted file mode 100755
index eb2b392e8..000000000
--- a/contrib/word2vec/demo-phrase-accuracy.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-make
-if [ ! -e text8 ]; then
-  wget http://mattmahoney.net/dc/text8.zip -O text8.gz
-  gzip -d text8.gz -f
-fi
-echo ----------------------------------------------------------------------------------------------------------------
-echo Note that the accuracy and coverage of the test set questions is going to be low with this small training corpus
-echo To achieve better accuracy, larger training set is needed
-echo ----------------------------------------------------------------------------------------------------------------
-time ./word2phrase -train text8 -output text8-phrase -threshold 500 -debug 2 -min-count 3
-time ./word2vec -train text8-phrase -output vectors-phrase.bin -cbow 0 -size 300 -window 10 -negative 0 -hs 1 -sample 1e-3 -threads 12 -binary 1 -min-count 3
-./compute-accuracy vectors-phrase.bin <questions-phrases.txt
diff --git a/contrib/word2vec/demo-phrases.sh b/contrib/word2vec/demo-phrases.sh
deleted file mode 100755
index c833b8149..000000000
--- a/contrib/word2vec/demo-phrases.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-make
-if [ ! -e text8 ]; then
-  wget http://mattmahoney.net/dc/text8.zip -O text8.gz
-  gzip -d text8.gz -f
-fi
-time ./word2phrase -train text8 -output text8-phrase -threshold 500 -debug 2
-time ./word2vec -train text8-phrase -output vectors-phrase.bin -cbow 0 -size 300 -window 10 -negative 0 -hs 1 -sample 1e-3 -threads 12 -binary 1
-./distance vectors-phrase.bin
\ No newline at end of file
diff --git a/contrib/word2vec/demo-word-accuracy.sh b/contrib/word2vec/demo-word-accuracy.sh
deleted file mode 100755
index ffe828ab1..000000000
--- a/contrib/word2vec/demo-word-accuracy.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-make
-if [ ! -e text8 ]; then
-  wget http://mattmahoney.net/dc/text8.zip -O text8.gz
-  gzip -d text8.gz -f
-fi
-time ./word2vec -train text8 -output vectors.bin -cbow 0 -size 200 -window 5 -negative 0 -hs 1 -sample 1e-3 -threads 12 -binary 1
-./compute-accuracy vectors.bin 30000 < questions-words.txt
-# to compute accuracy with the full vocabulary, use: ./compute-accuracy vectors.bin < questions-words.txt
diff --git a/contrib/word2vec/demo-word.sh b/contrib/word2vec/demo-word.sh
deleted file mode 100755
index 0df5bd510..000000000
--- a/contrib/word2vec/demo-word.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-make
-if [ ! -e text8 ]; then
-  wget http://mattmahoney.net/dc/text8.zip -O text8.gz
-  gzip -d text8.gz -f
-fi
-time ./word2vec -train text8 -output vectors.bin -cbow 0 -size 200 -window 5 -negative 0 -hs 1 -sample 1e-3 -threads 12 -binary 1
-./distance vectors.bin
\ No newline at end of file
diff --git a/contrib/word2vec/distance.c b/contrib/word2vec/distance.c
deleted file mode 100644
index fbeb24a66..000000000
--- a/contrib/word2vec/distance.c
+++ /dev/null
@@ -1,136 +0,0 @@
-//  Copyright 2013 Google Inc. All Rights Reserved.
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-#include <stdio.h>
-#include <string.h>
-#include <math.h>
-#include <malloc.h>
-
-const long long max_size = 2000;         // max length of strings
-const long long N = 40;                  // number of closest words that will be shown
-const long long max_w = 50;              // max length of vocabulary entries
-
-int main(int argc, char **argv) {
-  FILE *f;
-  char st1[max_size];
-  char *bestw[N];
-  char file_name[max_size], st[100][max_size];
-  float dist, len, bestd[N], vec[max_size];
-  long long words, size, a, b, c, d, cn, bi[100];
-  char ch;
-  float *M;
-  char *vocab;
-  if (argc < 2) {
-    printf("Usage: ./distance <FILE>\nwhere FILE contains word projections in the BINARY FORMAT\n");
-    return 0;
-  }
-  strcpy(file_name, argv[1]);
-  f = fopen(file_name, "rb");
-  if (f == NULL) {
-    printf("Input file not found\n");
-    return -1;
-  }
-  fscanf(f, "%lld", &words);
-  fscanf(f, "%lld", &size);
-  vocab = (char *)malloc((long long)words * max_w * sizeof(char));
-  for (a = 0; a < N; a++) bestw[a] = (char *)malloc(max_size * sizeof(char));
-  M = (float *)malloc((long long)words * (long long)size * sizeof(float));
-  if (M == NULL) {
-    printf("Cannot allocate memory: %lld MB    %lld  %lld\n", (long long)words * size * sizeof(float) / 1048576, words, size);
-    return -1;
-  }
-  for (b = 0; b < words; b++) {
-    fscanf(f, "%s%c", &vocab[b * max_w], &ch);
-    for (a = 0; a < size; a++) fread(&M[a + b * size], sizeof(float), 1, f);
-    len = 0;
-    for (a = 0; a < size; a++) len += M[a + b * size] * M[a + b * size];
-    len = sqrt(len);
-    for (a = 0; a < size; a++) M[a + b * size] /= len;
-  }
-  fclose(f);
-  while (1) {
-    for (a = 0; a < N; a++) bestd[a] = 0;
-    for (a = 0; a < N; a++) bestw[a][0] = 0;
-    printf("Enter word or sentence (EXIT to break): ");
-    a = 0;
-    while (1) {
-      st1[a] = fgetc(stdin);
-      if ((st1[a] == '\n') || (a >= max_size - 1)) {
-        st1[a] = 0;
-        break;
-      }
-      a++;
-    }
-    if (!strcmp(st1, "EXIT")) break;
-    cn = 0;
-    b = 0;
-    c = 0;
-    while (1) {
-      st[cn][b] = st1[c];
-      b++;
-      c++;
-      st[cn][b] = 0;
-      if (st1[c] == 0) break;
-      if (st1[c] == ' ') {
-        cn++;
-        b = 0;
-        c++;
-      }
-    }
-    cn++;
-    for (a = 0; a < cn; a++) {
-      for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st[a])) break;
-      if (b == words) b = -1;
-      bi[a] = b;
-      printf("\nWord: %s  Position in vocabulary: %lld\n", st[a], bi[a]);
-      if (b == -1) {
-        printf("Out of dictionary word!\n");
-        break;
-      }
-    }
-    if (b == -1) continue;
-    printf("\n                                              Word       Cosine distance\n------------------------------------------------------------------------\n");
-    for (a = 0; a < size; a++) vec[a] = 0;
-    for (b = 0; b < cn; b++) {
-      if (bi[b] == -1) continue;
-      for (a = 0; a < size; a++) vec[a] += M[a + bi[b] * size];
-    }
-    len = 0;
-    for (a = 0; a < size; a++) len += vec[a] * vec[a];
-    len = sqrt(len);
-    for (a = 0; a < size; a++) vec[a] /= len;
-    for (a = 0; a < N; a++) bestd[a] = -1;
-    for (a = 0; a < N; a++) bestw[a][0] = 0;
-    for (c = 0; c < words; c++) {
-      a = 0;
-      for (b = 0; b < cn; b++) if (bi[b] == c) a = 1;
-      if (a == 1) continue;
-      dist = 0;
-      for (a = 0; a < size; a++) dist += vec[a] * M[a + c * size];
-      for (a = 0; a < N; a++) {
-        if (dist > bestd[a]) {
-          for (d = N - 1; d > a; d--) {
-            bestd[d] = bestd[d - 1];
-            strcpy(bestw[d], bestw[d - 1]);
-          }
-          bestd[a] = dist;
-          strcpy(bestw[a], &vocab[c * max_w]);
-          break;
-        }
-      }
-    }
-    for (a = 0; a < N; a++) printf("%50s\t\t%f\n", bestw[a], bestd[a]);
-  }
-  return 0;
-}
diff --git a/contrib/word2vec/makefile b/contrib/word2vec/makefile
deleted file mode 100644
index b50ee0c3d..000000000
--- a/contrib/word2vec/makefile
+++ /dev/null
@@ -1,20 +0,0 @@
-CC = gcc
-#The -Ofast might not work with older versions of gcc; in that case, use -O2
-CFLAGS = -lm -pthread -Ofast -march=native -Wall -funroll-loops -Wno-unused-result -g
-
-all: word2vec word2phrase distance word-analogy compute-accuracy
-
-word2vec : word2vec.c
-	$(CC) word2vec.c -o word2vec $(CFLAGS)
-word2phrase : word2phrase.c
-	$(CC) word2phrase.c -o word2phrase $(CFLAGS)
-distance : distance.c
-	$(CC) distance.c -o distance $(CFLAGS)
-word-analogy : word-analogy.c
-	$(CC) word-analogy.c -o word-analogy $(CFLAGS)
-compute-accuracy : compute-accuracy.c
-	$(CC) compute-accuracy.c -o compute-accuracy $(CFLAGS)
-	chmod +x *.sh
-
-clean:
-	rm -rf word2vec word2phrase distance word-analogy compute-accuracy
\ No newline at end of file
diff --git a/contrib/word2vec/word-analogy.c b/contrib/word2vec/word-analogy.c
deleted file mode 100644
index ea91ab141..000000000
--- a/contrib/word2vec/word-analogy.c
+++ /dev/null
@@ -1,138 +0,0 @@
-//  Copyright 2013 Google Inc. All Rights Reserved.
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-#include <stdio.h>
-#include <string.h>
-#include <math.h>
-#include <malloc.h>
-
-const long long max_size = 2000;         // max length of strings
-const long long N = 40;                  // number of closest words that will be shown
-const long long max_w = 50;              // max length of vocabulary entries
-
-int main(int argc, char **argv) {
-  FILE *f;
-  char st1[max_size];
-  char bestw[N][max_size];
-  char file_name[max_size], st[100][max_size];
-  float dist, len, bestd[N], vec[max_size];
-  long long words, size, a, b, c, d, cn, bi[100];
-  char ch;
-  float *M;
-  char *vocab;
-  if (argc < 2) {
-    printf("Usage: ./word-analogy <FILE>\nwhere FILE contains word projections in the BINARY FORMAT\n");
-    return 0;
-  }
-  strcpy(file_name, argv[1]);
-  f = fopen(file_name, "rb");
-  if (f == NULL) {
-    printf("Input file not found\n");
-    return -1;
-  }
-  fscanf(f, "%lld", &words);
-  fscanf(f, "%lld", &size);
-  vocab = (char *)malloc((long long)words * max_w * sizeof(char));
-  M = (float *)malloc((long long)words * (long long)size * sizeof(float));
-  if (M == NULL) {
-    printf("Cannot allocate memory: %lld MB    %lld  %lld\n", (long long)words * size * sizeof(float) / 1048576, words, size);
-    return -1;
-  }
-  for (b = 0; b < words; b++) {
-    fscanf(f, "%s%c", &vocab[b * max_w], &ch);
-    for (a = 0; a < size; a++) fread(&M[a + b * size], sizeof(float), 1, f);
-    len = 0;
-    for (a = 0; a < size; a++) len += M[a + b * size] * M[a + b * size];
-    len = sqrt(len);
-    for (a = 0; a < size; a++) M[a + b * size] /= len;
-  }
-  fclose(f);
-  while (1) {
-    for (a = 0; a < N; a++) bestd[a] = 0;
-    for (a = 0; a < N; a++) bestw[a][0] = 0;
-    printf("Enter three words (EXIT to break): ");
-    a = 0;
-    while (1) {
-      st1[a] = fgetc(stdin);
-      if ((st1[a] == '\n') || (a >= max_size - 1)) {
-        st1[a] = 0;
-        break;
-      }
-      a++;
-    }
-    if (!strcmp(st1, "EXIT")) break;
-    cn = 0;
-    b = 0;
-    c = 0;
-    while (1) {
-      st[cn][b] = st1[c];
-      b++;
-      c++;
-      st[cn][b] = 0;
-      if (st1[c] == 0) break;
-      if (st1[c] == ' ') {
-        cn++;
-        b = 0;
-        c++;
-      }
-    }
-    cn++;
-    if (cn < 3) {
-      printf("Only %lld words were entered.. three words are needed at the input to perform the calculation\n", cn);
-      continue;
-    }
-    for (a = 0; a < cn; a++) {
-      for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st[a])) break;
-      if (b == words) b = 0;
-      bi[a] = b;
-      printf("\nWord: %s  Position in vocabulary: %lld\n", st[a], bi[a]);
-      if (b == 0) {
-        printf("Out of dictionary word!\n");
-        break;
-      }
-    }
-    if (b == 0) continue;
-    printf("\n                                              Word              Distance\n------------------------------------------------------------------------\n");
-    for (a = 0; a < size; a++) vec[a] = M[a + bi[1] * size] - M[a + bi[0] * size] + M[a + bi[2] * size];
-    len = 0;
-    for (a = 0; a < size; a++) len += vec[a] * vec[a];
-    len = sqrt(len);
-    for (a = 0; a < size; a++) vec[a] /= len;
-    for (a = 0; a < N; a++) bestd[a] = 0;
-    for (a = 0; a < N; a++) bestw[a][0] = 0;
-    for (c = 0; c < words; c++) {
-      if (c == bi[0]) continue;
-      if (c == bi[1]) continue;
-      if (c == bi[2]) continue;
-      a = 0;
-      for (b = 0; b < cn; b++) if (bi[b] == c) a = 1;
-      if (a == 1) continue;
-      dist = 0;
-      for (a = 0; a < size; a++) dist += vec[a] * M[a + c * size];
-      for (a = 0; a < N; a++) {
-        if (dist > bestd[a]) {
-          for (d = N - 1; d > a; d--) {
-            bestd[d] = bestd[d - 1];
-            strcpy(bestw[d], bestw[d - 1]);
-          }
-          bestd[a] = dist;
-          strcpy(bestw[a], &vocab[c * max_w]);
-          break;
-        }
-      }
-    }
-    for (a = 0; a < N; a++) printf("%50s\t\t%f\n", bestw[a], bestd[a]);
-  }
-  return 0;
-}
diff --git a/contrib/word2vec/word2phrase.c b/contrib/word2vec/word2phrase.c
deleted file mode 100644
index 24238bc5b..000000000
--- a/contrib/word2vec/word2phrase.c
+++ /dev/null
@@ -1,292 +0,0 @@
-//  Copyright 2013 Google Inc. All Rights Reserved.
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
-#include <pthread.h>
-
-#define MAX_STRING 60
-
-const int vocab_hash_size = 500000000; // Maximum 500M entries in the vocabulary
-
-typedef float real;                    // Precision of float numbers
-
-struct vocab_word {
-  long long cn;
-  char *word;
-};
-
-char train_file[MAX_STRING], output_file[MAX_STRING];
-struct vocab_word *vocab;
-int debug_mode = 2, min_count = 5, *vocab_hash, min_reduce = 1;
-long long vocab_max_size = 10000, vocab_size = 0;
-long long train_words = 0;
-real threshold = 100;
-
-unsigned long long next_random = 1;
-
-// Reads a single word from a file, assuming space + tab + EOL to be word boundaries
-void ReadWord(char *word, FILE *fin) {
-  int a = 0, ch;
-  while (!feof(fin)) {
-    ch = fgetc(fin);
-    if (ch == 13) continue;
-    if ((ch == ' ') || (ch == '\t') || (ch == '\n')) {
-      if (a > 0) {
-        if (ch == '\n') ungetc(ch, fin);
-        break;
-      }
-      if (ch == '\n') {
-        strcpy(word, (char *)"</s>");
-        return;
-      } else continue;
-    }
-    word[a] = ch;
-    a++;
-    if (a >= MAX_STRING - 1) a--;   // Truncate too long words
-  }
-  word[a] = 0;
-}
-
-// Returns hash value of a word
-int GetWordHash(char *word) {
-  unsigned long long a, hash = 1;
-  for (a = 0; a < strlen(word); a++) hash = hash * 257 + word[a];
-  hash = hash % vocab_hash_size;
-  return hash;
-}
-
-// Returns position of a word in the vocabulary; if the word is not found, returns -1
-int SearchVocab(char *word) {
-  unsigned int hash = GetWordHash(word);
-  while (1) {
-    if (vocab_hash[hash] == -1) return -1;
-    if (!strcmp(word, vocab[vocab_hash[hash]].word)) return vocab_hash[hash];
-    hash = (hash + 1) % vocab_hash_size;
-  }
-  return -1;
-}
-
-// Reads a word and returns its index in the vocabulary
-int ReadWordIndex(FILE *fin) {
-  char word[MAX_STRING];
-  ReadWord(word, fin);
-  if (feof(fin)) return -1;
-  return SearchVocab(word);
-}
-
-// Adds a word to the vocabulary
-int AddWordToVocab(char *word) {
-  unsigned int hash, length = strlen(word) + 1;
-  if (length > MAX_STRING) length = MAX_STRING;
-  vocab[vocab_size].word = (char *)calloc(length, sizeof(char));
-  strcpy(vocab[vocab_size].word, word);
-  vocab[vocab_size].cn = 0;
-  vocab_size++;
-  // Reallocate memory if needed
-  if (vocab_size + 2 >= vocab_max_size) {
-    vocab_max_size += 10000;
-    vocab=(struct vocab_word *)realloc(vocab, vocab_max_size * sizeof(struct vocab_word));
-  }
-  hash = GetWordHash(word);
-  while (vocab_hash[hash] != -1) hash = (hash + 1) % vocab_hash_size;
-  vocab_hash[hash]=vocab_size - 1;
-  return vocab_size - 1;
-}
-
-// Used later for sorting by word counts
-int VocabCompare(const void *a, const void *b) {
-    return ((struct vocab_word *)b)->cn - ((struct vocab_word *)a)->cn;
-}
-
-// Sorts the vocabulary by frequency using word counts
-void SortVocab() {
-  int a;
-  unsigned int hash;
-  // Sort the vocabulary and keep </s> at the first position
-  qsort(&vocab[1], vocab_size - 1, sizeof(struct vocab_word), VocabCompare);
-  for (a = 0; a < vocab_hash_size; a++) vocab_hash[a] = -1;
-  for (a = 0; a < vocab_size; a++) {
-    // Words occuring less than min_count times will be discarded from the vocab
-    if (vocab[a].cn < min_count) {
-      vocab_size--;
-      free(vocab[vocab_size].word);
-    } else {
-      // Hash will be re-computed, as after the sorting it is not actual
-      hash = GetWordHash(vocab[a].word);
-      while (vocab_hash[hash] != -1) hash = (hash + 1) % vocab_hash_size;
-      vocab_hash[hash] = a;
-    }
-  }
-  vocab = (struct vocab_word *)realloc(vocab, vocab_size * sizeof(struct vocab_word));
-}
-
-// Reduces the vocabulary by removing infrequent tokens
-void ReduceVocab() {
-  int a, b = 0;
-  unsigned int hash;
-  for (a = 0; a < vocab_size; a++) if (vocab[a].cn > min_reduce) {
-    vocab[b].cn = vocab[a].cn;
-    vocab[b].word = vocab[a].word;
-    b++;
-  } else free(vocab[a].word);
-  vocab_size = b;
-  for (a = 0; a < vocab_hash_size; a++) vocab_hash[a] = -1;
-  for (a = 0; a < vocab_size; a++) {
-    // Hash will be re-computed, as it is not actual
-    hash = GetWordHash(vocab[a].word);
-    while (vocab_hash[hash] != -1) hash = (hash + 1) % vocab_hash_size;
-    vocab_hash[hash] = a;
-  }
-  fflush(stdout);
-  min_reduce++;
-}
-
-void LearnVocabFromTrainFile() {
-  char word[MAX_STRING], last_word[MAX_STRING], bigram_word[MAX_STRING * 2];
-  FILE *fin;
-  long long a, i, start = 1;
-  for (a = 0; a < vocab_hash_size; a++) vocab_hash[a] = -1;
-  fin = fopen(train_file, "rb");
-  if (fin == NULL) {
-    printf("ERROR: training data file not found!\n");
-    exit(1);
-  }
-  vocab_size = 0;
-  AddWordToVocab((char *)"</s>");
-  while (1) {
-    ReadWord(word, fin);
-    if (feof(fin)) break;
-    if (!strcmp(word, "</s>")) {
-      start = 1;
-      continue;
-    } else start = 0;
-    train_words++;
-    if ((debug_mode > 1) && (train_words % 100000 == 0)) {
-      printf("Words processed: %lldK     Vocab size: %lldK  %c", train_words / 1000, vocab_size / 1000, 13);
-      fflush(stdout);
-    }
-    i = SearchVocab(word);
-    if (i == -1) {
-      a = AddWordToVocab(word);
-      vocab[a].cn = 1;
-    } else vocab[i].cn++;
-    if (start) continue;
-    sprintf(bigram_word, "%s_%s", last_word, word);
-    bigram_word[MAX_STRING - 1] = 0;
-    strcpy(last_word, word);
-    i = SearchVocab(bigram_word);
-    if (i == -1) {
-      a = AddWordToVocab(bigram_word);
-      vocab[a].cn = 1;
-    } else vocab[i].cn++;
-    if (vocab_size > vocab_hash_size * 0.7) ReduceVocab();
-  }
-  SortVocab();
-  if (debug_mode > 0) {
-    printf("\nVocab size (unigrams + bigrams): %lld\n", vocab_size);
-    printf("Words in train file: %lld\n", train_words);
-  }
-  fclose(fin);
-}
-
-void TrainModel() {
-  long long pa = 0, pb = 0, pab = 0, oov, i, li = -1, cn = 0;
-  char word[MAX_STRING], last_word[MAX_STRING], bigram_word[MAX_STRING * 2];
-  real score;
-  FILE *fo, *fin;
-  printf("Starting training using file %s\n", train_file);
-  LearnVocabFromTrainFile();
-  fin = fopen(train_file, "rb");
-  fo = fopen(output_file, "wb");
-  word[0] = 0;
-  while (1) {
-    strcpy(last_word, word);
-    ReadWord(word, fin);
-    if (feof(fin)) break;
-    if (!strcmp(word, "</s>")) {
-      fprintf(fo, "\n");
-      continue;
-    }
-    cn++;
-    if ((debug_mode > 1) && (cn % 100000 == 0)) {
-      printf("Words written: %lldK%c", cn / 1000, 13);
-      fflush(stdout);
-    }
-    oov = 0;
-    i = SearchVocab(word);
-    if (i == -1) oov = 1; else pb = vocab[i].cn;
-    if (li == -1) oov = 1;
-    li = i;
-    sprintf(bigram_word, "%s_%s", last_word, word);
-    bigram_word[MAX_STRING - 1] = 0;
-    i = SearchVocab(bigram_word);
-    if (i == -1) oov = 1; else pab = vocab[i].cn;
-    if (pa < min_count) oov = 1;
-    if (pb < min_count) oov = 1;
-    if (oov) score = 0; else score = (pab - min_count) / (real)pa / (real)pb * (real)train_words;
-    if (score > threshold) {
-      fprintf(fo, "_%s", word);
-      pb = 0;
-    } else fprintf(fo, " %s", word);
-    pa = pb;
-  }
-  fclose(fo);
-  fclose(fin);
-}
-
-int ArgPos(char *str, int argc, char **argv) {
-  int a;
-  for (a = 1; a < argc; a++) if (!strcmp(str, argv[a])) {
-    if (a == argc - 1) {
-      printf("Argument missing for %s\n", str);
-      exit(1);
-    }
-    return a;
-  }
-  return -1;
-}
-
-int main(int argc, char **argv) {
-  int i;
-  if (argc == 1) {
-    printf("WORD2PHRASE tool v0.1a\n\n");
-    printf("Options:\n");
-    printf("Parameters for training:\n");
-    printf("\t-train <file>\n");
-    printf("\t\tUse text data from <file> to train the model\n");
-    printf("\t-output <file>\n");
-    printf("\t\tUse <file> to save the resulting word vectors / word clusters / phrases\n");
-    printf("\t-min-count <int>\n");
-    printf("\t\tThis will discard words that appear less than <int> times; default is 5\n");
-    printf("\t-threshold <float>\n");
-    printf("\t\t The <float> value represents threshold for forming the phrases (higher means less phrases); default 100\n");
-    printf("\t-debug <int>\n");
-    printf("\t\tSet the debug mode (default = 2 = more info during training)\n");
-    printf("\nExamples:\n");
-    printf("./word2phrase -train text.txt -output phrases.txt -threshold 100 -debug 2\n\n");
-    return 0;
-  }
-  if ((i = ArgPos((char *)"-train", argc, argv)) > 0) strcpy(train_file, argv[i + 1]);
-  if ((i = ArgPos((char *)"-debug", argc, argv)) > 0) debug_mode = atoi(argv[i + 1]);
-  if ((i = ArgPos((char *)"-output", argc, argv)) > 0) strcpy(output_file, argv[i + 1]);
-  if ((i = ArgPos((char *)"-min-count", argc, argv)) > 0) min_count = atoi(argv[i + 1]);
-  if ((i = ArgPos((char *)"-threshold", argc, argv)) > 0) threshold = atof(argv[i + 1]);
-  vocab = (struct vocab_word *)calloc(vocab_max_size, sizeof(struct vocab_word));
-  vocab_hash = (int *)calloc(vocab_hash_size, sizeof(int));
-  TrainModel();
-  return 0;
-}