path: root/mgizapp/src/model1.cpp

/*

EGYPT Toolkit for Statistical Machine Translation
Written by Yaser Al-Onaizan, Jan Curin, Michael Jahr, Kevin Knight, John Lafferty, Dan Melamed, David Purdy, Franz Och, Noah Smith, and David Yarowsky.

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
USA.

*/
#include "model1.h"
#include "Globals.h"
#include "utility.h"
#include "Parameter.h"

extern short NoEmptyWord;
extern int VerboseSentence;

extern short NCPUS;

GLOBAL_PARAMETER2(int,Model1_Dump_Freq,"MODEL 1 DUMP FREQUENCY","t1","dump frequency of Model 1",PARLEV_OUTPUT,0);
int NumberOfVALIalignments=100;

model1::model1(
  const char* efname,
  vcbList& evcblist,
  vcbList& fvcblist,
  tmodel<COUNT, PROB> &_tTable,
  Perplexity & _perp,
  sentenceHandler& _sHandler1,
  Perplexity* _testPerp,
  sentenceHandler* _testHandler,
  Perplexity& _trainViterbiPerp,
  Perplexity* _testViterbiPerp)
  :
  report_info(_perp,_sHandler1,_testPerp,_testHandler,_trainViterbiPerp,_testViterbiPerp),
  efFilename(efname), Elist(evcblist), Flist(fvcblist),
  eTotalWCount(Elist.totalVocab()), fTotalWCount(Flist.totalVocab()),
  noEnglishWords(Elist.size()), noFrenchWords(Flist.size()), tTable(_tTable),
  evlist(Elist.getVocabList()), fvlist(Flist.getVocabList())
{}

model1::model1(const model1& m1, int _threadID) :
  report_info(m1),
  efFilename(m1.efFilename),
  Elist(m1.Elist),
  Flist(m1.Flist),
  eTotalWCount(m1.eTotalWCount),
  fTotalWCount(m1.fTotalWCount),
  noEnglishWords(m1.noEnglishWords),
  noFrenchWords(m1.noFrenchWords),
  tTable(m1.tTable),
  evlist(m1.evlist),
  fvlist(m1.fvlist)
{}

void model1::initialize_table_uniformly(sentenceHandler& sHandler1)
{
  WordIndex i, j;

  cout << "Initialize tTable\n";

  sentPair sent ;
  sHandler1.rewind();
  while(sHandler1.getNextSentence(sent)) {
    Vector<WordIndex>& es = sent.eSent;
    Vector<WordIndex>& fs = sent.fSent;
    PROB uniform = 1.0/es.size() ;
    for( i=0; i < es.size(); i++)
      for(j=1; j < fs.size(); j++)
        tTable.insert(es[i],fs[j],0,uniform);
  }
}

struct em_loop_t {
  model1 *m1;
  int it;
  int nthread;
  Dictionary *dict;
  bool useDict;
  int result;
  pthread_t thread;
  int valid ;
};

void* exe_emloop(void *arg)
{
  em_loop_t* em =(em_loop_t *) arg;
  em->result = em->m1->em_thread(em->it,em->nthread,*em->dict,em->useDict);
  return arg;
}

int model1::em_thread(int noIterations, int nthread, /*Perplexity& perp, sentenceHandler& sHandler1, */
                      Dictionary& dictionary, bool useDict /*Perplexity* testPerp, sentenceHandler* testHandler,
										     Perplexity& trainViterbiPerp, Perplexity* testViterbiPerp */ )
{
  string modelName="Model1", shortModelName="1";
  time_t st = time(NULL);
  string tfile, number, alignfile, test_alignfile;
  bool dump_files = false ;
  cout << "==========================================================\n";
  cout << modelName << " Training Started at: "<< my_ctime(&st) << "\n";
  int it = noIterations;
  cout <<  "-----------\n" << modelName << ": Iteration " << it << '\n';
  dump_files = (Model1_Dump_Freq != 0) &&  ((it % Model1_Dump_Freq)  == 0 || noIterations == it) && !NODUMPS ;
//    dump_files = true;
  number = "";
  int n = it;
  do {
    number.insert((size_t)0, (size_t)1, (char)(n % 10 + '0'));
  } while((n /= 10) > 0);
  alignfile = Prefix + ".A" + shortModelName + "." + number + ".part" ;
  alignfile = alignfile + represent_number(nthread, 3);

  em_loop(it,perp, sHandler1, false, dump_files, alignfile.c_str(), dictionary, useDict, trainViterbiPerp);
  return 0;
}

int model1::em_with_tricks(int noIterations, /*Perplexity& perp, sentenceHandler& sHandler1, */
                           bool seedModel1, Dictionary& dictionary, bool useDict /*Perplexity* testPerp, sentenceHandler* testHandler,
										     Perplexity& trainViterbiPerp, Perplexity* testViterbiPerp */
                           , bool dumpCount ,  const char* dumpCountName, bool useString)  // If specified, then will dump files before last iteration
{
  double minErrors=1.0;
  int minIter=0;
  string modelName="Model1",shortModelName="1";
  time_t st, it_st, fn, it_fn;
  string tfile, number, alignfile, test_alignfile;
  bool dump_files = false ;
  st = time(NULL);
  sHandler1.rewind();
  cout << "==========================================================\n";
  cout << modelName << " Training Started at: "<< my_ctime(&st) << "\n";
  for(int it = 1; it <= noIterations; it++) {
    it_st = time(NULL);
    cout <<  "-----------\n" << modelName << ": Iteration " << it << '\n';
    dump_files = (Model1_Dump_Freq != 0) &&  ((it % Model1_Dump_Freq)  == 0 || it == noIterations) && !NODUMPS ;
    //dump_files = true;
    number = "";
    int n = it;
    do {
      number.insert((size_t)0, (size_t)1, (char)(n % 10 + '0'));
    } while((n /= 10) > 0);
    tfile = Prefix + ".t" + shortModelName + "." + number ;
    alignfile = Prefix + ".A" + shortModelName + "." + number+".part0" ;
    test_alignfile = Prefix +".tst.A" + shortModelName + "." + number ;
    initAL();
    threadID = 0;
    int th;
    vector<em_loop_t> ths;
    ths.resize(NCPUS);
    sHandler1.rewind();
    for (th=1; th<NCPUS; th++) {
      ths[th].m1=this;
      ths[th].it = it;
      ths[th].nthread = th;
      ths[th].dict = & dictionary;
      ths[th].useDict = useDict;
      ths[th].result = 0;
      ths[th].valid = pthread_create(&(ths[th].thread),NULL,exe_emloop,&(ths[th]));
      if(ths[th].valid) {
        cerr << "Error starting thread " << th << endl;
      }
    }
    em_loop(it,perp, sHandler1, seedModel1, dump_files, alignfile.c_str(), dictionary, useDict, trainViterbiPerp);
    perp.record("Model1");
    trainViterbiPerp.record("Model1");
    errorReportAL(cout, "IBM-1");

    cerr << "Main thread done, waiting" << endl;;
    for (th=1; th<NCPUS; th++) {
      pthread_join((ths[th].thread),NULL);
      cerr << "Thread " << th << "done" << endl;
    }
    if (testPerp && testHandler) // calculate test perplexity
      em_loop(it,*testPerp, *testHandler, seedModel1, dump_files, test_alignfile.c_str(), dictionary, useDict, *testViterbiPerp, true);
    if( errorsAL()<minErrors ) {
      minErrors=errorsAL();
      minIter=it;
    }
    //if (dump_files){
    //    if( OutputInAachenFormat==1 )
    //        tTable.printCountTable(tfile.c_str(),Elist.getVocabList(),Flist.getVocabList(),1);
    //}
    cerr << "Normalizing T " << endl;

    /**
     If asked for dumping count table, just dump it.
     */
    if(dumpCount && it == noIterations) {
      string realTableName = dumpCountName;
      realTableName += ".t.count";
      tTable.printCountTable(realTableName.c_str(),Elist.getVocabList(),Flist.getVocabList(),useString);
    }

    tTable.normalizeTable(Elist, Flist);
    //cout << tTable.getProb(2,2) << endl;
    cerr << " DONE Normalizing " << endl;
    cout << modelName << ": ("<<it<<") TRAIN CROSS-ENTROPY " << perp.cross_entropy()
         << " PERPLEXITY " << perp.perplexity() << '\n';
    if (testPerp && testHandler)
      cout << modelName << ": ("<<it<<") TEST CROSS-ENTROPY " << (*testPerp).cross_entropy()
           << " PERPLEXITY " << (*testPerp).perplexity()
           << '\n';
    cout << modelName << ": ("<<it<<") VITERBI TRAIN CROSS-ENTROPY " << trainViterbiPerp.cross_entropy()
         << " PERPLEXITY " << trainViterbiPerp.perplexity() << '\n';
    if (testPerp && testHandler)
      cout << modelName << ": ("<<
           it<<") VITERBI TEST CROSS-ENTROPY "
           << (*testViterbiPerp).cross_entropy()
           << " PERPLEXITY " << (*testViterbiPerp).perplexity()
           << '\n';
    if (dump_files) {
      if( OutputInAachenFormat==0 )
        tTable.printProbTable(tfile.c_str(),Elist.getVocabList(),
                              Flist.getVocabList(),OutputInAachenFormat);
    }
    it_fn = time(NULL);
    cout << "Model 1 Iteration: " << it<< " took: " << difftime(it_fn, it_st) << " seconds\n";


  }
  fn = time(NULL) ;
  cout <<  "Entire " << modelName << " Training took: " << difftime(fn, st) << " seconds\n";
  return minIter;
}

bool model1::load_table(const char* tname)
{
  /* This function loads the t table from the given file; use it
     when you want to load results from previous t training
     without doing any new training.
     NAS, 7/11/99
  */
  cout << "Model1: loading t table \n" ;
  return tTable.readProbTable(tname);
}


extern float MINCOUNTINCREASE;
void model1::em_loop(int it,Perplexity& perp, sentenceHandler& sHandler1, bool seedModel1,
                     bool dump_alignment, const char* alignfile, Dictionary& dict, bool useDict, Perplexity& viterbi_perp, bool test)
{
  WordIndex i, j, l, m ;
  double cross_entropy;
  int pair_no=0 ;
  perp.clear();
  viterbi_perp.clear();
  ofstream of2;
  // for each sentence pair in the corpus
  if (dump_alignment||FEWDUMPS)
    of2.open(alignfile);
  PROB uniform = 1.0/noFrenchWords ;
  sentPair sent ;

  while(sHandler1.getNextSentence(sent)) {
    Vector<WordIndex>& es = sent.eSent;
    Vector<WordIndex>& fs = sent.fSent;
    const float so  = sent.getCount();
    l = es.size() - 1;
    m = fs.size() - 1;
    cross_entropy = log(1.0);
    Vector<WordIndex> viterbi_alignment(fs.size());
    double viterbi_score = 1 ;

#ifdef WIN32
    bool *eindict = new bool[l + 1];
    bool *findict = new bool[m + 1];
    bool **indict  = new bool*[m + 1];
    for(int _i = 0; _i < m+1; _i++)
      indict[_i] = new bool[l + 1];
#else
    bool eindict[l + 1];
    bool findict[m + 1];
    bool indict[m + 1][l + 1];
#endif
    if(it == 1 && useDict) {
      for(unsigned int dummy = 0; dummy <= l; dummy++) eindict[dummy] = false;
      for(unsigned int dummy = 0; dummy <= m; dummy++) {
        findict[dummy] = false;
        for(unsigned int dummy2 = 0; dummy2 <= l; dummy2++)
          indict[dummy][dummy2] = false;
      }
      for(j = 0; j <= m; j++)
        for(i = 0; i <= l; i++)
          if(dict.indict(fs[j], es[i])) {
            eindict[i] = findict[j] = indict[j][i] = true;
          }
    }

    for(j=1; j <= m; j++) {
      // entries  that map fs to all possible ei in this sentence.
      Vector<LpPair<COUNT,PROB> *> sPtrCache(es.size(),0); // cache pointers to table
      LpPair<COUNT,PROB> **sPtrCachePtr;

      PROB denom = 0.0;
      WordIndex best_i = 0 ; // i for which fj is best maped to ei
      PROB word_best_score = 0 ;  // score for the best mapping of fj
      if (it == 1 && !seedModel1) {
        denom = uniform  * es.size() ;
        word_best_score = uniform ;
      } else
        for((i=0),(sPtrCachePtr=&sPtrCache[0]); i <= l; i++,sPtrCachePtr++) {
          PROB e(0.0) ;
          (*sPtrCachePtr) = tTable.getPtr(es[i], fs[j]) ;
          if ((*sPtrCachePtr) != 0 && (*((*sPtrCachePtr))).prob > PROB_SMOOTH)
            e = (*((*sPtrCachePtr))).prob;
          else e = PROB_SMOOTH ;
          denom += e  ;
          if (e > word_best_score) {
            word_best_score = e ;
            best_i = i ;
          }
        }
      viterbi_alignment[j] = best_i ;
      viterbi_score *= word_best_score ; /// denom ;
      if (denom == 0) {
        if (test)
          cerr << "WARNING: denom is zero (TEST)\n";
        else
          cerr << "WARNING: denom is zero (TRAIN)\n";
      }
      cross_entropy += log(denom) ;
      if (!test) {
        if(denom > 0) {
          COUNT val = COUNT(so) / (COUNT) double(denom) ;
          /* this if loop implements a constraint on counting:
          count(es[i], fs[j]) is implemented if and only if
          es[i] and fs[j] occur together in the dictionary,
          OR
          es[i] does not occur in the dictionary with any fs[x] and
          fs[j] does not occur in the dictionary with any es[y]
          */
          if(it == 1 && useDict) {
            for((i=0),(sPtrCachePtr=&sPtrCache[0]); i <= l; i++,sPtrCachePtr++) {
              if(indict[j][i] || (!findict[j] && !eindict[i])) {
                PROB e(0.0) ;
                if (it == 1 && !seedModel1)
                  e =  uniform  ;
                else if ((*sPtrCachePtr) != 0 &&  (*((*sPtrCachePtr))).prob > PROB_SMOOTH)
                  e = (*((*sPtrCachePtr))).prob;
                else e = PROB_SMOOTH ;
                COUNT x=e*val;
                if( (it==1 && !seedModel1)||x>MINCOUNTINCREASE )
                  /*    if ((*sPtrCachePtr) != 0)
                      (*((*sPtrCachePtr))).count += x;
                  else 	      */
                  tTable.incCount(es[i], fs[j], x);
              } /* end of if */
            } /* end of for i */
          } /* end of it == 1 */
          // Old code:
          else {
            for((i=0),(sPtrCachePtr=&sPtrCache[0]); i <= l; i++,sPtrCachePtr++) {
              //for(i=0; i <= l; i++) {
              PROB e(0.0) ;
              if (it == 1 && !seedModel1)
                e =  uniform  ;
              else if ((*sPtrCachePtr) != 0 &&  (*((*sPtrCachePtr))).prob > PROB_SMOOTH)
                e = (*((*sPtrCachePtr))).prob;
              else e = PROB_SMOOTH ;
              //if( !(i==0) )
              //cout << "COUNT(e): " << e << " " << MINCOUNTINCREASE << endl;
              COUNT x=e*val;
              if( pair_no==VerboseSentence )
                cout << i << "(" << evlist[es[i]].word << ")," << j << "(" << fvlist[fs[j]].word << ")=" << x << endl;
              if( (it==1 && !seedModel1)||x>MINCOUNTINCREASE ) {
                /*if( NoEmptyWord==0 || i!=0 )
                if ((*sPtrCachePtr) != 0)
                (*((*sPtrCachePtr))).count += x;
                else */
                //cerr << i << " " << j << " (+) " << endl;
                //cerr.flush();
                //cerr << es[i] << " " << fs[j] << " (=) "<< endl;
                //cerr.flush();
                tTable.incCount(es[i], fs[j], x);
                //cerr << es[i] << " " << fs[j] << " (-) "<< endl;
                //cerr.flush();
              }
            } /* end of for i */
          } // end of else
        } // end of if (denom > 0)
      }// if (!test)
    } // end of for (j) ;
    sHandler1.setProbOfSentence(sent,cross_entropy);
    //cerr << sent << "CE: " << cross_entropy << " " << so << endl;
    perp.addFactor(cross_entropy-m*log(l+1.0), so, l, m,1);
    viterbi_perp.addFactor(log(viterbi_score)-m*log(l+1.0), so, l, m,1);
    if (dump_alignment||(FEWDUMPS&&sent.sentenceNo<1000))
      printAlignToFile(es, fs, evlist, fvlist, of2, viterbi_alignment, sent.sentenceNo, viterbi_score);
    addAL(viterbi_alignment,sent.sentenceNo,l);
    pair_no++;
#ifdef WIN32
    delete[] eindict;
    delete[] findict;
    for(int _i = 0; _i < m+1; _i++)
      delete[] indict[_i];
    delete[] indict;
#endif
  } /* of while */
}

CTTableDiff<COUNT,PROB>* model1::one_step_em(int it, bool seedModel1,
    Dictionary& dictionary, bool useDict)
{
  CTTableDiff<COUNT,PROB> *diff = new CTTableDiff<COUNT,PROB>();
  double minErrors=1.0;
  string modelName="Model1",shortModelName="1";
  time_t st, it_st, fn;
  string tfile, number, alignfile, test_alignfile;
  bool dump_files = false ;
  st = time(NULL);
  sHandler1.rewind();
  cout << "==========================================================\n";
  cout << modelName << " Training Started at: "<< my_ctime(&st) << "\n";
  it_st = time(NULL);
  cout <<  "-----------\n" << modelName << ": Iteration " << it << '\n';
  dump_files = (Model1_Dump_Freq != 0) &&  ((it % Model1_Dump_Freq)  == 0) && !NODUMPS ;
  number = "";
  int n = it;
  do {
    number.insert((size_t)0, (size_t)1, (char)(n % 10 + '0'));
  } while((n /= 10) > 0);
  tfile = Prefix + ".t" + shortModelName + "." + number ;
  alignfile = Prefix + ".A1" ;
  test_alignfile = Prefix +".tst.A" + shortModelName + "." + number ;
  initAL();
  em_loop_1(diff,it,perp, sHandler1, seedModel1,
            dump_files, alignfile.c_str(), dictionary, useDict, trainViterbiPerp);
  //if (testPerp && testHandler) // calculate test perplexity
  //    em_loop(it,*testPerp, *testHandler, seedModel1, dump_files, test_alignfile.c_str(), dictionary, useDict, *testViterbiPerp, true);
  if( errorsAL()<minErrors ) minErrors=errorsAL();
  fn = time(NULL) ;
  cout <<  "Partial " << modelName << " Training took: " << difftime(fn, it_st) << " seconds\n";
  return diff;
}

void model1::combine_one(CTTableDiff<COUNT,PROB>* cb)
{
  cb->AugmentTTable(tTable);
}

void model1::recombine()
{
  tTable.normalizeTable(Elist, Flist);
}

void save_table(const char* tname)
{
  /*         if (dump_files){
   *             if( OutputInAachenFormat==0 )
   *                 tTable.printProbTable(tfile.c_str(),Elist.getVocabList(),Flist.getVocabList(),OutputInAachenFormat);
   */

}


void model1::em_loop_1(CTTableDiff<COUNT,PROB> *diff,int it,Perplexity& perp, sentenceHandler& sHandler1, bool seedModel1,
                       bool dump_alignment, const char* alignfile, Dictionary& dict, bool useDict, Perplexity& viterbi_perp, bool test)
{
  WordIndex i, j, l, m ;
  double cross_entropy;
  int pair_no=0 ;
  perp.clear();
  viterbi_perp.clear();
  ofstream of2;
  // for each sentence pair in the corpus
  if (dump_alignment||FEWDUMPS)
    of2.open(alignfile);
  PROB uniform = 1.0/noFrenchWords ;
  sentPair sent ;
  sHandler1.rewind();
  while(sHandler1.getNextSentence(sent)) {
    Vector<WordIndex>& es = sent.eSent;
    Vector<WordIndex>& fs = sent.fSent;
    const float so  = sent.getCount();
    l = es.size() - 1;
    m = fs.size() - 1;
    cross_entropy = log(1.0);
    Vector<WordIndex> viterbi_alignment(fs.size());
    double viterbi_score = 1 ;

#ifdef WIN32
    bool *eindict = new bool[l + 1];
    bool *findict = new bool[m + 1];
    bool **indict  = new bool*[m + 1];
    for(int _i = 0; _i < m+1; _i++)
      indict[_i] = new bool[l + 1];
#else
    bool eindict[l + 1];
    bool findict[m + 1];
    bool indict[m + 1][l + 1];
#endif
    if(it == 1 && useDict) {
      for(unsigned int dummy = 0; dummy <= l; dummy++) eindict[dummy] = false;
      for(unsigned int dummy = 0; dummy <= m; dummy++) {
        findict[dummy] = false;
        for(unsigned int dummy2 = 0; dummy2 <= l; dummy2++)
          indict[dummy][dummy2] = false;
      }
      for(j = 0; j <= m; j++)
        for(i = 0; i <= l; i++)
          if(dict.indict(fs[j], es[i])) {
            eindict[i] = findict[j] = indict[j][i] = true;
          }
    }

    for(j=1; j <= m; j++) {
      // entries  that map fs to all possible ei in this sentence.
      Vector<LpPair<COUNT,PROB> *> sPtrCache(es.size(),0); // cache pointers to table
      //Vector<COUNT *> sPtrCacheDif(es.size(),0); // cache pointers to table
      LpPair<COUNT,PROB> **sPtrCachePtr;
      //COUNT **sPtrCachePtrDif;

      PROB denom = 0.0;
      WordIndex best_i = 0 ; // i for which fj is best maped to ei
      PROB word_best_score = 0 ;  // score for the best mapping of fj
      if (it == 1 && !seedModel1) {
        denom = uniform  * es.size() ;
        word_best_score = uniform ;
      } else {
        for((i=0),(sPtrCachePtr=&sPtrCache[0]); i <= l; i++,sPtrCachePtr++) {
          PROB e(0.0) ;
          (*sPtrCachePtr) = tTable.getPtr(es[i], fs[j]) ;
          //(*sPtrCachePtrDif) = diff->GetPtr(es[i], fs[j]) ;
          if ((*sPtrCachePtr) != 0 && (*((*sPtrCachePtr))).prob > PROB_SMOOTH)
            e = (*((*sPtrCachePtr))).prob;
          else e = PROB_SMOOTH ;
          denom += e  ;
          if (e > word_best_score) {
            word_best_score = e ;
            best_i = i ;
          }
        }
      }
      viterbi_alignment[j] = best_i ;
      viterbi_score *= word_best_score ; /// denom ;
      if (denom == 0) {
        if (test)
          cerr << "WARNING: denom is zero (TEST)\n";
        else
          cerr << "WARNING: denom is zero (TRAIN)\n";
      }
      cross_entropy += log(denom) ;
      if (!test) {
        if(denom > 0) {
          COUNT val = COUNT(so) / (COUNT) double(denom) ;
          /* this if loop implements a constraint on counting:
          count(es[i], fs[j]) is implemented if and only if
          es[i] and fs[j] occur together in the dictionary,
          OR
          es[i] does not occur in the dictionary with any fs[x] and
          fs[j] does not occur in the dictionary with any es[y]
          */
          if(it == 1 && useDict) {
            for((i=0),(sPtrCachePtr=&sPtrCache[0]);
                i <= l; i++,sPtrCachePtr++) {
              if(indict[j][i] || (!findict[j] && !eindict[i])) {
                PROB e(0.0) ;
                if (it == 1 && !seedModel1)
                  e =  uniform  ;
                else if ((*sPtrCachePtr) != 0 &&  (*((*sPtrCachePtr))).prob > PROB_SMOOTH)
                  e = (*((*sPtrCachePtr))).prob;
                else e = PROB_SMOOTH ;
                COUNT x=e*val;
                if( it==1||x>MINCOUNTINCREASE ) {
                  /*if ((*sPtrCachePtr) != 0){
                      (*((*sPtrCachePtr))).count += x;
                  } else {*/
                  tTable.incCount(es[i], fs[j], x);
                  //}
                  diff->incCount(es[i], fs[j], x);
                }
              } /* end of if */
            } /* end of for i */
          } /* end of it == 1 */
          // Old code:
          else {
            for((i=0),(sPtrCachePtr=&sPtrCache[0]); i <= l; i++,sPtrCachePtr++) {
              //for(i=0; i <= l; i++) {
              PROB e(0.0) ;
              if (it == 1 && !seedModel1)
                e =  uniform  ;
              else if ((*sPtrCachePtr) != 0 &&  (*((*sPtrCachePtr))).prob > PROB_SMOOTH)
                e = (*((*sPtrCachePtr))).prob;
              else e = PROB_SMOOTH ;
              //if( !(i==0) )
              //cout << "COUNT(e): " << e << " " << MINCOUNTINCREASE << endl;
              COUNT x=e*val;
              if( pair_no==VerboseSentence )
                cout << i << "(" << evlist[es[i]].word << "),"
                     << j << "(" << fvlist[fs[j]].word << ")=" << x << endl;
              if( it==1||x>MINCOUNTINCREASE )
                if( NoEmptyWord==0 || ( NoEmptyWord==0 || i!=0 )) {
                  /*if ((*sPtrCachePtr) != 0){
                      (*((*sPtrCachePtr))).count += x;
                  } else 	      */
                  tTable.incCount(es[i], fs[j], x);
                  diff->incCount(es[i], fs[j], x);
                }
            } /* end of for i */
          } // end of else
        } // end of if (denom > 0)
      }// if (!test)
    } // end of for (j) ;
    sHandler1.setProbOfSentence(sent,cross_entropy);
    //cerr << sent << "CE: " << cross_entropy << " " << so << endl;
    perp.addFactor(cross_entropy-m*log(l+1.0), so, l, m,1);
    viterbi_perp.addFactor(log(viterbi_score)-m*log(l+1.0), so, l, m,1);
    if (dump_alignment||(FEWDUMPS&&sent.sentenceNo<1000))
      printAlignToFile(es, fs, evlist, fvlist, of2, viterbi_alignment, sent.sentenceNo, viterbi_score);
    addAL(viterbi_alignment,sent.sentenceNo,l);
    pair_no++;
#ifdef WIN32
    delete[] eindict;
    delete[] findict;
    for(int _i = 0; _i < m+1; _i++)
      delete[] indict[_i];
    delete[] indict;
#endif
  } /* of while */
  sHandler1.rewind();
  perp.record("Model1");
  viterbi_perp.record("Model1");
  errorReportAL(cout, "IBM-1");

}