diff options
author | mfederico <mfederico@1f5c12ca-751b-0410-a591-d2e778427230> | 2006-08-12 02:22:23 +0400 |
---|---|---|
committer | mfederico <mfederico@1f5c12ca-751b-0410-a591-d2e778427230> | 2006-08-12 02:22:23 +0400 |
commit | 1898ce03b47c1f25ea72d139c526b256712408a6 (patch) | |
tree | 0006c673d1c1f199aa41c906505a34892f429238 /irstlm | |
parent | e6914693a147b35866f052ec2f0719619daa6045 (diff) |
No log message.
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@669 1f5c12ca-751b-0410-a591-d2e778427230
Diffstat (limited to 'irstlm')
-rw-r--r-- | irstlm/src/lmtable.cpp | 17 | ||||
-rw-r--r-- | irstlm/src/lmtable.h | 27 | ||||
-rw-r--r-- | irstlm/src/ngramcache.cpp | 8 |
3 files changed, 28 insertions, 24 deletions
diff --git a/irstlm/src/lmtable.cpp b/irstlm/src/lmtable.cpp index e3677eed9..05593a976 100644 --- a/irstlm/src/lmtable.cpp +++ b/irstlm/src/lmtable.cpp @@ -49,7 +49,8 @@ lmtable::lmtable(){ memset(info, 0, sizeof(info)); memset(NumCenters, 0, sizeof(NumCenters)); - bicache=NULL; + max_cache_lev=0; + for (int i=0;i<=maxlev;i++) lmtcache[i]=NULL; probcache=NULL; statecache=NULL; @@ -575,7 +576,7 @@ int lmtable::get(ngram& ng,int n,int lev){ //if (l==2) cout <<"bicache: searching:" << ng <<"\n"; - if (bicache && l==2 && bicache->get(ng.wordp(n),(char *)&found)) + if (lmtcache[l] && lmtcache[l]->get(ng.wordp(n),(char *)&found)) hit=1; else search(table[l] + (offset * nodesize(ndt)), @@ -587,13 +588,11 @@ int lmtable::get(ngram& ng,int n,int lev){ LMT_FIND, &found); - if (bicache && l==2 && hit==0){ - //if (bicache->isfull()) bicache->reset(); - //cout << "bicache :" << ng <<"\n"; - bicache->add(ng.wordp(n),(char *)&found); - } - - if (!found) return 0; + //insert both found and not found items!!! + if (lmtcache[l] && hit==0) + lmtcache[l]->add(ng.wordp(n),(char *)&found); + + if (!found) return 0; ng.link=found; ng.info=ndt; diff --git a/irstlm/src/lmtable.h b/irstlm/src/lmtable.h index c06b5de63..32a291d45 100644 --- a/irstlm/src/lmtable.h +++ b/irstlm/src/lmtable.h @@ -136,9 +136,10 @@ class lmtable{ int backoff_state; //improve access speed - ngramcache* bicache; + ngramcache* lmtcache[LMTMAXLEV+1]; ngramcache* probcache; ngramcache* statecache; + int max_cache_lev; public: @@ -147,10 +148,12 @@ public: lmtable(); ~lmtable(){ - if (bicache){ - std::cerr << "Bigram Cache: "; bicache->stat(); - delete bicache; + for (int i=2;i<=max_cache_lev;i++) + if (lmtcache[i]){ + std::cerr << i <<"-gram cache: "; lmtcache[i]->stat(); + delete lmtcache[i]; } + if (probcache){ std::cerr << "Prob Cache: "; probcache->stat(); delete probcache; @@ -181,22 +184,24 @@ public: statecache=new ngramcache(maxlev-1,sizeof(char *),200000); } - void init_bicache(){ - assert(bicache==NULL); - bicache=new ngramcache(2,sizeof(char *),200000); + void init_lmtcaches(int uptolev){ + max_cache_lev=uptolev; + for (int i=2;i<=max_cache_lev;i++){ + assert(lmtcache[i]==NULL); + lmtcache[i]=new ngramcache(i,sizeof(char *),2000000); + } } - void check_cache_levels(){ if (probcache && probcache->isfull()) probcache->reset(); if (statecache && statecache->isfull()) statecache->reset(); - if (bicache && bicache->isfull()) bicache->reset(); + for (int i=2;i<=max_cache_lev;i++) + if (lmtcache[i]->isfull()) lmtcache[i]->reset(); } bool is_probcache_active(){return probcache!=NULL;} bool is_statecache_active(){return statecache!=NULL;} - bool is_bicache_active(){return bicache!=NULL;} - + bool are_lmtcaches_active(){return lmtcache[2]!=NULL;} void configure(int n,bool quantized){ maxlev=n; diff --git a/irstlm/src/ngramcache.cpp b/irstlm/src/ngramcache.cpp index ea8e309c2..3b9ea1e2f 100644 --- a/irstlm/src/ngramcache.cpp +++ b/irstlm/src/ngramcache.cpp @@ -34,8 +34,8 @@ ngramcache::ngramcache(int n,int size,int maxentries){ infosize=size; maxn=maxentries; entries=0; - ht=new htable(maxn * 2, ngsize * sizeof(int),INT,NULL); //load factor 2 - mp=new mempool(ngsize * sizeof(int)+infosize,maxn/10); + ht=new htable(maxn * 5, ngsize * sizeof(int),INT,NULL); //lower load factor to reduce collisions + mp=new mempool(ngsize * sizeof(int)+infosize,maxn/5); accesses=0; hits=0; }; @@ -50,8 +50,8 @@ ngramcache::~ngramcache(){ void ngramcache::reset(){ ht->stat(); delete ht;delete mp; - ht=new htable(maxn * 2, ngsize * sizeof(int),INT,NULL); //load factor 2 - mp=new mempool(ngsize * sizeof(int)+infosize,maxn/10); + ht=new htable(maxn * 5, ngsize * sizeof(int),INT,NULL); //load factor 2 + mp=new mempool(ngsize * sizeof(int)+infosize,maxn/5); entries=0; } |