blob: 92e7eb25e0de53edef13628872e253a950398e8c (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
#pragma once
#include "marian.h"
#include "generic.h"
#include "logits.h"
#include "data/shortlist.h"
#include "layers/factory.h"
namespace marian {
class LSH;
namespace mlp {
class Output : public LayerBase, public IUnaryLogitLayer, public IHasShortList {
private:
// parameters held by this layer
Expr Wt_; // weight matrix is stored transposed for efficiency
Expr b_;
Expr lemmaEt_; // re-embedding matrix for lemmas [lemmaDimEmb x lemmaVocabSize]
bool isLegacyUntransposedW{false}; // legacy-model emulation: W is stored in non-transposed form
bool hasBias_{true};
Expr cachedShortWt_; // short-listed version, cached (cleared by clear())
Expr cachedShortb_; // these match the current value of shortlist_
Expr cachedShortLemmaEt_;
Ptr<FactoredVocab> factoredVocab_;
// optional parameters set/updated after construction
Expr tiedParam_;
Ptr<data::Shortlist> shortlist_;
Ptr<LSH> lsh_;
void lazyConstruct(int inputDim);
public:
Output(Ptr<ExpressionGraph> graph, Ptr<Options> options)
: LayerBase(graph, options),
hasBias_{!options->get<bool>("output-omit-bias", false)} {
clear();
}
void tieTransposed(Expr tied) {
if (Wt_)
ABORT_IF(tiedParam_.get() != tied.get(), "Tied output projection cannot be changed once weights have been created");
else
tiedParam_ = tied;
}
void setShortlist(Ptr<data::Shortlist> shortlist) override final {
if (shortlist_)
ABORT_IF(shortlist.get() != shortlist_.get(), "Output shortlist cannot be changed except after clear()");
else {
ABORT_IF(cachedShortWt_ || cachedShortb_ || cachedShortLemmaEt_, "No shortlist but cached parameters??");
shortlist_ = shortlist;
}
// cachedShortWt_ and cachedShortb_ will be created lazily inside apply()
}
// this is expected to be called in sync with graph->clear(), which invalidates
// cachedShortWt_ etc. in the graph's short-term cache
void clear() override final {
shortlist_ = nullptr;
cachedShortWt_ = nullptr;
cachedShortb_ = nullptr;
cachedShortLemmaEt_ = nullptr;
}
Logits applyAsLogits(Expr input) override final;
};
} // namespace mlp
}
|
