diff options
author | Soumith Chintala <soumith@gmail.com> | 2015-08-08 19:55:00 +0300 |
---|---|---|
committer | Soumith Chintala <soumith@gmail.com> | 2015-08-08 19:55:00 +0300 |
commit | 8551a3706f2bc5a34d19c3b6e3a645219646b566 (patch) | |
tree | 99d2b673ba420dfa1a02e1dfb02656e306b6d1af | |
parent | e3452817c8b4dd9a5d0836c2bb1b4e46641c6f60 (diff) | |
parent | 5c081a12bf27abd9851d14ea981ab7b738f1c9f0 (diff) |
Merge pull request #61 from kashif/adam
remove redundant lambda param from adam
-rw-r--r-- | adam.lua | 9 |
1 files changed, 3 insertions, 6 deletions
@@ -10,7 +10,6 @@ ARGS: - 'config.beta1' : first moment coefficient - 'config.beta2' : second moment coefficient - 'config.epsilon' : for numerical stability -- 'config.lambda' : first moment decay - 'state' : a table describing the state of the optimizer; after each call the state is modified @@ -29,7 +28,6 @@ function optim.adam(opfunc, x, config, state) local beta1 = config.beta1 or 0.9 local beta2 = config.beta2 or 0.999 local epsilon = config.epsilon or 1e-8 - local lambda = config.lambda or 1-1e-8 -- (1) evaluate f(x) and df/dx local fx, dfdx = opfunc(x) @@ -44,10 +42,9 @@ function optim.adam(opfunc, x, config, state) state.denom = state.denom or x.new(dfdx:size()):zero() state.t = state.t + 1 - -- Decay the first moment running average coefficient - local bt1 = beta1 * lambda^(state.t - 1) - - state.m:mul(bt1):add(1-bt1, dfdx) + + -- Decay the first and second moment running average coefficient + state.m:mul(beta1):add(1-beta1, dfdx) state.v:mul(beta2):addcmul(1-beta2, dfdx, dfdx) state.denom:copy(state.v):sqrt():add(epsilon) |