diff options
author | Soumith Chintala <soumith@gmail.com> | 2016-06-15 10:50:07 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2016-06-15 10:50:07 +0300 |
commit | f6be4bb195e3e128ab027326255172ff36b6c63c (patch) | |
tree | f532915e9dca0d7e5482c1d16d5d6ed4c8f792ee | |
parent | 2f75fecbbd148877ba4a7345138376b74b2d509f (diff) | |
parent | 8adbeedcbc1f6b3bd4afc3f25bb0a77084000d5b (diff) |
Merge pull request #119 from chenb67/master
add weight decay support to adamax
-rw-r--r-- | adamax.lua | 6 |
1 files changed, 6 insertions, 0 deletions
@@ -28,10 +28,16 @@ function optim.adamax(opfunc, x, config, state) local beta1 = config.beta1 or 0.9 local beta2 = config.beta2 or 0.999 local epsilon = config.epsilon or 1e-38 + local wd = config.weightDecay or 0 -- (1) evaluate f(x) and df/dx local fx, dfdx = opfunc(x) + -- (2) weight decay + if wd ~= 0 then + dfdx:add(wd, x) + end + -- Initialization state.t = state.t or 0 -- Exponential moving average of gradient values |