|
|
|
@ -98,10 +98,11 @@ bool SparseApplyProximalAdagradCPUKernel::Launch(const std::vector<kernel::Addre
|
|
|
|
|
size_t start_index = var_outer_dim_size_ * index;
|
|
|
|
|
size_t end_index = start_index + var_outer_dim_size_;
|
|
|
|
|
for (size_t j = start_index, k = var_outer_dim_size_ * i; j < end_index; ++j, ++k) {
|
|
|
|
|
accum[j] += grad[k] * grad[k];
|
|
|
|
|
auto summed_grad = unique_sparse_grad.value_[k];
|
|
|
|
|
accum[j] += summed_grad * summed_grad;
|
|
|
|
|
auto learning_rate = lr * (1 / std::sqrt(accum[j]));
|
|
|
|
|
auto prox_v = var[j];
|
|
|
|
|
prox_v -= grad[k] * learning_rate;
|
|
|
|
|
prox_v -= summed_grad * learning_rate;
|
|
|
|
|
if (l1 > 0) {
|
|
|
|
|
var[j] = Sign(prox_v) * std::fmax(std::fabs(prox_v) - learning_rate * l1, static_cast<float>(0.0)) /
|
|
|
|
|
(1 + l2 * learning_rate);
|
|
|
|
|