Add elementwise math operations (#343)
* Add elementwise math operations This allows use to use expressions like: y=log(1+exp(x)) Also added unittests for ActivationFunction * Enforce keyword arguments for non-positional arguments * Add LogActivation to docavx_docs
emailweixu
8 years ago
committed by
luotao1
parent
568d9cff1d
commit
6c3a678c9a
@ -0,0 +1,66 @@
|
||||
/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include "paddle/gserver/layers/DataLayer.h"
|
||||
#include "ModelConfig.pb.h"
|
||||
#include "paddle/trainer/Trainer.h"
|
||||
|
||||
#include "TestUtil.h"
|
||||
#include "LayerGradUtil.h"
|
||||
|
||||
using namespace paddle; // NOLINT
|
||||
using namespace std; // NOLINT
|
||||
|
||||
P_DECLARE_bool(use_gpu);
|
||||
P_DECLARE_bool(thread_local_rand_use_global_seed);
|
||||
|
||||
void testActivation(const string& act) {
|
||||
LOG(INFO) << "test activation: " << act;
|
||||
size_t size = 10;
|
||||
TestConfig config;
|
||||
config.biasSize = 0;
|
||||
config.layerConfig.set_type("addto");
|
||||
config.layerConfig.set_size(size);
|
||||
config.layerConfig.set_active_type(act);
|
||||
config.inputDefs.push_back({INPUT_DATA, "layer_0", size, 0});
|
||||
config.layerConfig.add_inputs();
|
||||
for (auto useGpu : {false, true}) {
|
||||
testLayerGrad(config,
|
||||
act + "_activation",
|
||||
100,
|
||||
/* trans= */false,
|
||||
useGpu,
|
||||
/* useWeight */true);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(Activation, activation) {
|
||||
auto types = ActivationFunction::getAllRegisteredTypes();
|
||||
std::set<string> excluded{"sequence_softmax"};
|
||||
for (auto type : types) {
|
||||
if (excluded.count(type)) continue;
|
||||
testActivation(type);
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
testing::InitGoogleTest(&argc, argv);
|
||||
initMain(argc, argv);
|
||||
FLAGS_thread_local_rand_use_global_seed = true;
|
||||
srand(1);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
@ -0,0 +1,64 @@
|
||||
# Copyright (c) 2016 Baidu, Inc. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from .layers import LayerOutput, mixed_layer, identity_projection, \
|
||||
slope_intercept_layer
|
||||
from .attrs import is_compatible_with
|
||||
from .default_decorators import *
|
||||
import activations as act
|
||||
|
||||
__all__ = []
|
||||
|
||||
def register_unary_math_op(op_name, act):
|
||||
def op(input, name=None):
|
||||
return mixed_layer(input=[identity_projection(input=input)],
|
||||
name=name,
|
||||
act=act)
|
||||
op = wrap_name_default(op_name)(op)
|
||||
op.__doc__ = type(act).__doc__
|
||||
globals()[op_name] = op
|
||||
__all__.append(op_name)
|
||||
|
||||
register_unary_math_op('exp', act.ExpActivation())
|
||||
register_unary_math_op('log', act.LogActivation())
|
||||
register_unary_math_op('abs', act.AbsActivation())
|
||||
register_unary_math_op('sigmoid', act.SigmoidActivation())
|
||||
register_unary_math_op('tanh', act.TanhActivation())
|
||||
register_unary_math_op('square', act.SquareActivation())
|
||||
|
||||
def add(layeroutput, other):
|
||||
if is_compatible_with(other, float):
|
||||
return slope_intercept_layer(input=layeroutput, intercept=other)
|
||||
assert isinstance(other, LayerOutput)
|
||||
return mixed_layer(input=[identity_projection(input=layeroutput),
|
||||
identity_projection(input=other)])
|
||||
|
||||
LayerOutput.__radd__ = add
|
||||
LayerOutput.__add__ = add
|
||||
|
||||
def sub(layeroutput, other):
|
||||
if is_compatible_with(other, float):
|
||||
return slope_intercept_layer(input=layeroutput, intercept=other)
|
||||
assert isinstance(other, LayerOutput)
|
||||
neg = slope_intercept_layer(input=other, slope=-1.0)
|
||||
return mixed_layer(input=[identity_projection(input=layeroutput),
|
||||
identity_projection(input=neg)])
|
||||
|
||||
LayerOutput.__sub__ = sub
|
||||
|
||||
def rsub(layeroutput, other):
|
||||
neg = slope_intercept_layer(input=layeroutput, slope=-1.0)
|
||||
return add(neg, other)
|
||||
|
||||
LayerOutput.__rsub__ = rsub
|
||||
@ -0,0 +1,24 @@
|
||||
from paddle.trainer_config_helpers import *
|
||||
from paddle.trainer_config_helpers import math
|
||||
|
||||
settings(
|
||||
batch_size=1000,
|
||||
learning_rate=1e-5
|
||||
)
|
||||
|
||||
x = data_layer(name='data', size=100)
|
||||
x = math.exp(x)
|
||||
x = math.log(x)
|
||||
x = math.abs(x)
|
||||
x = math.sigmoid(x)
|
||||
x = math.square(x)
|
||||
x = math.square(x)
|
||||
y = 1 + x
|
||||
y = y + 1
|
||||
y = x + y
|
||||
y = y - x
|
||||
y = y - 2
|
||||
y = 2 - y
|
||||
|
||||
outputs(y)
|
||||
|
||||
Loading…
Reference in new issue