lvmengsi
6 years ago
committed by
GitHub
parent
c7f36e7c00
commit
4155e62559
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,121 @@
|
||||
/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include "paddle/fluid/framework/eigen.h"
|
||||
#include "paddle/fluid/framework/op_registry.h"
|
||||
#include "paddle/fluid/operators/norm_utils.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
using Tensor = framework::Tensor;
|
||||
using LoDTensor = framework::LoDTensor;
|
||||
using DataLayout = framework::DataLayout;
|
||||
|
||||
template <typename T>
|
||||
using EigenArrayMap =
|
||||
Eigen::Map<Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic>>;
|
||||
template <typename T>
|
||||
using ConstEigenArrayMap =
|
||||
Eigen::Map<const Eigen::Array<T, Eigen::Dynamic, Eigen::Dynamic>>;
|
||||
template <typename T>
|
||||
using EigenVectorArrayMap = Eigen::Map<Eigen::Array<T, Eigen::Dynamic, 1>>;
|
||||
template <typename T>
|
||||
using ConstEigenVectorArrayMap =
|
||||
Eigen::Map<const Eigen::Array<T, Eigen::Dynamic, 1>>;
|
||||
|
||||
class InstanceNormOp : public framework::OperatorWithKernel {
|
||||
public:
|
||||
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||
void InferShape(framework::InferShapeContext *ctx) const override;
|
||||
|
||||
protected:
|
||||
framework::OpKernelType GetExpectedKernelType(
|
||||
const framework::ExecutionContext &ctx) const override;
|
||||
};
|
||||
|
||||
class InstanceNormGradOp : public framework::OperatorWithKernel {
|
||||
public:
|
||||
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||
void InferShape(framework::InferShapeContext *ctx) const override;
|
||||
|
||||
protected:
|
||||
framework::OpKernelType GetExpectedKernelType(
|
||||
const framework::ExecutionContext &ctx) const override;
|
||||
};
|
||||
|
||||
class InstanceNormDoubleGradOp : public framework::OperatorWithKernel {
|
||||
public:
|
||||
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||
void InferShape(framework::InferShapeContext *ctx) const override;
|
||||
|
||||
protected:
|
||||
framework::OpKernelType GetExpectedKernelType(
|
||||
const framework::ExecutionContext &ctx) const override;
|
||||
};
|
||||
|
||||
class InstanceNormOpMaker : public framework::OpProtoAndCheckerMaker {
|
||||
public:
|
||||
void Make() override;
|
||||
};
|
||||
|
||||
class InstanceNormGradMaker : public framework::SingleGradOpDescMaker {
|
||||
public:
|
||||
using framework::SingleGradOpDescMaker::SingleGradOpDescMaker;
|
||||
|
||||
protected:
|
||||
std::unique_ptr<framework::OpDesc> Apply() const override;
|
||||
};
|
||||
|
||||
class InstanceNormDoubleGradMaker : public framework::SingleGradOpDescMaker {
|
||||
public:
|
||||
using framework::SingleGradOpDescMaker::SingleGradOpDescMaker;
|
||||
|
||||
protected:
|
||||
std::unique_ptr<framework::OpDesc> Apply() const override;
|
||||
};
|
||||
|
||||
class InstanceNormOpInferVarType
|
||||
: public framework::PassInDtypeAndVarTypeToOutput {
|
||||
protected:
|
||||
std::unordered_map<std::string, std::string> GetInputOutputWithSameType()
|
||||
const override {
|
||||
return std::unordered_map<std::string, std::string>{{"X", "Y"}};
|
||||
}
|
||||
};
|
||||
|
||||
template <typename DeviceContext, typename T>
|
||||
class InstanceNormKernel : public framework::OpKernel<T> {
|
||||
public:
|
||||
void Compute(const framework::ExecutionContext &ctx) const override;
|
||||
};
|
||||
|
||||
template <typename DeviceContext, typename T>
|
||||
class InstanceNormGradKernel : public framework::OpKernel<T> {
|
||||
public:
|
||||
void Compute(const framework::ExecutionContext &ctx) const override;
|
||||
};
|
||||
|
||||
template <typename DeviceContext, typename T>
|
||||
class InstanceNormDoubleGradKernel : public framework::OpKernel<T> {
|
||||
public:
|
||||
void Compute(const framework::ExecutionContext &ctx) const override;
|
||||
};
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,46 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
#include <memory>
|
||||
#include "paddle/fluid/framework/op_registry.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
using DataLayout = framework::DataLayout;
|
||||
|
||||
inline void ExtractNCWHD(const framework::DDim &dims,
|
||||
const DataLayout &data_layout, int *N, int *C, int *H,
|
||||
int *W, int *D) {
|
||||
*N = dims[0];
|
||||
if (dims.size() == 2) {
|
||||
*C = dims[1];
|
||||
*H = 1;
|
||||
*W = 1;
|
||||
*D = 1;
|
||||
} else {
|
||||
*C = data_layout == DataLayout::kNCHW ? dims[1] : dims[dims.size() - 1];
|
||||
*H = data_layout == DataLayout::kNCHW ? dims[2] : dims[1];
|
||||
*W = dims.size() > 3
|
||||
? (data_layout == DataLayout::kNCHW ? dims[3] : dims[2])
|
||||
: 1;
|
||||
*D = dims.size() > 4
|
||||
? (data_layout == DataLayout::kNCHW ? dims[4] : dims[3])
|
||||
: 1;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,188 @@
|
||||
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
import unittest
|
||||
import numpy as np
|
||||
import paddle.fluid.core as core
|
||||
import paddle.fluid as fluid
|
||||
from paddle.fluid.op import Operator
|
||||
from op_test import OpTest
|
||||
|
||||
|
||||
def _reference_instance_norm_naive(x, scale, bias, epsilon, mean, var):
|
||||
x_shape = x.shape
|
||||
if len(x_shape) == 2:
|
||||
x = np.reshape(x, (x.shape[0], x.shape[1], 1, 1))
|
||||
n, c, h, w = x.shape
|
||||
|
||||
mean_tile = np.reshape(mean, (n, c, 1, 1))
|
||||
mean_tile = np.tile(mean_tile, (1, 1, h, w))
|
||||
var_tile = np.reshape(var, (n, c, 1, 1))
|
||||
var_tile = np.tile(var_tile, (1, 1, h, w))
|
||||
|
||||
x_norm = (x - mean_tile) / np.sqrt(var_tile + epsilon).astype('float32')
|
||||
scale_tile = np.reshape(scale, (1, c, 1, 1))
|
||||
scale_tile = np.tile(scale_tile, (n, 1, h, w))
|
||||
bias_tile = np.reshape(bias, (1, c, 1, 1))
|
||||
bias_tile = np.tile(bias_tile, (n, 1, h, w))
|
||||
y = scale_tile * x_norm + bias_tile
|
||||
if len(x_shape) == 2:
|
||||
y = np.reshape(y, x_shape)
|
||||
return y, mean, var
|
||||
|
||||
|
||||
def _reference_instance_norm_grad(x, d_y, scale, mean, var, epsilon):
|
||||
# d_scale = sum(d_y * (x-mean) / sqrt(var+epsilon))
|
||||
# d_offset = sum(d_y)
|
||||
# d_x = scale / sqrt(var+epsilon) * (d_y - np.mean(d_y, axis=(2,3)) - (x-mean)/sqrt(var+epsilon)* np.mean(y_grad * (x-mean)/sqrt(var+epsilon), axis=(2,3)))
|
||||
n, c, h, w = x.shape
|
||||
|
||||
d_bias = np.sum(d_y, axis=(0, 2, 3))
|
||||
|
||||
mean_tile = np.reshape(mean, (n, c, 1, 1))
|
||||
mean_tile = np.tile(mean_tile, (1, 1, h, w))
|
||||
var_tile = np.reshape(var, (n, c, 1, 1))
|
||||
var_tile = np.tile(var_tile, (1, 1, h, w))
|
||||
|
||||
d_scale = np.sum(d_y * (x - mean_tile) * var_tile, axis=(0, 2, 3))
|
||||
var_inv = var_tile
|
||||
scale_tile = np.reshape(scale, (1, c, 1, 1))
|
||||
scale_tile = np.tile(scale_tile, (n, 1, h, w))
|
||||
|
||||
d_x = scale_tile * var_inv * (d_y - np.mean(
|
||||
d_y, axis=(2, 3), keepdims=True) - (x - mean_tile) * var_inv * np.mean(
|
||||
d_y * (x - mean_tile) * var_inv, axis=(2, 3), keepdims=True))
|
||||
return d_x, d_scale, d_bias
|
||||
|
||||
|
||||
def _cal_mean_variance(x, epsilon, mean_shape):
|
||||
mean = np.reshape(np.mean(x, axis=(2, 3)), mean_shape)
|
||||
var = np.reshape(np.var(x, axis=(2, 3)), mean_shape)
|
||||
return mean, var
|
||||
|
||||
|
||||
class TestInstanceNormOpTraining(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.epsilon = 1e-5
|
||||
self.init_test_case()
|
||||
|
||||
def init_test_case(self):
|
||||
self.use_global_stats = False
|
||||
self.no_grad_set = set()
|
||||
self.fetch_list = [
|
||||
'y', 'saved_mean', 'saved_variance', 'x@GRAD', 'scale@GRAD',
|
||||
'bias@GRAD'
|
||||
]
|
||||
|
||||
def __assert_close(self, tensor, np_array, msg, atol=1e-4):
|
||||
self.assertTrue(np.allclose(np.array(tensor), np_array, atol=atol), msg)
|
||||
|
||||
def set_global_mean_var(self, mean_shape, x):
|
||||
mean, variance = _cal_mean_variance(x, self.epsilon, mean_shape)
|
||||
return mean, variance
|
||||
|
||||
def test_forward_backward(self):
|
||||
def test_with_place(place, shape):
|
||||
epsilon = self.epsilon
|
||||
n, c, h, w = shape[0], shape[1], shape[2], shape[3]
|
||||
scale_shape = [c]
|
||||
mean_shape = [n * c]
|
||||
|
||||
np.random.seed()
|
||||
x = np.random.random_sample(shape).astype(np.float32)
|
||||
scale = np.random.random_sample(scale_shape).astype(np.float32)
|
||||
bias = np.random.random_sample(scale_shape).astype(np.float32)
|
||||
mean, variance = self.set_global_mean_var(mean_shape, x)
|
||||
d_y = np.random.random_sample(shape).astype(np.float32)
|
||||
|
||||
y, saved_mean, variance_tmp = _reference_instance_norm_naive(
|
||||
x, scale, bias, epsilon, mean, variance)
|
||||
|
||||
saved_variance = 1 / np.sqrt(variance_tmp + epsilon)
|
||||
|
||||
d_x, d_scale, d_bias = _reference_instance_norm_grad(
|
||||
x, d_y, scale, saved_mean, saved_variance, epsilon)
|
||||
|
||||
var_dict = locals()
|
||||
var_dict['y@GRAD'] = d_y
|
||||
var_dict['x@GRAD'] = d_x
|
||||
var_dict['scale@GRAD'] = d_scale
|
||||
var_dict['bias@GRAD'] = d_bias
|
||||
|
||||
var_names = [
|
||||
'x', 'scale', 'bias', 'y', 'saved_mean', 'saved_variance'
|
||||
]
|
||||
ground_truth = {name: var_dict[name] for name in var_names}
|
||||
|
||||
program = fluid.Program()
|
||||
with fluid.program_guard(program):
|
||||
block = program.global_block()
|
||||
for name in ground_truth:
|
||||
block.create_var(
|
||||
name=name,
|
||||
dtype='float32',
|
||||
shape=ground_truth[name].shape)
|
||||
in_op = block.append_op(
|
||||
type="instance_norm",
|
||||
inputs={
|
||||
"X": block.var("x"),
|
||||
"Scale": block.var("scale"),
|
||||
"Bias": block.var("bias"),
|
||||
},
|
||||
outputs={
|
||||
"Y": block.var("y"),
|
||||
"SavedMean": block.var("saved_mean"),
|
||||
"SavedVariance": block.var("saved_variance")
|
||||
},
|
||||
attrs={"epsilon": epsilon, })
|
||||
|
||||
block.create_var(name="y@GRAD", dtype='float32', shape=y.shape)
|
||||
|
||||
grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc(
|
||||
in_op.desc, self.no_grad_set, [])
|
||||
grad_op_desc = grad_op_desc_list[0]
|
||||
new_op_desc = block.desc.append_op()
|
||||
new_op_desc.copy_from(grad_op_desc)
|
||||
for var_name in grad_op_desc.output_arg_names():
|
||||
block.desc.var(var_name.encode("ascii"))
|
||||
grad_op_desc.infer_var_type(block.desc)
|
||||
grad_op_desc.infer_shape(block.desc)
|
||||
for arg in grad_op_desc.output_arg_names():
|
||||
grad_var = block.desc.find_var(arg.encode("ascii"))
|
||||
grad_var.set_dtype(core.VarDesc.VarType.FP32)
|
||||
|
||||
exe = fluid.Executor(place)
|
||||
out = exe.run(program,
|
||||
feed={
|
||||
name: var_dict[name]
|
||||
for name in ['x', 'scale', 'bias', 'y@GRAD']
|
||||
},
|
||||
fetch_list=self.fetch_list)
|
||||
|
||||
for id, name in enumerate(self.fetch_list):
|
||||
self.__assert_close(var_dict[name], out[id], name)
|
||||
print("op test forward passes: ", str(place))
|
||||
|
||||
places = [core.CPUPlace()]
|
||||
|
||||
if core.is_compiled_with_cuda() and core.op_support_gpu(
|
||||
"instance_norm"):
|
||||
places.append(core.CUDAPlace(0))
|
||||
for place in places:
|
||||
test_with_place(place, [2, 3, 4, 5])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@ -0,0 +1,53 @@
|
||||
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import unittest
|
||||
import numpy as np
|
||||
|
||||
import paddle.fluid as fluid
|
||||
import paddle.fluid.layers as layers
|
||||
import paddle.fluid.core as core
|
||||
import gradient_checker
|
||||
|
||||
from decorator_helper import prog_scope
|
||||
|
||||
|
||||
class TestInstanceNormDoubleGradCheck(unittest.TestCase):
|
||||
@prog_scope()
|
||||
def func(self, place):
|
||||
prog = fluid.Program()
|
||||
with fluid.program_guard(prog):
|
||||
np.random.seed()
|
||||
shape = [2, 3, 4, 5]
|
||||
dtype = "float32"
|
||||
eps = 0.005
|
||||
atol = 1e-4
|
||||
x = layers.create_parameter(dtype=dtype, shape=shape, name='x')
|
||||
z = fluid.layers.instance_norm(input=x)
|
||||
x_arr = np.random.uniform(-1, 1, shape).astype(dtype)
|
||||
gradient_checker.double_grad_check(
|
||||
[x], z, x_init=x_arr, atol=atol, place=place, eps=eps)
|
||||
|
||||
def test_grad(self):
|
||||
places = [fluid.CPUPlace()]
|
||||
if core.is_compiled_with_cuda():
|
||||
places.append(fluid.CUDAPlace(0))
|
||||
for p in places:
|
||||
self.func(p)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in new issue