yangzhenzhang
5 years ago
parent
b33687e84c
commit
b34c0e7a17
File diff suppressed because it is too large
Load Diff
@ -1,188 +0,0 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "parallel/ops_info/generator_info.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "ir/value.h"
|
||||
#include "parallel/device_matrix.h"
|
||||
#include "parallel/strategy.h"
|
||||
#include "parallel/tensor_layout/tensor_redistribution.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace parallel {
|
||||
Status GeneratorBase::InferTensorMap() {
|
||||
TensorMap output_tensor_map = {MAP_NONE};
|
||||
outputs_tensor_map_.push_back(output_tensor_map);
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status GeneratorBase::InferTensorInfo() {
|
||||
Shape output_shape = outputs_shape_.at(0);
|
||||
Shape output_slice_shape = outputs_shape_.at(0);
|
||||
|
||||
TensorLayout output_tensor_layout;
|
||||
if (output_tensor_layout.InitFromVector(dev_matrix_shape_, outputs_tensor_map_[0], output_shape) != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << " : Creat output tensor layout failed.";
|
||||
return FAILED;
|
||||
}
|
||||
TensorInfo output_tensor_info(output_tensor_layout, output_shape, output_slice_shape);
|
||||
outputs_tensor_info_.push_back(output_tensor_info);
|
||||
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status GeneratorBase::InferDevMatrixShape() {
|
||||
std::vector<Dimensions> stra = strategy_->GetInputDim();
|
||||
Dimensions input_strategy = stra.at(0);
|
||||
|
||||
dev_matrix_shape_ = input_strategy;
|
||||
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status GeneratorBase::SetCostUnderStrategy(const StrategyPtr &strategy) {
|
||||
if (SetCostUnderStrategyBase(strategy) != SUCCESS) {
|
||||
if (is_auto_parallel_) {
|
||||
MS_LOG(DEBUG) << name_ << " : Set cost under strategy failed.";
|
||||
} else {
|
||||
MS_LOG(ERROR) << name_ << " : Set cost under strategy failed.";
|
||||
}
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status DropoutGenMaskInfo::GenerateStrategies(int32_t stage_id) {
|
||||
if (input_value_.empty()) {
|
||||
MS_LOG(ERROR) << name_ << " : Input value is empty.";
|
||||
return FAILED;
|
||||
}
|
||||
Shape param = GetValue<const std::vector<int>>(input_value_[0]);
|
||||
if (param.empty()) {
|
||||
MS_LOG(ERROR) << name_ << " : Input value [0] is empty.";
|
||||
return FAILED;
|
||||
}
|
||||
// Now,only support batch parallel.
|
||||
CheckGlobalDeviceManager();
|
||||
is_auto_parallel_ = true;
|
||||
size_t dev_num = g_device_manager->GetDeviceListByStageId(stage_id).size();
|
||||
Dimensions strategy(param.size() - 1, 1);
|
||||
(void)strategy.insert(strategy.begin(), SizeToInt(dev_num));
|
||||
std::vector<Dimensions> stra = {strategy};
|
||||
StrategyPtr sp = std::make_shared<Strategy>(stage_id, stra);
|
||||
if (SetCostUnderStrategy(sp) == SUCCESS) {
|
||||
MS_LOG(INFO) << name_ << " : Successfully generated batch-parallel-strategy.";
|
||||
PrintStrategy(sp);
|
||||
} else {
|
||||
MS_LOG(ERROR) << name_ << " : Generating batch-parallel-strategy failed.";
|
||||
return FAILED;
|
||||
}
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status DropoutGenMaskInfo::CheckStrategy(const StrategyPtr &strategy) {
|
||||
if (strategy->GetInputNumber() != 1) {
|
||||
if (is_auto_parallel_) {
|
||||
MS_LOG(DEBUG) << name_ << " : The strategy is wrong.";
|
||||
} else {
|
||||
MS_LOG(ERROR) << name_ << " : The strategy is wrong.";
|
||||
}
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status DropoutGenMaskInfo::InferReplaceOps(const StrategyPtr &strategy) {
|
||||
Shape shape = GetValue<const std::vector<int>>(input_value_[0]);
|
||||
Strategys stra = strategy->GetInputDim();
|
||||
Dimensions input_strategy = stra.at(0);
|
||||
int32_t dev_num = *(input_strategy.begin());
|
||||
if (dev_num <= 0) {
|
||||
MS_LOG(ERROR) << name_ << " : The number of devices should not be less than 0.";
|
||||
return FAILED;
|
||||
}
|
||||
// Batch parallel
|
||||
if (shape[0] % dev_num != 0) {
|
||||
MS_LOG(ERROR) << name_ << " : The shape " << shape[0] << " can't be exact divided by device number " << dev_num;
|
||||
return FAILED;
|
||||
}
|
||||
shape[0] = shape[0] / dev_num;
|
||||
ValuePtr shape_ptr = MakeValue(shape);
|
||||
Attr attr_0 = std::make_pair(SEED0, attrs_[SEED0]);
|
||||
Attr attr_1 = std::make_pair(SEED1, attrs_[SEED1]);
|
||||
OperatorAttrs attrs = {attr_0, attr_1};
|
||||
Attr param_0 = std::make_pair(SHAPE, shape_ptr);
|
||||
Attr param_1 = std::make_pair(KEEP_PROB, input_value_[1]);
|
||||
OperatorParams params = {std::make_pair(param_0, 1), std::make_pair(param_1, 2)};
|
||||
OperatorArgs args = std::make_pair(attrs, params);
|
||||
replace_op_ = {std::make_pair(DROPOUT_GEN_MASK, args)};
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
std::shared_ptr<std::vector<std::vector<int32_t>>> DropoutGenMaskInfo::GenerateBatchStrategies() {
|
||||
if (input_value_.empty()) {
|
||||
MS_LOG(EXCEPTION) << name_ << " : Input value is empty.";
|
||||
}
|
||||
Shape param = GetValue<const std::vector<int>>(input_value_[0]);
|
||||
if (param.empty()) {
|
||||
MS_LOG(EXCEPTION) << name_ << " : Input value [0] is empty.";
|
||||
}
|
||||
// Now,only support batch parallel.
|
||||
CheckGlobalDeviceManager();
|
||||
size_t dev_num = g_device_manager->GetDeviceListByStageId(0).size();
|
||||
Dimensions strategy(param.size() - 1, 1);
|
||||
(void)strategy.insert(strategy.begin(), SizeToInt(dev_num));
|
||||
std::vector<Dimensions> strategy_v = {strategy};
|
||||
return std::make_shared<std::vector<std::vector<int32_t>>>(strategy_v);
|
||||
}
|
||||
|
||||
Status GeneratorBase::Init(const StrategyPtr &strategy) {
|
||||
if (InitWithAutoRepeatCalc(strategy) != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << " : Init failed.";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
if (InferReplaceOps(strategy) != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << " : Infer replace ops failed.";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
MS_LOG(INFO) << name_ << " : Init success.";
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status GeneratorBase::InitForCostModel(const StrategyPtr &strategy) {
|
||||
if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) {
|
||||
if (is_auto_parallel_) {
|
||||
MS_LOG(DEBUG) << name_ << " : Init for cost model failed.";
|
||||
} else {
|
||||
MS_LOG(ERROR) << name_ << " : Init for cost model failed.";
|
||||
}
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
MS_LOG(INFO) << name_ << " : Init for cost model success.";
|
||||
return SUCCESS;
|
||||
}
|
||||
} // namespace parallel
|
||||
} // namespace mindspore
|
||||
@ -1,70 +0,0 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_PARALLEL_OPS_INFO_GENERATOR_INFO_H_
|
||||
#define MINDSPORE_CCSRC_PARALLEL_OPS_INFO_GENERATOR_INFO_H_
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "parallel/auto_parallel/operator_costmodel.h"
|
||||
#include "parallel/ops_info/operator_info.h"
|
||||
#include "parallel/strategy.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace parallel {
|
||||
class GeneratorBase : public OperatorInfo {
|
||||
public:
|
||||
GeneratorBase(const std::string &operator_name, const Shapes &inputs_shape, const Shapes &outputs_shape,
|
||||
const PrimitiveAttrs &attrs)
|
||||
: OperatorInfo(operator_name, inputs_shape, outputs_shape, attrs, std::make_shared<GeneratorBaseCost>()) {}
|
||||
|
||||
~GeneratorBase() override = default;
|
||||
|
||||
Status Init(const StrategyPtr &strategy) override;
|
||||
Status SetCostUnderStrategy(const StrategyPtr &strategy) override;
|
||||
Status InitForCostModel(const StrategyPtr &strategy) override;
|
||||
|
||||
protected:
|
||||
// For now, generator ops don't have attributes
|
||||
Status GetAttrs() override { return Status::SUCCESS; }
|
||||
Status InferTensorMap() override;
|
||||
Status InferTensorInfo() override;
|
||||
Status InferDevMatrixShape() override;
|
||||
Status InferMirrorOps() override { return SUCCESS; }
|
||||
Status InferForwardCommunication() override { return SUCCESS; }
|
||||
virtual Status InferReplaceOps(const StrategyPtr &strategy) = 0;
|
||||
};
|
||||
|
||||
class DropoutGenMaskInfo : public GeneratorBase {
|
||||
public:
|
||||
DropoutGenMaskInfo(const std::string &name, const Shapes &inputs_shape, const Shapes &outputs_shape,
|
||||
const PrimitiveAttrs &attrs)
|
||||
: GeneratorBase(name, inputs_shape, outputs_shape, attrs) {}
|
||||
~DropoutGenMaskInfo() override = default;
|
||||
Status GenerateStrategies(int32_t stage_id) override;
|
||||
std::shared_ptr<std::vector<std::vector<int32_t>>> GenerateBatchStrategies() override;
|
||||
|
||||
protected:
|
||||
Status CheckStrategy(const StrategyPtr &strategy) override;
|
||||
Status InferReplaceOps(const StrategyPtr &strategy) override;
|
||||
};
|
||||
} // namespace parallel
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_PARALLEL_OPS_INFO_GENERATOR_INFO_H_
|
||||
@ -1,166 +0,0 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include <string>
|
||||
#include <list>
|
||||
#include <vector>
|
||||
#include "common/common_test.h"
|
||||
#include "parallel/strategy.h"
|
||||
#include "parallel/ops_info/dropout_do_mask_info.h"
|
||||
#include "parallel/device_manager.h"
|
||||
#include "parallel/step_parallel.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace parallel {
|
||||
|
||||
class DropoutDoMaskInfo;
|
||||
using DropoutDoMaskInfoPtr = std::shared_ptr<DropoutDoMaskInfo>;
|
||||
DropoutDoMaskInfoPtr do_mask;
|
||||
|
||||
class TestDropoutDoMaskInfo : public UT::Common {
|
||||
public:
|
||||
TestDropoutDoMaskInfo() {}
|
||||
void SetUp();
|
||||
void TearDown() {}
|
||||
};
|
||||
|
||||
void TestDropoutDoMaskInfo::SetUp() {
|
||||
std::vector<int32_t> dev_list;
|
||||
|
||||
for (int32_t i = 0; i < 34; i++) {
|
||||
dev_list.push_back(i);
|
||||
}
|
||||
|
||||
std::vector<int32_t> stage_map;
|
||||
stage_map.push_back(32);
|
||||
stage_map.push_back(2);
|
||||
|
||||
int32_t local_dev = 0;
|
||||
|
||||
// create a new g_device_manager
|
||||
g_device_manager = std::make_shared<DeviceManager>();
|
||||
g_device_manager->Init(dev_list, local_dev, stage_map, "hccl");
|
||||
|
||||
std::unordered_map<std::string, ValuePtr> attr;
|
||||
|
||||
Shapes inputs_shape = {{32, 128}, {64}, {}};
|
||||
Shapes outputs_shape = {{32, 128}};
|
||||
do_mask = std::make_shared<DropoutDoMaskInfo>("do_mask_info", inputs_shape, outputs_shape, attr);
|
||||
}
|
||||
|
||||
TEST_F(TestDropoutDoMaskInfo, InferDevMatrixShape) {
|
||||
std::vector<Dimensions> stra = {{4, 8}};
|
||||
StrategyPtr strategy = NewStrategy(0, stra);
|
||||
|
||||
do_mask->Init(strategy);
|
||||
std::vector<int32_t> dev_matrix_shape = do_mask->dev_matrix_shape();
|
||||
|
||||
std::vector<int32_t> expect = {4, 8};
|
||||
ASSERT_EQ(dev_matrix_shape, expect);
|
||||
}
|
||||
|
||||
TEST_F(TestDropoutDoMaskInfo, InferSliceShape) {
|
||||
std::vector<Dimensions> stra = {{4, 8}};
|
||||
StrategyPtr strategy = NewStrategy(0, stra);
|
||||
|
||||
do_mask->Init(strategy);
|
||||
std::vector<TensorInfo> inputs = do_mask->inputs_tensor_info();
|
||||
std::vector<TensorInfo> outputs = do_mask->outputs_tensor_info();
|
||||
|
||||
Shape input_a_slice_shape_expect = {8, 16};
|
||||
Shape input_b_slice_shape_expect = {64};
|
||||
Shape output_slice_shape_expect = {8, 16};
|
||||
|
||||
TensorInfo input_a_tensor_info = inputs.at(0);
|
||||
TensorInfo input_b_tensor_info = inputs.at(1);
|
||||
TensorInfo output_tensor_info = outputs.at(0);
|
||||
Shape input_a_slice_shape = input_a_tensor_info.slice_shape();
|
||||
Shape input_b_slice_shape = input_b_tensor_info.slice_shape();
|
||||
Shape output_slice_shape = output_tensor_info.slice_shape();
|
||||
|
||||
ASSERT_EQ(input_a_slice_shape, input_a_slice_shape_expect);
|
||||
ASSERT_EQ(input_b_slice_shape, input_b_slice_shape_expect);
|
||||
ASSERT_EQ(output_slice_shape, output_slice_shape_expect);
|
||||
}
|
||||
|
||||
TEST_F(TestDropoutDoMaskInfo, GetTensorLayout) {
|
||||
std::vector<Dimensions> stra = {{4, 8}};
|
||||
StrategyPtr strategy = NewStrategy(0, stra);
|
||||
|
||||
do_mask->Init(strategy);
|
||||
std::vector<TensorInfo> inputs = do_mask->inputs_tensor_info();
|
||||
std::vector<TensorInfo> outputs = do_mask->outputs_tensor_info();
|
||||
|
||||
TensorMap input_a_map_expect = {1, 0};
|
||||
TensorMap input_b_map_expect = {-1};
|
||||
TensorMap output_map_expect = {1, 0};
|
||||
|
||||
TensorInfo input_a_tensor_info = inputs.at(0);
|
||||
TensorInfo input_b_tensor_info = inputs.at(1);
|
||||
TensorInfo output_tensor_info = outputs.at(0);
|
||||
Map input_a_tensor_map = input_a_tensor_info.tensor_layout().origin_tensor_map();
|
||||
Map input_b_tensor_map = input_b_tensor_info.tensor_layout().origin_tensor_map();
|
||||
Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map();
|
||||
|
||||
ASSERT_EQ(input_a_tensor_map.array(), input_a_map_expect);
|
||||
ASSERT_EQ(input_b_tensor_map.array(), input_b_map_expect);
|
||||
ASSERT_EQ(output_tensor_map.array(), output_map_expect);
|
||||
}
|
||||
|
||||
TEST_F(TestDropoutDoMaskInfo, GetForwardOp) {
|
||||
std::vector<Dimensions> stra = {{4, 8}};
|
||||
StrategyPtr strategy = NewStrategy(0, stra);
|
||||
|
||||
do_mask->Init(strategy);
|
||||
OperatorVector forward_op = do_mask->forward_op();
|
||||
size_t size = forward_op.size();
|
||||
|
||||
ASSERT_EQ(size, 0);
|
||||
}
|
||||
|
||||
TEST_F(TestDropoutDoMaskInfo, CheckStrategy1) {
|
||||
std::vector<Dimensions> stra = {{4, 8, 2}};
|
||||
StrategyPtr strategy = NewStrategy(0, stra);
|
||||
|
||||
Status ret = do_mask->Init(strategy);
|
||||
ASSERT_EQ(ret, FAILED);
|
||||
}
|
||||
|
||||
TEST_F(TestDropoutDoMaskInfo, CheckStrategy2) {
|
||||
std::vector<Dimensions> stra = {{8, 8}};
|
||||
StrategyPtr strategy = NewStrategy(0, stra);
|
||||
|
||||
Status ret = do_mask->Init(strategy);
|
||||
ASSERT_EQ(ret, FAILED);
|
||||
}
|
||||
|
||||
TEST_F(TestDropoutDoMaskInfo, CheckStrategy3) {
|
||||
std::vector<Dimensions> stra = {{4, 8}, {4, 8}};
|
||||
StrategyPtr strategy = NewStrategy(0, stra);
|
||||
|
||||
Status ret = do_mask->Init(strategy);
|
||||
ASSERT_EQ(ret, FAILED);
|
||||
}
|
||||
|
||||
TEST_F(TestDropoutDoMaskInfo, CheckStrategy4) {
|
||||
std::vector<Dimensions> stra = {{4, 8}};
|
||||
StrategyPtr strategy = NewStrategy(0, stra);
|
||||
|
||||
Status ret = do_mask->Init(strategy);
|
||||
ASSERT_EQ(ret, SUCCESS);
|
||||
}
|
||||
} // namespace parallel
|
||||
} // namespace mindspore
|
||||
@ -1,137 +0,0 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include <string>
|
||||
#include <list>
|
||||
#include <vector>
|
||||
#include "common/common_test.h"
|
||||
#include "parallel/strategy.h"
|
||||
#include "parallel/ops_info/generator_info.h"
|
||||
#include "parallel/device_manager.h"
|
||||
#include "parallel/step_parallel.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace parallel {
|
||||
|
||||
class DropoutGenMaskInfo;
|
||||
using DropoutGenMaskInfoPtr = std::shared_ptr<DropoutGenMaskInfo>;
|
||||
DropoutGenMaskInfoPtr gen_mask;
|
||||
|
||||
class TestDropoutGenMaskInfo : public UT::Common {
|
||||
public:
|
||||
TestDropoutGenMaskInfo() {}
|
||||
void SetUp();
|
||||
void TearDown() {}
|
||||
};
|
||||
|
||||
void TestDropoutGenMaskInfo::SetUp() {
|
||||
std::vector<int32_t> dev_list;
|
||||
|
||||
for (int32_t i = 0; i < 10; i++) {
|
||||
dev_list.push_back(i);
|
||||
}
|
||||
|
||||
std::vector<int32_t> stage_map;
|
||||
stage_map.push_back(8);
|
||||
stage_map.push_back(2);
|
||||
|
||||
int32_t local_dev = 0;
|
||||
|
||||
// create a new g_device_manager
|
||||
g_device_manager = std::make_shared<DeviceManager>();
|
||||
g_device_manager->Init(dev_list, local_dev, stage_map, "hccl");
|
||||
|
||||
std::unordered_map<std::string, ValuePtr> attr;
|
||||
|
||||
Shapes inputs_shape;
|
||||
Shapes outputs_shape = {{128}};
|
||||
std::vector<int> shape = {32, 128};
|
||||
ValuePtr val0 = MakeValue(shape);
|
||||
ValuePtr val1;
|
||||
std::vector<ValuePtr> val = {val0, val1};
|
||||
gen_mask = std::make_shared<DropoutGenMaskInfo>("gen_mask_info", inputs_shape, outputs_shape, attr);
|
||||
gen_mask->set_input_value(val);
|
||||
}
|
||||
|
||||
TEST_F(TestDropoutGenMaskInfo, InferDevMatrixShape) {
|
||||
std::vector<Dimensions> stra = {{8, 1}};
|
||||
StrategyPtr strategy = NewStrategy(0, stra);
|
||||
|
||||
gen_mask->Init(strategy);
|
||||
std::vector<int32_t> dev_matrix_shape = gen_mask->dev_matrix_shape();
|
||||
|
||||
std::vector<int32_t> expect = {8, 1};
|
||||
ASSERT_EQ(dev_matrix_shape, expect);
|
||||
}
|
||||
|
||||
TEST_F(TestDropoutGenMaskInfo, InferSliceShape) {
|
||||
std::vector<Dimensions> stra = {{8, 1}};
|
||||
StrategyPtr strategy = NewStrategy(0, stra);
|
||||
|
||||
gen_mask->Init(strategy);
|
||||
std::vector<TensorInfo> outputs = gen_mask->outputs_tensor_info();
|
||||
|
||||
Shape output_slice_shape_expect = {128};
|
||||
|
||||
TensorInfo output_tensor_info = outputs.at(0);
|
||||
Shape output_slice_shape = output_tensor_info.slice_shape();
|
||||
|
||||
ASSERT_EQ(output_slice_shape, output_slice_shape_expect);
|
||||
}
|
||||
|
||||
TEST_F(TestDropoutGenMaskInfo, GetTensorLayout) {
|
||||
std::vector<Dimensions> stra = {{8, 1}};
|
||||
StrategyPtr strategy = NewStrategy(0, stra);
|
||||
|
||||
gen_mask->Init(strategy);
|
||||
std::vector<TensorInfo> outputs = gen_mask->outputs_tensor_info();
|
||||
|
||||
TensorMap output_map_expect = {-1};
|
||||
|
||||
TensorInfo output_tensor_info = outputs.at(0);
|
||||
Map output_tensor_map = output_tensor_info.tensor_layout().origin_tensor_map();
|
||||
|
||||
ASSERT_EQ(output_tensor_map.array(), output_map_expect);
|
||||
}
|
||||
|
||||
TEST_F(TestDropoutGenMaskInfo, GetForwardOp) {
|
||||
std::vector<Dimensions> stra = {{8, 1}};
|
||||
StrategyPtr strategy = NewStrategy(0, stra);
|
||||
|
||||
gen_mask->Init(strategy);
|
||||
OperatorVector forward_op = gen_mask->forward_op();
|
||||
size_t size = forward_op.size();
|
||||
|
||||
ASSERT_EQ(size, 0);
|
||||
}
|
||||
|
||||
TEST_F(TestDropoutGenMaskInfo, CheckStrategy1) {
|
||||
std::vector<Dimensions> stra = {{4, 8, 2}, {2, 3}};
|
||||
StrategyPtr strategy = NewStrategy(0, stra);
|
||||
|
||||
Status ret = gen_mask->Init(strategy);
|
||||
ASSERT_EQ(ret, FAILED);
|
||||
}
|
||||
|
||||
TEST_F(TestDropoutGenMaskInfo, CheckStrategy2) {
|
||||
std::vector<Dimensions> stra = {{8, 1}};
|
||||
StrategyPtr strategy = NewStrategy(0, stra);
|
||||
|
||||
Status ret = gen_mask->Init(strategy);
|
||||
ASSERT_EQ(ret, SUCCESS);
|
||||
}
|
||||
} // namespace parallel
|
||||
} // namespace mindspore
|
||||
@ -0,0 +1,94 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import numpy as np
|
||||
import mindspore as ms
|
||||
from mindspore import context, Tensor, Parameter
|
||||
from mindspore.nn import Cell, TrainOneStepCell, Momentum
|
||||
from mindspore.ops import operations as P
|
||||
from mindspore.common.api import _executor
|
||||
|
||||
|
||||
class Net(Cell):
|
||||
def __init__(self, mul_weight, strategy1=None, strategy2=None):
|
||||
super().__init__()
|
||||
self.mul = P.Mul().set_strategy(strategy1)
|
||||
self.mul2 = P.Mul().set_strategy(strategy1)
|
||||
self.dropout_do_mask = P.DropoutDoMask().set_strategy(strategy2)
|
||||
self.dropout_gen_mask = P.DropoutGenMask()
|
||||
self.get_shape = P.Shape()
|
||||
self.cast = P.Cast()
|
||||
self.mul_weight = Parameter(mul_weight, "w1")
|
||||
self.mul_weight2 = Parameter(mul_weight, "w2")
|
||||
self.keep_prob = Tensor(0.9)
|
||||
|
||||
def construct(self, x, b):
|
||||
out = self.mul(x, self.mul_weight)
|
||||
shape = self.get_shape(out)
|
||||
dtype = P.DType()(out)
|
||||
keep_prob = self.cast(self.keep_prob, dtype)
|
||||
mask = self.dropout_gen_mask(shape, keep_prob)
|
||||
out = self.dropout_do_mask(out, mask, keep_prob)
|
||||
out = self.mul2(out, self.mul_weight2)
|
||||
return out
|
||||
|
||||
|
||||
_x = Tensor(np.ones([128, 64]), dtype=ms.float32)
|
||||
_w1 = Tensor(np.ones([128, 64]), dtype=ms.float32)
|
||||
_b = Tensor(np.ones([128, 64]), dtype=ms.float32)
|
||||
|
||||
|
||||
def compile(net):
|
||||
optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9)
|
||||
train_net = TrainOneStepCell(net, optimizer)
|
||||
_executor.compile(train_net, _x, _b)
|
||||
context.reset_auto_parallel_context()
|
||||
|
||||
|
||||
def test_dropout_do_mask_data_parallel():
|
||||
context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0)
|
||||
strategy1 = ((16, 1), (16, 1))
|
||||
strategy2 = ((16, 1),)
|
||||
net = Net(_w1, strategy1, strategy2)
|
||||
compile(net)
|
||||
|
||||
|
||||
def test_dropout_do_mask_model_parallel():
|
||||
context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0)
|
||||
strategy1 = ((1, 16), (1, 16))
|
||||
strategy2 = ((1, 16),)
|
||||
net = Net(_w1, strategy1, strategy2)
|
||||
compile(net)
|
||||
|
||||
|
||||
def test_dropout_do_mask_hybrid_parallel():
|
||||
context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0)
|
||||
strategy1 = ((4, 4), (4, 4))
|
||||
strategy2 = ((4, 4),)
|
||||
net = Net(_w1, strategy1, strategy2)
|
||||
compile(net)
|
||||
|
||||
|
||||
def test_dropout_do_mask_auto_parallel():
|
||||
context.set_auto_parallel_context(parallel_mode="auto_parallel", device_num=16, global_rank=0)
|
||||
net = Net(_w1)
|
||||
compile(net)
|
||||
|
||||
|
||||
def test_dropout_do_mask_repeat_calc():
|
||||
context.set_auto_parallel_context(parallel_mode="semi_auto_parallel", device_num=16, global_rank=0)
|
||||
strategy1 = ((4, 4), (4, 4))
|
||||
strategy2 = ((2, 4),)
|
||||
net = Net(_w1, strategy1, strategy2)
|
||||
compile(net)
|
||||
Loading…
Reference in new issue