!3010 mindspore.dataset c_transform support for RandomApply RandomChoice, Compose and RandomSelectSubpolicy
Merge pull request !3010 from ZiruiWu/random_tensor_opspull/3010/MERGE
mindspore-ci-bot
5 years ago
committed by
Gitee
commit
60927ef130
@ -0,0 +1,66 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include "minddata/dataset/kernels/compose_op.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/dataset/core/tensor.h"
|
||||
#include "minddata/dataset/kernels/py_func_op.h"
|
||||
#include "minddata/dataset/kernels/tensor_op.h"
|
||||
#include "minddata/dataset/util/status.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
Status ComposeOp::OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) {
|
||||
std::vector<TensorShape> in_shapes = inputs;
|
||||
for (auto &op : ops_) {
|
||||
RETURN_IF_NOT_OK(op->OutputShape(in_shapes, outputs));
|
||||
in_shapes = std::move(outputs); // outputs become empty after move
|
||||
}
|
||||
outputs = std::move(in_shapes);
|
||||
return Status::OK();
|
||||
}
|
||||
Status ComposeOp::OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) {
|
||||
std::vector<DataType> in_types = inputs;
|
||||
for (auto &op : ops_) {
|
||||
RETURN_IF_NOT_OK(op->OutputType(in_types, outputs));
|
||||
in_types = std::move(outputs); // outputs become empty after move
|
||||
}
|
||||
outputs = std::move(in_types);
|
||||
return Status::OK();
|
||||
}
|
||||
Status ComposeOp::Compute(const TensorRow &inputs, TensorRow *outputs) {
|
||||
IO_CHECK_VECTOR(inputs, outputs);
|
||||
TensorRow in_rows = inputs;
|
||||
for (auto &op : ops_) {
|
||||
RETURN_IF_NOT_OK(op->Compute(in_rows, outputs));
|
||||
in_rows = std::move(*outputs); // after move, *outputs become empty
|
||||
}
|
||||
(*outputs) = std::move(in_rows);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
ComposeOp::ComposeOp(const std::vector<std::shared_ptr<TensorOp>> &ops) : ops_(ops) {
|
||||
if (ops_.empty()) {
|
||||
MS_LOG(ERROR) << "op_list is empty this might lead to Segmentation Fault.";
|
||||
} else if (ops_.size() == 1) {
|
||||
MS_LOG(WARNING) << "op_list has only 1 op. Compose is probably not needed.";
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
@ -0,0 +1,70 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef DATASET_KERNELS_COMPOSE_OP_
|
||||
#define DATASET_KERNELS_COMPOSE_OP_
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/dataset/core/tensor.h"
|
||||
#include "minddata/dataset/kernels/tensor_op.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
class ComposeOp : public TensorOp {
|
||||
public:
|
||||
/// constructor
|
||||
/// \param[in] ops list of TensorOps to compose into 1 TensorOp
|
||||
explicit ComposeOp(const std::vector<std::shared_ptr<TensorOp>> &ops);
|
||||
|
||||
/// default destructor
|
||||
~ComposeOp() override = default;
|
||||
|
||||
/// return the number of inputs the first tensorOp in compose takes
|
||||
/// \return number of input tensors
|
||||
uint32_t NumInput() override { return ops_.front()->NumInput(); }
|
||||
|
||||
/// return the number of outputs the last tensorOp in compose produces
|
||||
/// \return number of output tensors
|
||||
uint32_t NumOutput() override { return ops_.back()->NumOutput(); }
|
||||
|
||||
/// \param[in] inputs
|
||||
/// \param[out] outputs
|
||||
/// \return Status code
|
||||
Status OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) override;
|
||||
|
||||
/// \param[in] inputs
|
||||
/// \param[out] outputs
|
||||
/// \return Status code
|
||||
Status OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) override;
|
||||
|
||||
/// \param[in] input
|
||||
/// \param[out] output
|
||||
/// \return Status code
|
||||
Status Compute(const TensorRow &input, TensorRow *output) override;
|
||||
|
||||
std::string Name() const override { return kComposeOp; }
|
||||
|
||||
private:
|
||||
std::vector<std::shared_ptr<TensorOp>> ops_;
|
||||
};
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // DATASET_KERNELS_COMPOSE_OP_
|
||||
@ -0,0 +1,96 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include "minddata/dataset/kernels/image/random_select_subpolicy_op.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/dataset/core/tensor.h"
|
||||
#include "minddata/dataset/kernels/tensor_op.h"
|
||||
#include "minddata/dataset/util/status.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
Status RandomSelectSubpolicyOp::Compute(const TensorRow &input, TensorRow *output) {
|
||||
TensorRow in_row = input;
|
||||
size_t rand_num = rand_int_(gen_);
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(rand_num < policy_.size(), "invalid rand_num:" + std::to_string(rand_num));
|
||||
for (auto &sub : policy_[rand_num]) {
|
||||
if (rand_double_(gen_) <= sub.second) {
|
||||
RETURN_IF_NOT_OK(sub.first->Compute(in_row, output));
|
||||
in_row = std::move(*output);
|
||||
}
|
||||
}
|
||||
*output = std::move(in_row);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
uint32_t RandomSelectSubpolicyOp::NumInput() {
|
||||
uint32_t num_in = policy_.front().front().first->NumInput();
|
||||
for (auto &sub : policy_) {
|
||||
for (auto p : sub) {
|
||||
if (num_in != p.first->NumInput()) {
|
||||
MS_LOG(WARNING) << "Unable to determine numInput.";
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
return num_in;
|
||||
}
|
||||
|
||||
uint32_t RandomSelectSubpolicyOp::NumOutput() {
|
||||
uint32_t num_out = policy_.front().front().first->NumOutput();
|
||||
for (auto &sub : policy_) {
|
||||
for (auto p : sub) {
|
||||
if (num_out != p.first->NumOutput()) {
|
||||
MS_LOG(WARNING) << "Unable to determine numInput.";
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
return num_out;
|
||||
}
|
||||
|
||||
Status RandomSelectSubpolicyOp::OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) {
|
||||
outputs.clear();
|
||||
outputs.resize(NumOutput(), TensorShape::CreateUnknownRankShape());
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status RandomSelectSubpolicyOp::OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) {
|
||||
RETURN_IF_NOT_OK(policy_.front().front().first->OutputType(inputs, outputs));
|
||||
for (auto &sub : policy_) {
|
||||
for (auto p : sub) {
|
||||
std::vector<DataType> tmp_types;
|
||||
RETURN_IF_NOT_OK(p.first->OutputType(inputs, tmp_types));
|
||||
if (outputs != tmp_types) {
|
||||
outputs.clear();
|
||||
outputs.resize(NumOutput(), DataType(DataType::DE_UNKNOWN));
|
||||
return Status::OK();
|
||||
}
|
||||
}
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
RandomSelectSubpolicyOp::RandomSelectSubpolicyOp(const std::vector<Subpolicy> &policy)
|
||||
: gen_(GetSeed()), policy_(policy), rand_int_(0, policy.size() - 1), rand_double_(0, 1) {
|
||||
if (policy_.empty()) {
|
||||
MS_LOG(ERROR) << "policy in RandomSelectSubpolicyOp is empty.";
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
@ -0,0 +1,79 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef DATASET_KERNELS_IMAGE_RANDOM_SELECT_SUBPOLICY_OP_
|
||||
#define DATASET_KERNELS_IMAGE_RANDOM_SELECT_SUBPOLICY_OP_
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/dataset/core/tensor.h"
|
||||
#include "minddata/dataset/kernels/tensor_op.h"
|
||||
#include "minddata/dataset/util/random.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
using Subpolicy = std::vector<std::pair<std::shared_ptr<TensorOp>, double>>;
|
||||
|
||||
class RandomSelectSubpolicyOp : public TensorOp {
|
||||
public:
|
||||
/// constructor
|
||||
/// \param[in] policy policy to choose subpolicy from
|
||||
explicit RandomSelectSubpolicyOp(const std::vector<Subpolicy> &policy);
|
||||
|
||||
/// destructor
|
||||
~RandomSelectSubpolicyOp() override = default;
|
||||
|
||||
/// return number of input tensors
|
||||
/// \return number of inputs if all ops in policy have the same NumInput, otherwise return 0
|
||||
uint32_t NumInput() override;
|
||||
|
||||
/// return number of output tensors
|
||||
/// \return number of outputs if all ops in policy have the same NumOutput, otherwise return 0
|
||||
uint32_t NumOutput() override;
|
||||
|
||||
/// return unknown shapes
|
||||
/// \param[in] inputs
|
||||
/// \param[out] outputs
|
||||
/// \return Status Code
|
||||
Status OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) override;
|
||||
|
||||
/// return output type if all ops in policy return the same type, otherwise return unknown type
|
||||
/// \param[in] inputs
|
||||
/// \param[out] outputs
|
||||
/// \return Status Code
|
||||
Status OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) override;
|
||||
|
||||
/// \param[in] input
|
||||
/// \param[out] output
|
||||
/// \return Status code
|
||||
Status Compute(const TensorRow &input, TensorRow *output) override;
|
||||
|
||||
std::string Name() const override { return kRandomSelectSubpolicyOp; }
|
||||
|
||||
private:
|
||||
std::vector<Subpolicy> policy_;
|
||||
std::mt19937 gen_; // mersenne_twister_engine
|
||||
std::uniform_int_distribution<size_t> rand_int_;
|
||||
std::uniform_real_distribution<double> rand_double_;
|
||||
};
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // DATASET_KERNELS_IMAGE_RANDOM_SELECT_SUBPOLICY_OP_
|
||||
@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include "minddata/dataset/kernels/random_apply_op.h"
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/dataset/core/tensor.h"
|
||||
#include "minddata/dataset/kernels/tensor_op.h"
|
||||
#include "minddata/dataset/util/status.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
uint32_t RandomApplyOp::NumOutput() {
|
||||
if (compose_->NumOutput() != NumInput()) {
|
||||
MS_LOG(WARNING) << "NumOutput!=NumInput (randomApply would randomly affect number of outputs).";
|
||||
return 0;
|
||||
}
|
||||
return compose_->NumOutput();
|
||||
}
|
||||
|
||||
Status RandomApplyOp::OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) {
|
||||
RETURN_IF_NOT_OK(compose_->OutputShape(inputs, outputs));
|
||||
// randomApply either runs all ops or do nothing. If the two methods don't give the same result. return unknown shape.
|
||||
if (inputs != outputs) { // when RandomApply is not applied, input should be the same as output
|
||||
outputs.clear();
|
||||
outputs.resize(NumOutput(), TensorShape::CreateUnknownRankShape());
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
Status RandomApplyOp::OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) {
|
||||
RETURN_IF_NOT_OK(compose_->OutputType(inputs, outputs));
|
||||
if (inputs != outputs) { // when RandomApply is not applied, input should be the same as output
|
||||
outputs.clear();
|
||||
outputs.resize(NumOutput(), DataType(DataType::DE_UNKNOWN));
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
Status RandomApplyOp::Compute(const TensorRow &input, TensorRow *output) {
|
||||
if (rand_double_(gen_) <= prob_) {
|
||||
RETURN_IF_NOT_OK(compose_->Compute(input, output));
|
||||
} else {
|
||||
IO_CHECK_VECTOR(input, output);
|
||||
*output = input; // copy over the tensors
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
RandomApplyOp::RandomApplyOp(double prob, const std::vector<std::shared_ptr<TensorOp>> &ops)
|
||||
: prob_(prob), gen_(GetSeed()), rand_double_(0, 1) {
|
||||
compose_ = std::make_unique<ComposeOp>(ops);
|
||||
}
|
||||
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
@ -0,0 +1,79 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef DATASET_KERNELS_RANDOM_APPLY_OP_
|
||||
#define DATASET_KERNELS_RANDOM_APPLY_OP_
|
||||
|
||||
#include <memory>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/dataset/core/tensor.h"
|
||||
#include "minddata/dataset/kernels/compose_op.h"
|
||||
#include "minddata/dataset/kernels/tensor_op.h"
|
||||
#include "minddata/dataset/util/random.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
class RandomApplyOp : public TensorOp {
|
||||
public:
|
||||
/// constructor
|
||||
/// \param[in] prob probability whether the list of TensorOps will be applied
|
||||
/// \param[in] ops the list of TensorOps to apply with prob likelihood
|
||||
explicit RandomApplyOp(double prob, const std::vector<std::shared_ptr<TensorOp>> &ops);
|
||||
|
||||
/// default destructor
|
||||
~RandomApplyOp() = default;
|
||||
|
||||
/// return the number of inputs the first tensorOp in compose takes
|
||||
/// \return number of input tensors
|
||||
uint32_t NumInput() override { return compose_->NumInput(); }
|
||||
|
||||
/// return the number of outputs
|
||||
/// \return number of output tensors
|
||||
uint32_t NumOutput() override;
|
||||
|
||||
/// return output shape if randomApply won't affect the output shape, otherwise return unknown shape
|
||||
/// \param[in] inputs
|
||||
/// \param[out] outputs
|
||||
/// \return Status code
|
||||
Status OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) override;
|
||||
|
||||
/// return output type if randomApply won't affect the output type, otherwise return unknown type
|
||||
/// \param[in] inputs
|
||||
/// \param[out] outputs
|
||||
/// \return Status code
|
||||
Status OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) override;
|
||||
|
||||
/// \param[in] input
|
||||
/// \param[out] output
|
||||
/// \return Status code
|
||||
Status Compute(const TensorRow &input, TensorRow *output) override;
|
||||
|
||||
std::string Name() const override { return kRandomApplyOp; }
|
||||
|
||||
private:
|
||||
double prob_;
|
||||
std::shared_ptr<TensorOp> compose_;
|
||||
std::mt19937 gen_; // mersenne_twister_engine
|
||||
std::uniform_real_distribution<double> rand_double_;
|
||||
};
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // DATASET_KERNELS_RANDOM_APPLY_OP_
|
||||
@ -0,0 +1,97 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include "minddata/dataset/kernels/random_choice_op.h"
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/dataset/core/tensor.h"
|
||||
#include "minddata/dataset/kernels/tensor_op.h"
|
||||
#include "minddata/dataset/util/status.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
uint32_t RandomChoiceOp::NumInput() {
|
||||
uint32_t num_input = ops_.front()->NumInput();
|
||||
for (auto &op : ops_) {
|
||||
uint32_t cur_num = op->NumInput();
|
||||
if (num_input != cur_num && cur_num > 0) {
|
||||
MS_LOG(WARNING) << "Unable to determine NumInput, ops in RandomChoice don't take the same number of input.";
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
return num_input;
|
||||
}
|
||||
|
||||
uint32_t RandomChoiceOp::NumOutput() {
|
||||
uint32_t num_output = ops_.front()->NumOutput();
|
||||
for (auto &op : ops_) {
|
||||
uint32_t cur_num = op->NumOutput();
|
||||
if (num_output != cur_num) {
|
||||
MS_LOG(WARNING) << "Unable to determine NumInput, ops in RandomChoice don't have the same number of input.";
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
return num_output;
|
||||
}
|
||||
|
||||
Status RandomChoiceOp::OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) {
|
||||
RETURN_IF_NOT_OK(ops_.front()->OutputShape(inputs, outputs));
|
||||
for (auto &op : ops_) {
|
||||
std::vector<TensorShape> out_shapes;
|
||||
RETURN_IF_NOT_OK(op->OutputShape(inputs, out_shapes));
|
||||
if (outputs != out_shapes) {
|
||||
MS_LOG(WARNING) << "TensorOp in RandomChoice don't return the same tensorShape.";
|
||||
outputs.clear();
|
||||
outputs.resize(NumOutput(), TensorShape::CreateUnknownRankShape());
|
||||
return Status::OK();
|
||||
}
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status RandomChoiceOp::OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) {
|
||||
RETURN_IF_NOT_OK(ops_.front()->OutputType(inputs, outputs));
|
||||
for (auto &op : ops_) {
|
||||
std::vector<DataType> out_types;
|
||||
RETURN_IF_NOT_OK(op->OutputType(inputs, out_types));
|
||||
if (outputs != out_types) {
|
||||
MS_LOG(WARNING) << "TensorOp in RandomChoice don't return the same tensorType.";
|
||||
outputs.clear();
|
||||
outputs.resize(NumOutput(), DataType(DataType::DE_UNKNOWN));
|
||||
return Status::OK();
|
||||
}
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status RandomChoiceOp::Compute(const TensorRow &input, TensorRow *output) {
|
||||
size_t rand_num = rand_int_(gen_);
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(rand_num < ops_.size(), "invalid rand_num:" + std::to_string(rand_num));
|
||||
RETURN_IF_NOT_OK(ops_[rand_num]->Compute(input, output));
|
||||
return Status::OK();
|
||||
}
|
||||
RandomChoiceOp::RandomChoiceOp(const std::vector<std::shared_ptr<TensorOp>> &ops)
|
||||
: ops_(ops), gen_(GetSeed()), rand_int_(0, ops.size() - 1) {
|
||||
if (ops_.empty()) {
|
||||
MS_LOG(ERROR) << "op_list in RandomChoiceOp is empty.";
|
||||
} else if (ops_.size() == 1) {
|
||||
MS_LOG(WARNING) << "op_list has only 1 op, this op would be picked every time.";
|
||||
}
|
||||
}
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
@ -0,0 +1,77 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef DATASET_KERNELS_RANDOM_CHOICE_OP_
|
||||
#define DATASET_KERNELS_RANDOM_CHOICE_OP_
|
||||
|
||||
#include <memory>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/dataset/core/tensor.h"
|
||||
#include "minddata/dataset/kernels/tensor_op.h"
|
||||
#include "minddata/dataset/kernels/compose_op.h"
|
||||
#include "minddata/dataset/util/random.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
class RandomChoiceOp : public TensorOp {
|
||||
public:
|
||||
/// constructor
|
||||
/// \param[in] ops list of TensorOps to randomly choose 1 from
|
||||
explicit RandomChoiceOp(const std::vector<std::shared_ptr<TensorOp>> &ops);
|
||||
|
||||
/// default destructor
|
||||
~RandomChoiceOp() = default;
|
||||
|
||||
/// return the number of inputs. All op in ops_ should have the same number of inputs
|
||||
/// \return number of input tensors
|
||||
uint32_t NumInput() override;
|
||||
|
||||
/// return the number of outputs. All op in ops_ should have the same number of outputs
|
||||
/// \return number of input tensors
|
||||
uint32_t NumOutput() override;
|
||||
|
||||
/// return output shape if all ops in ops_ return the same shape, otherwise return unknown shape
|
||||
/// \param[in] inputs
|
||||
/// \param[out] outputs
|
||||
/// \return Status code
|
||||
Status OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) override;
|
||||
|
||||
/// return output type if all ops in ops_ return the same type, otherwise return unknown type
|
||||
/// \param[in] inputs
|
||||
/// \param[out] outputs
|
||||
/// \return Status code
|
||||
Status OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) override;
|
||||
|
||||
/// \param[in] input
|
||||
/// \param[out] output
|
||||
/// \return Status code
|
||||
Status Compute(const TensorRow &input, TensorRow *output) override;
|
||||
|
||||
std::string Name() const override { return kRandomChoiceOp; }
|
||||
|
||||
private:
|
||||
std::vector<std::shared_ptr<TensorOp>> ops_;
|
||||
std::mt19937 gen_; // mersenne_twister_engine
|
||||
std::uniform_int_distribution<size_t> rand_int_;
|
||||
};
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // DATASET_KERNELS_RANDOM_CHOICE_OP_
|
||||
@ -0,0 +1,50 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
import mindspore.common.dtype as mstype
|
||||
import mindspore.dataset as ds
|
||||
import mindspore.dataset.transforms.c_transforms as ops
|
||||
import mindspore.dataset.transforms.py_transforms as py_ops
|
||||
|
||||
|
||||
def test_compose():
|
||||
ds.config.set_seed(0)
|
||||
|
||||
def test_config(arr, op_list):
|
||||
try:
|
||||
data = ds.NumpySlicesDataset(arr, column_names="col", shuffle=False)
|
||||
data = data.map(input_columns=["col"], operations=ops.Compose(op_list))
|
||||
res = []
|
||||
for i in data.create_dict_iterator():
|
||||
res.append(i["col"].tolist())
|
||||
return res
|
||||
except (TypeError, ValueError) as e:
|
||||
return str(e)
|
||||
|
||||
# test simple compose with only 1 op, this would generate a warning
|
||||
assert test_config([[1, 0], [3, 4]], [ops.Fill(2)]) == [[2, 2], [2, 2]]
|
||||
# test 1 column -> 2columns -> 1 -> 2 -> 1
|
||||
assert test_config([[1, 0]], [ops.Duplicate(), ops.Concatenate(), ops.Duplicate(), ops.Concatenate()]) == [
|
||||
[1, 0] * 4]
|
||||
# test one python transform followed by a C transform. type after oneHot is float (mixed use-case)
|
||||
assert test_config([1, 0], [py_ops.OneHotOp(2), ops.TypeCast(mstype.int32)]) == [[[0, 1]], [[1, 0]]]
|
||||
# test exceptions. compose, randomApply randomChoice use the same validator
|
||||
assert "op_list[0] is not a c_transform op" in test_config([1, 0], [1, ops.TypeCast(mstype.int32)])
|
||||
# test empty op list
|
||||
assert "op_list can not be empty." in test_config([1, 0], [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_compose()
|
||||
@ -0,0 +1,48 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
import mindspore.common.dtype as mstype
|
||||
import mindspore.dataset as ds
|
||||
import mindspore.dataset.transforms.c_transforms as ops
|
||||
|
||||
|
||||
def test_random_apply():
|
||||
ds.config.set_seed(0)
|
||||
|
||||
def test_config(arr, op_list, prob=0.5):
|
||||
try:
|
||||
data = ds.NumpySlicesDataset(arr, column_names="col", shuffle=False)
|
||||
data = data.map(input_columns=["col"], operations=ops.RandomApply(op_list, prob))
|
||||
res = []
|
||||
for i in data.create_dict_iterator():
|
||||
res.append(i["col"].tolist())
|
||||
return res
|
||||
except (TypeError, ValueError) as e:
|
||||
return str(e)
|
||||
|
||||
res1 = test_config([[0, 1]], [ops.Duplicate(), ops.Concatenate()])
|
||||
assert res1 in [[[0, 1]], [[0, 1, 0, 1]]]
|
||||
# test single nested compose
|
||||
assert test_config([[0, 1, 2]], [ops.Compose([ops.Duplicate(), ops.Concatenate(), ops.Slice([0, 1, 2])])]) == [
|
||||
[0, 1, 2]]
|
||||
# test exception
|
||||
assert "is not of type (<class 'list'>" in test_config([1, 0], ops.TypeCast(mstype.int32))
|
||||
assert "Input prob is not within the required interval" in test_config([0, 1], [ops.Slice([0, 1])], 1.1)
|
||||
assert "is not of type (<class 'float'>" in test_config([1, 0], [ops.TypeCast(mstype.int32)], None)
|
||||
assert "op_list with value None is not of type (<class 'list'>" in test_config([1, 0], None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_random_apply()
|
||||
@ -0,0 +1,48 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
|
||||
import mindspore.dataset as ds
|
||||
import mindspore.dataset.transforms.c_transforms as ops
|
||||
|
||||
|
||||
def test_random_choice():
|
||||
ds.config.set_seed(0)
|
||||
|
||||
def test_config(arr, op_list):
|
||||
try:
|
||||
data = ds.NumpySlicesDataset(arr, column_names="col", shuffle=False)
|
||||
data = data.map(input_columns=["col"], operations=ops.RandomChoice(op_list))
|
||||
res = []
|
||||
for i in data.create_dict_iterator():
|
||||
res.append(i["col"].tolist())
|
||||
return res
|
||||
except (TypeError, ValueError) as e:
|
||||
return str(e)
|
||||
|
||||
# test whether a op would be randomly chosen. In order to prevent random failure, both results need to be checked
|
||||
res1 = test_config([[0, 1, 2]], [ops.PadEnd([4], 0), ops.Slice([0, 2])])
|
||||
assert res1 in [[[0, 1, 2, 0]], [[0, 2]]]
|
||||
|
||||
# test nested structure
|
||||
res2 = test_config([[0, 1, 2]], [ops.Compose([ops.Duplicate(), ops.Concatenate()]),
|
||||
ops.Compose([ops.Slice([0, 1]), ops.OneHot(2)])])
|
||||
assert res2 in [[[[1, 0], [0, 1]]], [[0, 1, 2, 0, 1, 2]]]
|
||||
# test random_choice where there is only 1 op
|
||||
assert test_config([[4, 3], [2, 1]], [ops.Slice([0])]) == [[4], [2]]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_random_choice()
|
||||
@ -0,0 +1,51 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
import mindspore.dataset as ds
|
||||
import mindspore.dataset.transforms.c_transforms as ops
|
||||
import mindspore.dataset.transforms.vision.c_transforms as visions
|
||||
|
||||
|
||||
def test_random_select_subpolicy():
|
||||
ds.config.set_seed(0)
|
||||
|
||||
def test_config(arr, policy):
|
||||
try:
|
||||
data = ds.NumpySlicesDataset(arr, column_names="col", shuffle=False)
|
||||
data = data.map(input_columns=["col"], operations=visions.RandomSelectSubpolicy(policy))
|
||||
res = []
|
||||
for i in data.create_dict_iterator():
|
||||
res.append(i["col"].tolist())
|
||||
return res
|
||||
except (TypeError, ValueError) as e:
|
||||
return str(e)
|
||||
|
||||
# 3 possible outcomes
|
||||
policy1 = [[(ops.PadEnd([4], 0), 0.5), (ops.Compose([ops.Duplicate(), ops.Concatenate()]), 1)],
|
||||
[(ops.Slice([0, 1]), 0.5), (ops.Duplicate(), 1), (ops.Concatenate(), 1)]]
|
||||
res1 = test_config([[1, 2, 3]], policy1)
|
||||
assert res1 in [[[1, 2, 1, 2]], [[1, 2, 3, 1, 2, 3]], [[1, 2, 3, 0, 1, 2, 3, 0]]]
|
||||
|
||||
# test exceptions
|
||||
assert "policy can not be empty." in test_config([[1, 2, 3]], [])
|
||||
assert "policy[0] can not be empty." in test_config([[1, 2, 3]], [[]])
|
||||
assert "op of (op, prob) in policy[1][0] is not a c_transform op (TensorOp) nor a callable pyfunc" in test_config(
|
||||
[[1, 2, 3]], [[(ops.PadEnd([4], 0), 0.5)], [(1, 0.4)]])
|
||||
assert "prob of (op, prob) policy[1][0] is not within the required interval of (0 to 1)" in test_config([[1]], [
|
||||
[(ops.Duplicate(), 0)], [(ops.Duplicate(), -0.1)]])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_random_select_subpolicy()
|
||||
Loading…
Reference in new issue