python part of random ops added random select sub policy validators added comments added, remaining issues addressed add more python test cases fix ci fix CI fix order of include files addr review cmts addr review cmts reorg file fix compile err address review cmts address review cmtspull/3010/head
Zirui Wu
5 years ago
parent
cda333f760
commit
d233c54139
@ -0,0 +1,66 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include "minddata/dataset/kernels/compose_op.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/dataset/core/tensor.h"
|
||||
#include "minddata/dataset/kernels/py_func_op.h"
|
||||
#include "minddata/dataset/kernels/tensor_op.h"
|
||||
#include "minddata/dataset/util/status.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
Status ComposeOp::OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) {
|
||||
std::vector<TensorShape> in_shapes = inputs;
|
||||
for (auto &op : ops_) {
|
||||
RETURN_IF_NOT_OK(op->OutputShape(in_shapes, outputs));
|
||||
in_shapes = std::move(outputs); // outputs become empty after move
|
||||
}
|
||||
outputs = std::move(in_shapes);
|
||||
return Status::OK();
|
||||
}
|
||||
Status ComposeOp::OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) {
|
||||
std::vector<DataType> in_types = inputs;
|
||||
for (auto &op : ops_) {
|
||||
RETURN_IF_NOT_OK(op->OutputType(in_types, outputs));
|
||||
in_types = std::move(outputs); // outputs become empty after move
|
||||
}
|
||||
outputs = std::move(in_types);
|
||||
return Status::OK();
|
||||
}
|
||||
Status ComposeOp::Compute(const TensorRow &inputs, TensorRow *outputs) {
|
||||
IO_CHECK_VECTOR(inputs, outputs);
|
||||
TensorRow in_rows = inputs;
|
||||
for (auto &op : ops_) {
|
||||
RETURN_IF_NOT_OK(op->Compute(in_rows, outputs));
|
||||
in_rows = std::move(*outputs); // after move, *outputs become empty
|
||||
}
|
||||
(*outputs) = std::move(in_rows);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
ComposeOp::ComposeOp(const std::vector<std::shared_ptr<TensorOp>> &ops) : ops_(ops) {
|
||||
if (ops_.empty()) {
|
||||
MS_LOG(ERROR) << "op_list is empty this might lead to Segmentation Fault.";
|
||||
} else if (ops_.size() == 1) {
|
||||
MS_LOG(WARNING) << "op_list has only 1 op. Compose is probably not needed.";
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
@ -0,0 +1,70 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef DATASET_KERNELS_COMPOSE_OP_
|
||||
#define DATASET_KERNELS_COMPOSE_OP_
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/dataset/core/tensor.h"
|
||||
#include "minddata/dataset/kernels/tensor_op.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
class ComposeOp : public TensorOp {
|
||||
public:
|
||||
/// constructor
|
||||
/// \param[in] ops list of TensorOps to compose into 1 TensorOp
|
||||
explicit ComposeOp(const std::vector<std::shared_ptr<TensorOp>> &ops);
|
||||
|
||||
/// default destructor
|
||||
~ComposeOp() override = default;
|
||||
|
||||
/// return the number of inputs the first tensorOp in compose takes
|
||||
/// \return number of input tensors
|
||||
uint32_t NumInput() override { return ops_.front()->NumInput(); }
|
||||
|
||||
/// return the number of outputs the last tensorOp in compose produces
|
||||
/// \return number of output tensors
|
||||
uint32_t NumOutput() override { return ops_.back()->NumOutput(); }
|
||||
|
||||
/// \param[in] inputs
|
||||
/// \param[out] outputs
|
||||
/// \return Status code
|
||||
Status OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) override;
|
||||
|
||||
/// \param[in] inputs
|
||||
/// \param[out] outputs
|
||||
/// \return Status code
|
||||
Status OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) override;
|
||||
|
||||
/// \param[in] input
|
||||
/// \param[out] output
|
||||
/// \return Status code
|
||||
Status Compute(const TensorRow &input, TensorRow *output) override;
|
||||
|
||||
std::string Name() const override { return kComposeOp; }
|
||||
|
||||
private:
|
||||
std::vector<std::shared_ptr<TensorOp>> ops_;
|
||||
};
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // DATASET_KERNELS_COMPOSE_OP_
|
||||
@ -0,0 +1,96 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include "minddata/dataset/kernels/image/random_select_subpolicy_op.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/dataset/core/tensor.h"
|
||||
#include "minddata/dataset/kernels/tensor_op.h"
|
||||
#include "minddata/dataset/util/status.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
Status RandomSelectSubpolicyOp::Compute(const TensorRow &input, TensorRow *output) {
|
||||
TensorRow in_row = input;
|
||||
size_t rand_num = rand_int_(gen_);
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(rand_num < policy_.size(), "invalid rand_num:" + std::to_string(rand_num));
|
||||
for (auto &sub : policy_[rand_num]) {
|
||||
if (rand_double_(gen_) <= sub.second) {
|
||||
RETURN_IF_NOT_OK(sub.first->Compute(in_row, output));
|
||||
in_row = std::move(*output);
|
||||
}
|
||||
}
|
||||
*output = std::move(in_row);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
uint32_t RandomSelectSubpolicyOp::NumInput() {
|
||||
uint32_t num_in = policy_.front().front().first->NumInput();
|
||||
for (auto &sub : policy_) {
|
||||
for (auto p : sub) {
|
||||
if (num_in != p.first->NumInput()) {
|
||||
MS_LOG(WARNING) << "Unable to determine numInput.";
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
return num_in;
|
||||
}
|
||||
|
||||
uint32_t RandomSelectSubpolicyOp::NumOutput() {
|
||||
uint32_t num_out = policy_.front().front().first->NumOutput();
|
||||
for (auto &sub : policy_) {
|
||||
for (auto p : sub) {
|
||||
if (num_out != p.first->NumOutput()) {
|
||||
MS_LOG(WARNING) << "Unable to determine numInput.";
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
return num_out;
|
||||
}
|
||||
|
||||
Status RandomSelectSubpolicyOp::OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) {
|
||||
outputs.clear();
|
||||
outputs.resize(NumOutput(), TensorShape::CreateUnknownRankShape());
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status RandomSelectSubpolicyOp::OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) {
|
||||
RETURN_IF_NOT_OK(policy_.front().front().first->OutputType(inputs, outputs));
|
||||
for (auto &sub : policy_) {
|
||||
for (auto p : sub) {
|
||||
std::vector<DataType> tmp_types;
|
||||
RETURN_IF_NOT_OK(p.first->OutputType(inputs, tmp_types));
|
||||
if (outputs != tmp_types) {
|
||||
outputs.clear();
|
||||
outputs.resize(NumOutput(), DataType(DataType::DE_UNKNOWN));
|
||||
return Status::OK();
|
||||
}
|
||||
}
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
RandomSelectSubpolicyOp::RandomSelectSubpolicyOp(const std::vector<Subpolicy> &policy)
|
||||
: gen_(GetSeed()), policy_(policy), rand_int_(0, policy.size() - 1), rand_double_(0, 1) {
|
||||
if (policy_.empty()) {
|
||||
MS_LOG(ERROR) << "policy in RandomSelectSubpolicyOp is empty.";
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
@ -0,0 +1,79 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef DATASET_KERNELS_IMAGE_RANDOM_SELECT_SUBPOLICY_OP_
|
||||
#define DATASET_KERNELS_IMAGE_RANDOM_SELECT_SUBPOLICY_OP_
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/dataset/core/tensor.h"
|
||||
#include "minddata/dataset/kernels/tensor_op.h"
|
||||
#include "minddata/dataset/util/random.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
using Subpolicy = std::vector<std::pair<std::shared_ptr<TensorOp>, double>>;
|
||||
|
||||
class RandomSelectSubpolicyOp : public TensorOp {
|
||||
public:
|
||||
/// constructor
|
||||
/// \param[in] policy policy to choose subpolicy from
|
||||
explicit RandomSelectSubpolicyOp(const std::vector<Subpolicy> &policy);
|
||||
|
||||
/// destructor
|
||||
~RandomSelectSubpolicyOp() override = default;
|
||||
|
||||
/// return number of input tensors
|
||||
/// \return number of inputs if all ops in policy have the same NumInput, otherwise return 0
|
||||
uint32_t NumInput() override;
|
||||
|
||||
/// return number of output tensors
|
||||
/// \return number of outputs if all ops in policy have the same NumOutput, otherwise return 0
|
||||
uint32_t NumOutput() override;
|
||||
|
||||
/// return unknown shapes
|
||||
/// \param[in] inputs
|
||||
/// \param[out] outputs
|
||||
/// \return Status Code
|
||||
Status OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) override;
|
||||
|
||||
/// return output type if all ops in policy return the same type, otherwise return unknown type
|
||||
/// \param[in] inputs
|
||||
/// \param[out] outputs
|
||||
/// \return Status Code
|
||||
Status OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) override;
|
||||
|
||||
/// \param[in] input
|
||||
/// \param[out] output
|
||||
/// \return Status code
|
||||
Status Compute(const TensorRow &input, TensorRow *output) override;
|
||||
|
||||
std::string Name() const override { return kRandomSelectSubpolicyOp; }
|
||||
|
||||
private:
|
||||
std::vector<Subpolicy> policy_;
|
||||
std::mt19937 gen_; // mersenne_twister_engine
|
||||
std::uniform_int_distribution<size_t> rand_int_;
|
||||
std::uniform_real_distribution<double> rand_double_;
|
||||
};
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // DATASET_KERNELS_IMAGE_RANDOM_SELECT_SUBPOLICY_OP_
|
||||
@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include "minddata/dataset/kernels/random_apply_op.h"
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/dataset/core/tensor.h"
|
||||
#include "minddata/dataset/kernels/tensor_op.h"
|
||||
#include "minddata/dataset/util/status.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
uint32_t RandomApplyOp::NumOutput() {
|
||||
if (compose_->NumOutput() != NumInput()) {
|
||||
MS_LOG(WARNING) << "NumOutput!=NumInput (randomApply would randomly affect number of outputs).";
|
||||
return 0;
|
||||
}
|
||||
return compose_->NumOutput();
|
||||
}
|
||||
|
||||
Status RandomApplyOp::OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) {
|
||||
RETURN_IF_NOT_OK(compose_->OutputShape(inputs, outputs));
|
||||
// randomApply either runs all ops or do nothing. If the two methods don't give the same result. return unknown shape.
|
||||
if (inputs != outputs) { // when RandomApply is not applied, input should be the same as output
|
||||
outputs.clear();
|
||||
outputs.resize(NumOutput(), TensorShape::CreateUnknownRankShape());
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
Status RandomApplyOp::OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) {
|
||||
RETURN_IF_NOT_OK(compose_->OutputType(inputs, outputs));
|
||||
if (inputs != outputs) { // when RandomApply is not applied, input should be the same as output
|
||||
outputs.clear();
|
||||
outputs.resize(NumOutput(), DataType(DataType::DE_UNKNOWN));
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
Status RandomApplyOp::Compute(const TensorRow &input, TensorRow *output) {
|
||||
if (rand_double_(gen_) <= prob_) {
|
||||
RETURN_IF_NOT_OK(compose_->Compute(input, output));
|
||||
} else {
|
||||
IO_CHECK_VECTOR(input, output);
|
||||
*output = input; // copy over the tensors
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
RandomApplyOp::RandomApplyOp(double prob, const std::vector<std::shared_ptr<TensorOp>> &ops)
|
||||
: prob_(prob), gen_(GetSeed()), rand_double_(0, 1) {
|
||||
compose_ = std::make_unique<ComposeOp>(ops);
|
||||
}
|
||||
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
@ -0,0 +1,79 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef DATASET_KERNELS_RANDOM_APPLY_OP_
|
||||
#define DATASET_KERNELS_RANDOM_APPLY_OP_
|
||||
|
||||
#include <memory>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/dataset/core/tensor.h"
|
||||
#include "minddata/dataset/kernels/compose_op.h"
|
||||
#include "minddata/dataset/kernels/tensor_op.h"
|
||||
#include "minddata/dataset/util/random.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
class RandomApplyOp : public TensorOp {
|
||||
public:
|
||||
/// constructor
|
||||
/// \param[in] prob probability whether the list of TensorOps will be applied
|
||||
/// \param[in] ops the list of TensorOps to apply with prob likelihood
|
||||
explicit RandomApplyOp(double prob, const std::vector<std::shared_ptr<TensorOp>> &ops);
|
||||
|
||||
/// default destructor
|
||||
~RandomApplyOp() = default;
|
||||
|
||||
/// return the number of inputs the first tensorOp in compose takes
|
||||
/// \return number of input tensors
|
||||
uint32_t NumInput() override { return compose_->NumInput(); }
|
||||
|
||||
/// return the number of outputs
|
||||
/// \return number of output tensors
|
||||
uint32_t NumOutput() override;
|
||||
|
||||
/// return output shape if randomApply won't affect the output shape, otherwise return unknown shape
|
||||
/// \param[in] inputs
|
||||
/// \param[out] outputs
|
||||
/// \return Status code
|
||||
Status OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) override;
|
||||
|
||||
/// return output type if randomApply won't affect the output type, otherwise return unknown type
|
||||
/// \param[in] inputs
|
||||
/// \param[out] outputs
|
||||
/// \return Status code
|
||||
Status OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) override;
|
||||
|
||||
/// \param[in] input
|
||||
/// \param[out] output
|
||||
/// \return Status code
|
||||
Status Compute(const TensorRow &input, TensorRow *output) override;
|
||||
|
||||
std::string Name() const override { return kRandomApplyOp; }
|
||||
|
||||
private:
|
||||
double prob_;
|
||||
std::shared_ptr<TensorOp> compose_;
|
||||
std::mt19937 gen_; // mersenne_twister_engine
|
||||
std::uniform_real_distribution<double> rand_double_;
|
||||
};
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // DATASET_KERNELS_RANDOM_APPLY_OP_
|
||||
@ -0,0 +1,97 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include "minddata/dataset/kernels/random_choice_op.h"
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/dataset/core/tensor.h"
|
||||
#include "minddata/dataset/kernels/tensor_op.h"
|
||||
#include "minddata/dataset/util/status.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
uint32_t RandomChoiceOp::NumInput() {
|
||||
uint32_t num_input = ops_.front()->NumInput();
|
||||
for (auto &op : ops_) {
|
||||
uint32_t cur_num = op->NumInput();
|
||||
if (num_input != cur_num && cur_num > 0) {
|
||||
MS_LOG(WARNING) << "Unable to determine NumInput, ops in RandomChoice don't take the same number of input.";
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
return num_input;
|
||||
}
|
||||
|
||||
uint32_t RandomChoiceOp::NumOutput() {
|
||||
uint32_t num_output = ops_.front()->NumOutput();
|
||||
for (auto &op : ops_) {
|
||||
uint32_t cur_num = op->NumOutput();
|
||||
if (num_output != cur_num) {
|
||||
MS_LOG(WARNING) << "Unable to determine NumInput, ops in RandomChoice don't have the same number of input.";
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
return num_output;
|
||||
}
|
||||
|
||||
Status RandomChoiceOp::OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) {
|
||||
RETURN_IF_NOT_OK(ops_.front()->OutputShape(inputs, outputs));
|
||||
for (auto &op : ops_) {
|
||||
std::vector<TensorShape> out_shapes;
|
||||
RETURN_IF_NOT_OK(op->OutputShape(inputs, out_shapes));
|
||||
if (outputs != out_shapes) {
|
||||
MS_LOG(WARNING) << "TensorOp in RandomChoice don't return the same tensorShape.";
|
||||
outputs.clear();
|
||||
outputs.resize(NumOutput(), TensorShape::CreateUnknownRankShape());
|
||||
return Status::OK();
|
||||
}
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status RandomChoiceOp::OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) {
|
||||
RETURN_IF_NOT_OK(ops_.front()->OutputType(inputs, outputs));
|
||||
for (auto &op : ops_) {
|
||||
std::vector<DataType> out_types;
|
||||
RETURN_IF_NOT_OK(op->OutputType(inputs, out_types));
|
||||
if (outputs != out_types) {
|
||||
MS_LOG(WARNING) << "TensorOp in RandomChoice don't return the same tensorType.";
|
||||
outputs.clear();
|
||||
outputs.resize(NumOutput(), DataType(DataType::DE_UNKNOWN));
|
||||
return Status::OK();
|
||||
}
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status RandomChoiceOp::Compute(const TensorRow &input, TensorRow *output) {
|
||||
size_t rand_num = rand_int_(gen_);
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(rand_num < ops_.size(), "invalid rand_num:" + std::to_string(rand_num));
|
||||
RETURN_IF_NOT_OK(ops_[rand_num]->Compute(input, output));
|
||||
return Status::OK();
|
||||
}
|
||||
RandomChoiceOp::RandomChoiceOp(const std::vector<std::shared_ptr<TensorOp>> &ops)
|
||||
: ops_(ops), gen_(GetSeed()), rand_int_(0, ops.size() - 1) {
|
||||
if (ops_.empty()) {
|
||||
MS_LOG(ERROR) << "op_list in RandomChoiceOp is empty.";
|
||||
} else if (ops_.size() == 1) {
|
||||
MS_LOG(WARNING) << "op_list has only 1 op, this op would be picked every time.";
|
||||
}
|
||||
}
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
@ -0,0 +1,77 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef DATASET_KERNELS_RANDOM_CHOICE_OP_
|
||||
#define DATASET_KERNELS_RANDOM_CHOICE_OP_
|
||||
|
||||
#include <memory>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/dataset/core/tensor.h"
|
||||
#include "minddata/dataset/kernels/tensor_op.h"
|
||||
#include "minddata/dataset/kernels/compose_op.h"
|
||||
#include "minddata/dataset/util/random.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
class RandomChoiceOp : public TensorOp {
|
||||
public:
|
||||
/// constructor
|
||||
/// \param[in] ops list of TensorOps to randomly choose 1 from
|
||||
explicit RandomChoiceOp(const std::vector<std::shared_ptr<TensorOp>> &ops);
|
||||
|
||||
/// default destructor
|
||||
~RandomChoiceOp() = default;
|
||||
|
||||
/// return the number of inputs. All op in ops_ should have the same number of inputs
|
||||
/// \return number of input tensors
|
||||
uint32_t NumInput() override;
|
||||
|
||||
/// return the number of outputs. All op in ops_ should have the same number of outputs
|
||||
/// \return number of input tensors
|
||||
uint32_t NumOutput() override;
|
||||
|
||||
/// return output shape if all ops in ops_ return the same shape, otherwise return unknown shape
|
||||
/// \param[in] inputs
|
||||
/// \param[out] outputs
|
||||
/// \return Status code
|
||||
Status OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) override;
|
||||
|
||||
/// return output type if all ops in ops_ return the same type, otherwise return unknown type
|
||||
/// \param[in] inputs
|
||||
/// \param[out] outputs
|
||||
/// \return Status code
|
||||
Status OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) override;
|
||||
|
||||
/// \param[in] input
|
||||
/// \param[out] output
|
||||
/// \return Status code
|
||||
Status Compute(const TensorRow &input, TensorRow *output) override;
|
||||
|
||||
std::string Name() const override { return kRandomChoiceOp; }
|
||||
|
||||
private:
|
||||
std::vector<std::shared_ptr<TensorOp>> ops_;
|
||||
std::mt19937 gen_; // mersenne_twister_engine
|
||||
std::uniform_int_distribution<size_t> rand_int_;
|
||||
};
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // DATASET_KERNELS_RANDOM_CHOICE_OP_
|
||||
@ -0,0 +1,50 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
import mindspore.common.dtype as mstype
|
||||
import mindspore.dataset as ds
|
||||
import mindspore.dataset.transforms.c_transforms as ops
|
||||
import mindspore.dataset.transforms.py_transforms as py_ops
|
||||
|
||||
|
||||
def test_compose():
|
||||
ds.config.set_seed(0)
|
||||
|
||||
def test_config(arr, op_list):
|
||||
try:
|
||||
data = ds.NumpySlicesDataset(arr, column_names="col", shuffle=False)
|
||||
data = data.map(input_columns=["col"], operations=ops.Compose(op_list))
|
||||
res = []
|
||||
for i in data.create_dict_iterator():
|
||||
res.append(i["col"].tolist())
|
||||
return res
|
||||
except (TypeError, ValueError) as e:
|
||||
return str(e)
|
||||
|
||||
# test simple compose with only 1 op, this would generate a warning
|
||||
assert test_config([[1, 0], [3, 4]], [ops.Fill(2)]) == [[2, 2], [2, 2]]
|
||||
# test 1 column -> 2columns -> 1 -> 2 -> 1
|
||||
assert test_config([[1, 0]], [ops.Duplicate(), ops.Concatenate(), ops.Duplicate(), ops.Concatenate()]) == [
|
||||
[1, 0] * 4]
|
||||
# test one python transform followed by a C transform. type after oneHot is float (mixed use-case)
|
||||
assert test_config([1, 0], [py_ops.OneHotOp(2), ops.TypeCast(mstype.int32)]) == [[[0, 1]], [[1, 0]]]
|
||||
# test exceptions. compose, randomApply randomChoice use the same validator
|
||||
assert "op_list[0] is not a c_transform op" in test_config([1, 0], [1, ops.TypeCast(mstype.int32)])
|
||||
# test empty op list
|
||||
assert "op_list can not be empty." in test_config([1, 0], [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_compose()
|
||||
@ -0,0 +1,48 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
import mindspore.common.dtype as mstype
|
||||
import mindspore.dataset as ds
|
||||
import mindspore.dataset.transforms.c_transforms as ops
|
||||
|
||||
|
||||
def test_random_apply():
|
||||
ds.config.set_seed(0)
|
||||
|
||||
def test_config(arr, op_list, prob=0.5):
|
||||
try:
|
||||
data = ds.NumpySlicesDataset(arr, column_names="col", shuffle=False)
|
||||
data = data.map(input_columns=["col"], operations=ops.RandomApply(op_list, prob))
|
||||
res = []
|
||||
for i in data.create_dict_iterator():
|
||||
res.append(i["col"].tolist())
|
||||
return res
|
||||
except (TypeError, ValueError) as e:
|
||||
return str(e)
|
||||
|
||||
res1 = test_config([[0, 1]], [ops.Duplicate(), ops.Concatenate()])
|
||||
assert res1 in [[[0, 1]], [[0, 1, 0, 1]]]
|
||||
# test single nested compose
|
||||
assert test_config([[0, 1, 2]], [ops.Compose([ops.Duplicate(), ops.Concatenate(), ops.Slice([0, 1, 2])])]) == [
|
||||
[0, 1, 2]]
|
||||
# test exception
|
||||
assert "is not of type (<class 'list'>" in test_config([1, 0], ops.TypeCast(mstype.int32))
|
||||
assert "Input prob is not within the required interval" in test_config([0, 1], [ops.Slice([0, 1])], 1.1)
|
||||
assert "is not of type (<class 'float'>" in test_config([1, 0], [ops.TypeCast(mstype.int32)], None)
|
||||
assert "op_list with value None is not of type (<class 'list'>" in test_config([1, 0], None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_random_apply()
|
||||
@ -0,0 +1,48 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
|
||||
import mindspore.dataset as ds
|
||||
import mindspore.dataset.transforms.c_transforms as ops
|
||||
|
||||
|
||||
def test_random_choice():
|
||||
ds.config.set_seed(0)
|
||||
|
||||
def test_config(arr, op_list):
|
||||
try:
|
||||
data = ds.NumpySlicesDataset(arr, column_names="col", shuffle=False)
|
||||
data = data.map(input_columns=["col"], operations=ops.RandomChoice(op_list))
|
||||
res = []
|
||||
for i in data.create_dict_iterator():
|
||||
res.append(i["col"].tolist())
|
||||
return res
|
||||
except (TypeError, ValueError) as e:
|
||||
return str(e)
|
||||
|
||||
# test whether a op would be randomly chosen. In order to prevent random failure, both results need to be checked
|
||||
res1 = test_config([[0, 1, 2]], [ops.PadEnd([4], 0), ops.Slice([0, 2])])
|
||||
assert res1 in [[[0, 1, 2, 0]], [[0, 2]]]
|
||||
|
||||
# test nested structure
|
||||
res2 = test_config([[0, 1, 2]], [ops.Compose([ops.Duplicate(), ops.Concatenate()]),
|
||||
ops.Compose([ops.Slice([0, 1]), ops.OneHot(2)])])
|
||||
assert res2 in [[[[1, 0], [0, 1]]], [[0, 1, 2, 0, 1, 2]]]
|
||||
# test random_choice where there is only 1 op
|
||||
assert test_config([[4, 3], [2, 1]], [ops.Slice([0])]) == [[4], [2]]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_random_choice()
|
||||
@ -0,0 +1,51 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
import mindspore.dataset as ds
|
||||
import mindspore.dataset.transforms.c_transforms as ops
|
||||
import mindspore.dataset.transforms.vision.c_transforms as visions
|
||||
|
||||
|
||||
def test_random_select_subpolicy():
|
||||
ds.config.set_seed(0)
|
||||
|
||||
def test_config(arr, policy):
|
||||
try:
|
||||
data = ds.NumpySlicesDataset(arr, column_names="col", shuffle=False)
|
||||
data = data.map(input_columns=["col"], operations=visions.RandomSelectSubpolicy(policy))
|
||||
res = []
|
||||
for i in data.create_dict_iterator():
|
||||
res.append(i["col"].tolist())
|
||||
return res
|
||||
except (TypeError, ValueError) as e:
|
||||
return str(e)
|
||||
|
||||
# 3 possible outcomes
|
||||
policy1 = [[(ops.PadEnd([4], 0), 0.5), (ops.Compose([ops.Duplicate(), ops.Concatenate()]), 1)],
|
||||
[(ops.Slice([0, 1]), 0.5), (ops.Duplicate(), 1), (ops.Concatenate(), 1)]]
|
||||
res1 = test_config([[1, 2, 3]], policy1)
|
||||
assert res1 in [[[1, 2, 1, 2]], [[1, 2, 3, 1, 2, 3]], [[1, 2, 3, 0, 1, 2, 3, 0]]]
|
||||
|
||||
# test exceptions
|
||||
assert "policy can not be empty." in test_config([[1, 2, 3]], [])
|
||||
assert "policy[0] can not be empty." in test_config([[1, 2, 3]], [[]])
|
||||
assert "op of (op, prob) in policy[1][0] is not a c_transform op (TensorOp) nor a callable pyfunc" in test_config(
|
||||
[[1, 2, 3]], [[(ops.PadEnd([4], 0), 0.5)], [(1, 0.4)]])
|
||||
assert "prob of (op, prob) policy[1][0] is not within the required interval of (0 to 1)" in test_config([[1]], [
|
||||
[(ops.Duplicate(), 0)], [(ops.Duplicate(), -0.1)]])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_random_select_subpolicy()
|
||||
Loading…
Reference in new issue