mindspore-ci-bot
5 years ago
committed by
Gitee
commit
b091f74c23
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,122 @@
|
|||||||
|
/**
|
||||||
|
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
#include <iostream>
|
||||||
|
#include <memory>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "minddata/dataset/core/client.h"
|
||||||
|
#include "common/common.h"
|
||||||
|
#include "common/utils.h"
|
||||||
|
#include "gtest/gtest.h"
|
||||||
|
#include "utils/log_adapter.h"
|
||||||
|
#include "minddata/dataset/engine/datasetops/source/csv_op.h"
|
||||||
|
#include "minddata/dataset/util/status.h"
|
||||||
|
|
||||||
|
|
||||||
|
namespace common = mindspore::common;
|
||||||
|
|
||||||
|
using namespace mindspore::dataset;
|
||||||
|
using mindspore::MsLogLevel::INFO;
|
||||||
|
using mindspore::ExceptionType::NoExceptionType;
|
||||||
|
using mindspore::LogStream;
|
||||||
|
|
||||||
|
class MindDataTestCSVOp : public UT::DatasetOpTesting {
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
TEST_F(MindDataTestCSVOp, TestCSVBasic) {
|
||||||
|
// Start with an empty execution tree
|
||||||
|
auto tree = std::make_shared<ExecutionTree>();
|
||||||
|
|
||||||
|
std::string dataset_path;
|
||||||
|
dataset_path = datasets_root_path_ + "/testCSV/1.csv";
|
||||||
|
|
||||||
|
std::vector<std::shared_ptr<CsvOp::BaseRecord>> column_default_list;
|
||||||
|
column_default_list.push_back(std::make_shared<CsvOp::Record<int>>(CsvOp::INT, 0));
|
||||||
|
column_default_list.push_back(std::make_shared<CsvOp::Record<int>>(CsvOp::INT, 0));
|
||||||
|
column_default_list.push_back(std::make_shared<CsvOp::Record<int>>(CsvOp::INT, 0));
|
||||||
|
column_default_list.push_back(std::make_shared<CsvOp::Record<int>>(CsvOp::INT, 0));
|
||||||
|
std::shared_ptr<CsvOp> op;
|
||||||
|
CsvOp::Builder builder;
|
||||||
|
builder.SetCsvFilesList({dataset_path})
|
||||||
|
.SetRowsPerBuffer(16)
|
||||||
|
.SetNumWorkers(16)
|
||||||
|
.SetShuffleFiles(false)
|
||||||
|
.SetOpConnectorSize(2)
|
||||||
|
.SetFieldDelim(',')
|
||||||
|
.SetColumDefault(column_default_list)
|
||||||
|
.SetColumName({"col1", "col2", "col3", "col4"});
|
||||||
|
|
||||||
|
Status rc = builder.Build(&op);
|
||||||
|
ASSERT_TRUE(rc.IsOk());
|
||||||
|
|
||||||
|
rc = tree->AssociateNode(op);
|
||||||
|
ASSERT_TRUE(rc.IsOk());
|
||||||
|
|
||||||
|
rc = tree->AssignRoot(op);
|
||||||
|
ASSERT_TRUE(rc.IsOk());
|
||||||
|
|
||||||
|
MS_LOG(INFO) << "Launching tree and begin iteration.";
|
||||||
|
rc = tree->Prepare();
|
||||||
|
ASSERT_TRUE(rc.IsOk());
|
||||||
|
|
||||||
|
rc = tree->Launch();
|
||||||
|
ASSERT_TRUE(rc.IsOk());
|
||||||
|
|
||||||
|
// Start the loop of reading tensors from our pipeline
|
||||||
|
DatasetIterator di(tree);
|
||||||
|
TensorRow tensor_list;
|
||||||
|
rc = di.FetchNextTensorRow(&tensor_list);
|
||||||
|
ASSERT_TRUE(rc.IsOk());
|
||||||
|
|
||||||
|
int row_count = 0;
|
||||||
|
while (!tensor_list.empty()) {
|
||||||
|
// Display the tensor by calling the printer on it
|
||||||
|
for (int i = 0; i < tensor_list.size(); i++) {
|
||||||
|
std::ostringstream ss;
|
||||||
|
ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl;
|
||||||
|
MS_LOG(INFO) << "Tensor print: " << ss.str() << ".";
|
||||||
|
}
|
||||||
|
|
||||||
|
rc = di.FetchNextTensorRow(&tensor_list);
|
||||||
|
ASSERT_TRUE(rc.IsOk());
|
||||||
|
row_count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
ASSERT_EQ(row_count, 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_F(MindDataTestCSVOp, TestTotalRows) {
|
||||||
|
std::string csv_file1 = datasets_root_path_ + "/testCSV/1.csv";
|
||||||
|
std::string csv_file2 = datasets_root_path_ + "/testCSV/size.csv";
|
||||||
|
std::vector<std::string> files;
|
||||||
|
files.push_back(csv_file1);
|
||||||
|
int64_t total_rows = 0;
|
||||||
|
CsvOp::CountAllFileRows(files, false, &total_rows);
|
||||||
|
ASSERT_EQ(total_rows, 3);
|
||||||
|
files.clear();
|
||||||
|
|
||||||
|
files.push_back(csv_file2);
|
||||||
|
CsvOp::CountAllFileRows(files, false, &total_rows);
|
||||||
|
ASSERT_EQ(total_rows, 5);
|
||||||
|
files.clear();
|
||||||
|
|
||||||
|
files.push_back(csv_file1);
|
||||||
|
files.push_back(csv_file2);
|
||||||
|
CsvOp::CountAllFileRows(files, false, &total_rows);
|
||||||
|
ASSERT_EQ(total_rows, 8);
|
||||||
|
files.clear();
|
||||||
|
}
|
||||||
|
|
|
|
|
Can't render this file because it contains an unexpected character in line 3 and column 12.
|
|
|
|
@ -0,0 +1,238 @@
|
|||||||
|
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ==============================================================================
|
||||||
|
import mindspore.dataset as ds
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
DATA_FILE = '../data/dataset/testCSV/1.csv'
|
||||||
|
|
||||||
|
|
||||||
|
def test_csv_dataset_basic():
|
||||||
|
"""
|
||||||
|
Test CSV with repeat, skip and so on
|
||||||
|
"""
|
||||||
|
TRAIN_FILE = '../data/dataset/testCSV/1.csv'
|
||||||
|
|
||||||
|
buffer = []
|
||||||
|
data = ds.CSVDataset(
|
||||||
|
TRAIN_FILE,
|
||||||
|
column_defaults=["0", 0, 0.0, "0"],
|
||||||
|
column_names=['1', '2', '3', '4'],
|
||||||
|
shuffle=False)
|
||||||
|
data = data.repeat(2)
|
||||||
|
data = data.skip(2)
|
||||||
|
for d in data.create_dict_iterator():
|
||||||
|
buffer.append(d)
|
||||||
|
assert len(buffer) == 4
|
||||||
|
|
||||||
|
|
||||||
|
def test_csv_dataset_one_file():
|
||||||
|
data = ds.CSVDataset(
|
||||||
|
DATA_FILE,
|
||||||
|
column_defaults=["1", "2", "3", "4"],
|
||||||
|
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||||
|
shuffle=False)
|
||||||
|
buffer = []
|
||||||
|
for d in data.create_dict_iterator():
|
||||||
|
buffer.append(d)
|
||||||
|
assert len(buffer) == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_csv_dataset_all_file():
|
||||||
|
APPEND_FILE = '../data/dataset/testCSV/2.csv'
|
||||||
|
data = ds.CSVDataset(
|
||||||
|
[DATA_FILE, APPEND_FILE],
|
||||||
|
column_defaults=["1", "2", "3", "4"],
|
||||||
|
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||||
|
shuffle=False)
|
||||||
|
buffer = []
|
||||||
|
for d in data.create_dict_iterator():
|
||||||
|
buffer.append(d)
|
||||||
|
assert len(buffer) == 10
|
||||||
|
|
||||||
|
|
||||||
|
def test_csv_dataset_num_samples():
|
||||||
|
data = ds.CSVDataset(
|
||||||
|
DATA_FILE,
|
||||||
|
column_defaults=["1", "2", "3", "4"],
|
||||||
|
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||||
|
shuffle=False, num_samples=2)
|
||||||
|
count = 0
|
||||||
|
for _ in data.create_dict_iterator():
|
||||||
|
count += 1
|
||||||
|
assert count == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_csv_dataset_distribution():
|
||||||
|
TEST_FILE = '../data/dataset/testCSV/1.csv'
|
||||||
|
data = ds.CSVDataset(
|
||||||
|
TEST_FILE,
|
||||||
|
column_defaults=["1", "2", "3", "4"],
|
||||||
|
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||||
|
shuffle=False, num_shards=2, shard_id=0)
|
||||||
|
count = 0
|
||||||
|
for _ in data.create_dict_iterator():
|
||||||
|
count += 1
|
||||||
|
assert count == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_csv_dataset_quoted():
|
||||||
|
TEST_FILE = '../data/dataset/testCSV/quoted.csv'
|
||||||
|
data = ds.CSVDataset(
|
||||||
|
TEST_FILE,
|
||||||
|
column_defaults=["", "", "", ""],
|
||||||
|
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||||
|
shuffle=False)
|
||||||
|
buffer = []
|
||||||
|
for d in data.create_dict_iterator():
|
||||||
|
buffer.extend([d['col1'].item().decode("utf8"),
|
||||||
|
d['col2'].item().decode("utf8"),
|
||||||
|
d['col3'].item().decode("utf8"),
|
||||||
|
d['col4'].item().decode("utf8")])
|
||||||
|
assert buffer == ['a', 'b', 'c', 'd']
|
||||||
|
|
||||||
|
|
||||||
|
def test_csv_dataset_separated():
|
||||||
|
TEST_FILE = '../data/dataset/testCSV/separated.csv'
|
||||||
|
data = ds.CSVDataset(
|
||||||
|
TEST_FILE,
|
||||||
|
field_delim='|',
|
||||||
|
column_defaults=["", "", "", ""],
|
||||||
|
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||||
|
shuffle=False)
|
||||||
|
buffer = []
|
||||||
|
for d in data.create_dict_iterator():
|
||||||
|
buffer.extend([d['col1'].item().decode("utf8"),
|
||||||
|
d['col2'].item().decode("utf8"),
|
||||||
|
d['col3'].item().decode("utf8"),
|
||||||
|
d['col4'].item().decode("utf8")])
|
||||||
|
assert buffer == ['a', 'b', 'c', 'd']
|
||||||
|
|
||||||
|
|
||||||
|
def test_csv_dataset_embedded():
|
||||||
|
TEST_FILE = '../data/dataset/testCSV/embedded.csv'
|
||||||
|
data = ds.CSVDataset(
|
||||||
|
TEST_FILE,
|
||||||
|
column_defaults=["", "", "", ""],
|
||||||
|
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||||
|
shuffle=False)
|
||||||
|
buffer = []
|
||||||
|
for d in data.create_dict_iterator():
|
||||||
|
buffer.extend([d['col1'].item().decode("utf8"),
|
||||||
|
d['col2'].item().decode("utf8"),
|
||||||
|
d['col3'].item().decode("utf8"),
|
||||||
|
d['col4'].item().decode("utf8")])
|
||||||
|
assert buffer == ['a,b', 'c"d', 'e\nf', ' g ']
|
||||||
|
|
||||||
|
|
||||||
|
def test_csv_dataset_chinese():
|
||||||
|
TEST_FILE = '../data/dataset/testCSV/chinese.csv'
|
||||||
|
data = ds.CSVDataset(
|
||||||
|
TEST_FILE,
|
||||||
|
column_defaults=["", "", "", "", ""],
|
||||||
|
column_names=['col1', 'col2', 'col3', 'col4', 'col5'],
|
||||||
|
shuffle=False)
|
||||||
|
buffer = []
|
||||||
|
for d in data.create_dict_iterator():
|
||||||
|
buffer.extend([d['col1'].item().decode("utf8"),
|
||||||
|
d['col2'].item().decode("utf8"),
|
||||||
|
d['col3'].item().decode("utf8"),
|
||||||
|
d['col4'].item().decode("utf8"),
|
||||||
|
d['col5'].item().decode("utf8")])
|
||||||
|
assert buffer == ['大家', '早上好', '中午好', '下午好', '晚上好']
|
||||||
|
|
||||||
|
|
||||||
|
def test_csv_dataset_header():
|
||||||
|
TEST_FILE = '../data/dataset/testCSV/header.csv'
|
||||||
|
data = ds.CSVDataset(
|
||||||
|
TEST_FILE,
|
||||||
|
column_defaults=["", "", "", ""],
|
||||||
|
shuffle=False)
|
||||||
|
buffer = []
|
||||||
|
for d in data.create_dict_iterator():
|
||||||
|
buffer.extend([d['col1'].item().decode("utf8"),
|
||||||
|
d['col2'].item().decode("utf8"),
|
||||||
|
d['col3'].item().decode("utf8"),
|
||||||
|
d['col4'].item().decode("utf8")])
|
||||||
|
assert buffer == ['a', 'b', 'c', 'd']
|
||||||
|
|
||||||
|
|
||||||
|
def test_csv_dataset_number():
|
||||||
|
TEST_FILE = '../data/dataset/testCSV/number.csv'
|
||||||
|
data = ds.CSVDataset(
|
||||||
|
TEST_FILE,
|
||||||
|
column_defaults=[0.0, 0.0, 0, 0.0],
|
||||||
|
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||||
|
shuffle=False)
|
||||||
|
buffer = []
|
||||||
|
for d in data.create_dict_iterator():
|
||||||
|
buffer.extend([d['col1'].item(),
|
||||||
|
d['col2'].item(),
|
||||||
|
d['col3'].item(),
|
||||||
|
d['col4'].item()])
|
||||||
|
assert np.allclose(buffer, [3.0, 0.3, 4, 55.5])
|
||||||
|
|
||||||
|
|
||||||
|
def test_csv_dataset_size():
|
||||||
|
TEST_FILE = '../data/dataset/testCSV/size.csv'
|
||||||
|
data = ds.CSVDataset(
|
||||||
|
TEST_FILE,
|
||||||
|
column_defaults=[0.0, 0.0, 0, 0.0],
|
||||||
|
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||||
|
shuffle=False)
|
||||||
|
assert data.get_dataset_size() == 5
|
||||||
|
|
||||||
|
|
||||||
|
def test_csv_dataset_exception():
|
||||||
|
TEST_FILE = '../data/dataset/testCSV/exception.csv'
|
||||||
|
data = ds.CSVDataset(
|
||||||
|
TEST_FILE,
|
||||||
|
column_defaults=["", "", "", ""],
|
||||||
|
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||||
|
shuffle=False)
|
||||||
|
with pytest.raises(Exception) as err:
|
||||||
|
for _ in data.create_dict_iterator():
|
||||||
|
pass
|
||||||
|
assert "Failed to parse CSV file" in str(err.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_csv_dataset_type_error():
|
||||||
|
TEST_FILE = '../data/dataset/testCSV/exception.csv'
|
||||||
|
data = ds.CSVDataset(
|
||||||
|
TEST_FILE,
|
||||||
|
column_defaults=["", 0, "", ""],
|
||||||
|
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||||
|
shuffle=False)
|
||||||
|
with pytest.raises(Exception) as err:
|
||||||
|
for _ in data.create_dict_iterator():
|
||||||
|
pass
|
||||||
|
assert "invalid argument of stoi" in str(err.value)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
test_csv_dataset_basic()
|
||||||
|
test_csv_dataset_one_file()
|
||||||
|
test_csv_dataset_all_file()
|
||||||
|
test_csv_dataset_num_samples()
|
||||||
|
test_csv_dataset_distribution()
|
||||||
|
test_csv_dataset_quoted()
|
||||||
|
test_csv_dataset_separated()
|
||||||
|
test_csv_dataset_embedded()
|
||||||
|
test_csv_dataset_chinese()
|
||||||
|
test_csv_dataset_header()
|
||||||
|
test_csv_dataset_number()
|
||||||
|
test_csv_dataset_size()
|
||||||
|
test_csv_dataset_exception()
|
||||||
|
test_csv_dataset_type_error()
|
||||||
Loading…
Reference in new issue