jiangzhiwen
5 years ago
parent
5d42d00161
commit
2f506b7985
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,122 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "minddata/dataset/core/client.h"
|
||||
#include "common/common.h"
|
||||
#include "common/utils.h"
|
||||
#include "gtest/gtest.h"
|
||||
#include "utils/log_adapter.h"
|
||||
#include "minddata/dataset/engine/datasetops/source/csv_op.h"
|
||||
#include "minddata/dataset/util/status.h"
|
||||
|
||||
|
||||
namespace common = mindspore::common;
|
||||
|
||||
using namespace mindspore::dataset;
|
||||
using mindspore::MsLogLevel::INFO;
|
||||
using mindspore::ExceptionType::NoExceptionType;
|
||||
using mindspore::LogStream;
|
||||
|
||||
class MindDataTestCSVOp : public UT::DatasetOpTesting {
|
||||
|
||||
};
|
||||
|
||||
TEST_F(MindDataTestCSVOp, TestCSVBasic) {
|
||||
// Start with an empty execution tree
|
||||
auto tree = std::make_shared<ExecutionTree>();
|
||||
|
||||
std::string dataset_path;
|
||||
dataset_path = datasets_root_path_ + "/testCSV/1.csv";
|
||||
|
||||
std::vector<std::shared_ptr<CsvOp::BaseRecord>> column_default_list;
|
||||
column_default_list.push_back(std::make_shared<CsvOp::Record<int>>(CsvOp::INT, 0));
|
||||
column_default_list.push_back(std::make_shared<CsvOp::Record<int>>(CsvOp::INT, 0));
|
||||
column_default_list.push_back(std::make_shared<CsvOp::Record<int>>(CsvOp::INT, 0));
|
||||
column_default_list.push_back(std::make_shared<CsvOp::Record<int>>(CsvOp::INT, 0));
|
||||
std::shared_ptr<CsvOp> op;
|
||||
CsvOp::Builder builder;
|
||||
builder.SetCsvFilesList({dataset_path})
|
||||
.SetRowsPerBuffer(16)
|
||||
.SetNumWorkers(16)
|
||||
.SetShuffleFiles(false)
|
||||
.SetOpConnectorSize(2)
|
||||
.SetFieldDelim(',')
|
||||
.SetColumDefault(column_default_list)
|
||||
.SetColumName({"col1", "col2", "col3", "col4"});
|
||||
|
||||
Status rc = builder.Build(&op);
|
||||
ASSERT_TRUE(rc.IsOk());
|
||||
|
||||
rc = tree->AssociateNode(op);
|
||||
ASSERT_TRUE(rc.IsOk());
|
||||
|
||||
rc = tree->AssignRoot(op);
|
||||
ASSERT_TRUE(rc.IsOk());
|
||||
|
||||
MS_LOG(INFO) << "Launching tree and begin iteration.";
|
||||
rc = tree->Prepare();
|
||||
ASSERT_TRUE(rc.IsOk());
|
||||
|
||||
rc = tree->Launch();
|
||||
ASSERT_TRUE(rc.IsOk());
|
||||
|
||||
// Start the loop of reading tensors from our pipeline
|
||||
DatasetIterator di(tree);
|
||||
TensorRow tensor_list;
|
||||
rc = di.FetchNextTensorRow(&tensor_list);
|
||||
ASSERT_TRUE(rc.IsOk());
|
||||
|
||||
int row_count = 0;
|
||||
while (!tensor_list.empty()) {
|
||||
// Display the tensor by calling the printer on it
|
||||
for (int i = 0; i < tensor_list.size(); i++) {
|
||||
std::ostringstream ss;
|
||||
ss << "(" << tensor_list[i] << "): " << *tensor_list[i] << std::endl;
|
||||
MS_LOG(INFO) << "Tensor print: " << ss.str() << ".";
|
||||
}
|
||||
|
||||
rc = di.FetchNextTensorRow(&tensor_list);
|
||||
ASSERT_TRUE(rc.IsOk());
|
||||
row_count++;
|
||||
}
|
||||
|
||||
ASSERT_EQ(row_count, 3);
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestCSVOp, TestTotalRows) {
|
||||
std::string csv_file1 = datasets_root_path_ + "/testCSV/1.csv";
|
||||
std::string csv_file2 = datasets_root_path_ + "/testCSV/size.csv";
|
||||
std::vector<std::string> files;
|
||||
files.push_back(csv_file1);
|
||||
int64_t total_rows = 0;
|
||||
CsvOp::CountAllFileRows(files, false, &total_rows);
|
||||
ASSERT_EQ(total_rows, 3);
|
||||
files.clear();
|
||||
|
||||
files.push_back(csv_file2);
|
||||
CsvOp::CountAllFileRows(files, false, &total_rows);
|
||||
ASSERT_EQ(total_rows, 5);
|
||||
files.clear();
|
||||
|
||||
files.push_back(csv_file1);
|
||||
files.push_back(csv_file2);
|
||||
CsvOp::CountAllFileRows(files, false, &total_rows);
|
||||
ASSERT_EQ(total_rows, 8);
|
||||
files.clear();
|
||||
}
|
||||
|
|
|
|
|
Can't render this file because it contains an unexpected character in line 3 and column 12.
|
|
|
|
@ -0,0 +1,238 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
import mindspore.dataset as ds
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
DATA_FILE = '../data/dataset/testCSV/1.csv'
|
||||
|
||||
|
||||
def test_csv_dataset_basic():
|
||||
"""
|
||||
Test CSV with repeat, skip and so on
|
||||
"""
|
||||
TRAIN_FILE = '../data/dataset/testCSV/1.csv'
|
||||
|
||||
buffer = []
|
||||
data = ds.CSVDataset(
|
||||
TRAIN_FILE,
|
||||
column_defaults=["0", 0, 0.0, "0"],
|
||||
column_names=['1', '2', '3', '4'],
|
||||
shuffle=False)
|
||||
data = data.repeat(2)
|
||||
data = data.skip(2)
|
||||
for d in data.create_dict_iterator():
|
||||
buffer.append(d)
|
||||
assert len(buffer) == 4
|
||||
|
||||
|
||||
def test_csv_dataset_one_file():
|
||||
data = ds.CSVDataset(
|
||||
DATA_FILE,
|
||||
column_defaults=["1", "2", "3", "4"],
|
||||
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||
shuffle=False)
|
||||
buffer = []
|
||||
for d in data.create_dict_iterator():
|
||||
buffer.append(d)
|
||||
assert len(buffer) == 3
|
||||
|
||||
|
||||
def test_csv_dataset_all_file():
|
||||
APPEND_FILE = '../data/dataset/testCSV/2.csv'
|
||||
data = ds.CSVDataset(
|
||||
[DATA_FILE, APPEND_FILE],
|
||||
column_defaults=["1", "2", "3", "4"],
|
||||
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||
shuffle=False)
|
||||
buffer = []
|
||||
for d in data.create_dict_iterator():
|
||||
buffer.append(d)
|
||||
assert len(buffer) == 10
|
||||
|
||||
|
||||
def test_csv_dataset_num_samples():
|
||||
data = ds.CSVDataset(
|
||||
DATA_FILE,
|
||||
column_defaults=["1", "2", "3", "4"],
|
||||
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||
shuffle=False, num_samples=2)
|
||||
count = 0
|
||||
for _ in data.create_dict_iterator():
|
||||
count += 1
|
||||
assert count == 2
|
||||
|
||||
|
||||
def test_csv_dataset_distribution():
|
||||
TEST_FILE = '../data/dataset/testCSV/1.csv'
|
||||
data = ds.CSVDataset(
|
||||
TEST_FILE,
|
||||
column_defaults=["1", "2", "3", "4"],
|
||||
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||
shuffle=False, num_shards=2, shard_id=0)
|
||||
count = 0
|
||||
for _ in data.create_dict_iterator():
|
||||
count += 1
|
||||
assert count == 2
|
||||
|
||||
|
||||
def test_csv_dataset_quoted():
|
||||
TEST_FILE = '../data/dataset/testCSV/quoted.csv'
|
||||
data = ds.CSVDataset(
|
||||
TEST_FILE,
|
||||
column_defaults=["", "", "", ""],
|
||||
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||
shuffle=False)
|
||||
buffer = []
|
||||
for d in data.create_dict_iterator():
|
||||
buffer.extend([d['col1'].item().decode("utf8"),
|
||||
d['col2'].item().decode("utf8"),
|
||||
d['col3'].item().decode("utf8"),
|
||||
d['col4'].item().decode("utf8")])
|
||||
assert buffer == ['a', 'b', 'c', 'd']
|
||||
|
||||
|
||||
def test_csv_dataset_separated():
|
||||
TEST_FILE = '../data/dataset/testCSV/separated.csv'
|
||||
data = ds.CSVDataset(
|
||||
TEST_FILE,
|
||||
field_delim='|',
|
||||
column_defaults=["", "", "", ""],
|
||||
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||
shuffle=False)
|
||||
buffer = []
|
||||
for d in data.create_dict_iterator():
|
||||
buffer.extend([d['col1'].item().decode("utf8"),
|
||||
d['col2'].item().decode("utf8"),
|
||||
d['col3'].item().decode("utf8"),
|
||||
d['col4'].item().decode("utf8")])
|
||||
assert buffer == ['a', 'b', 'c', 'd']
|
||||
|
||||
|
||||
def test_csv_dataset_embedded():
|
||||
TEST_FILE = '../data/dataset/testCSV/embedded.csv'
|
||||
data = ds.CSVDataset(
|
||||
TEST_FILE,
|
||||
column_defaults=["", "", "", ""],
|
||||
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||
shuffle=False)
|
||||
buffer = []
|
||||
for d in data.create_dict_iterator():
|
||||
buffer.extend([d['col1'].item().decode("utf8"),
|
||||
d['col2'].item().decode("utf8"),
|
||||
d['col3'].item().decode("utf8"),
|
||||
d['col4'].item().decode("utf8")])
|
||||
assert buffer == ['a,b', 'c"d', 'e\nf', ' g ']
|
||||
|
||||
|
||||
def test_csv_dataset_chinese():
|
||||
TEST_FILE = '../data/dataset/testCSV/chinese.csv'
|
||||
data = ds.CSVDataset(
|
||||
TEST_FILE,
|
||||
column_defaults=["", "", "", "", ""],
|
||||
column_names=['col1', 'col2', 'col3', 'col4', 'col5'],
|
||||
shuffle=False)
|
||||
buffer = []
|
||||
for d in data.create_dict_iterator():
|
||||
buffer.extend([d['col1'].item().decode("utf8"),
|
||||
d['col2'].item().decode("utf8"),
|
||||
d['col3'].item().decode("utf8"),
|
||||
d['col4'].item().decode("utf8"),
|
||||
d['col5'].item().decode("utf8")])
|
||||
assert buffer == ['大家', '早上好', '中午好', '下午好', '晚上好']
|
||||
|
||||
|
||||
def test_csv_dataset_header():
|
||||
TEST_FILE = '../data/dataset/testCSV/header.csv'
|
||||
data = ds.CSVDataset(
|
||||
TEST_FILE,
|
||||
column_defaults=["", "", "", ""],
|
||||
shuffle=False)
|
||||
buffer = []
|
||||
for d in data.create_dict_iterator():
|
||||
buffer.extend([d['col1'].item().decode("utf8"),
|
||||
d['col2'].item().decode("utf8"),
|
||||
d['col3'].item().decode("utf8"),
|
||||
d['col4'].item().decode("utf8")])
|
||||
assert buffer == ['a', 'b', 'c', 'd']
|
||||
|
||||
|
||||
def test_csv_dataset_number():
|
||||
TEST_FILE = '../data/dataset/testCSV/number.csv'
|
||||
data = ds.CSVDataset(
|
||||
TEST_FILE,
|
||||
column_defaults=[0.0, 0.0, 0, 0.0],
|
||||
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||
shuffle=False)
|
||||
buffer = []
|
||||
for d in data.create_dict_iterator():
|
||||
buffer.extend([d['col1'].item(),
|
||||
d['col2'].item(),
|
||||
d['col3'].item(),
|
||||
d['col4'].item()])
|
||||
assert np.allclose(buffer, [3.0, 0.3, 4, 55.5])
|
||||
|
||||
|
||||
def test_csv_dataset_size():
|
||||
TEST_FILE = '../data/dataset/testCSV/size.csv'
|
||||
data = ds.CSVDataset(
|
||||
TEST_FILE,
|
||||
column_defaults=[0.0, 0.0, 0, 0.0],
|
||||
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||
shuffle=False)
|
||||
assert data.get_dataset_size() == 5
|
||||
|
||||
|
||||
def test_csv_dataset_exception():
|
||||
TEST_FILE = '../data/dataset/testCSV/exception.csv'
|
||||
data = ds.CSVDataset(
|
||||
TEST_FILE,
|
||||
column_defaults=["", "", "", ""],
|
||||
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||
shuffle=False)
|
||||
with pytest.raises(Exception) as err:
|
||||
for _ in data.create_dict_iterator():
|
||||
pass
|
||||
assert "Failed to parse CSV file" in str(err.value)
|
||||
|
||||
|
||||
def test_csv_dataset_type_error():
|
||||
TEST_FILE = '../data/dataset/testCSV/exception.csv'
|
||||
data = ds.CSVDataset(
|
||||
TEST_FILE,
|
||||
column_defaults=["", 0, "", ""],
|
||||
column_names=['col1', 'col2', 'col3', 'col4'],
|
||||
shuffle=False)
|
||||
with pytest.raises(Exception) as err:
|
||||
for _ in data.create_dict_iterator():
|
||||
pass
|
||||
assert "invalid argument of stoi" in str(err.value)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_csv_dataset_basic()
|
||||
test_csv_dataset_one_file()
|
||||
test_csv_dataset_all_file()
|
||||
test_csv_dataset_num_samples()
|
||||
test_csv_dataset_distribution()
|
||||
test_csv_dataset_quoted()
|
||||
test_csv_dataset_separated()
|
||||
test_csv_dataset_embedded()
|
||||
test_csv_dataset_chinese()
|
||||
test_csv_dataset_header()
|
||||
test_csv_dataset_number()
|
||||
test_csv_dataset_size()
|
||||
test_csv_dataset_exception()
|
||||
test_csv_dataset_type_error()
|
||||
Loading…
Reference in new issue