|
|
|
@ -179,12 +179,12 @@ MSRStatus ShardWriter::OpenForAppend(const std::string &path) {
|
|
|
|
return FAILED;
|
|
|
|
return FAILED;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
shard_header_ = std::make_shared<ShardHeader>(sh);
|
|
|
|
shard_header_ = std::make_shared<ShardHeader>(sh);
|
|
|
|
auto paths = shard_header_->get_shard_addresses();
|
|
|
|
auto paths = shard_header_->GetShardAddresses();
|
|
|
|
MSRStatus ret = set_header_size(shard_header_->get_header_size());
|
|
|
|
MSRStatus ret = SetHeaderSize(shard_header_->GetHeaderSize());
|
|
|
|
if (ret == FAILED) {
|
|
|
|
if (ret == FAILED) {
|
|
|
|
return FAILED;
|
|
|
|
return FAILED;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ret = set_page_size(shard_header_->get_page_size());
|
|
|
|
ret = SetPageSize(shard_header_->GetPageSize());
|
|
|
|
if (ret == FAILED) {
|
|
|
|
if (ret == FAILED) {
|
|
|
|
return FAILED;
|
|
|
|
return FAILED;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
@ -229,10 +229,10 @@ MSRStatus ShardWriter::SetShardHeader(std::shared_ptr<ShardHeader> header_data)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// set fields in mindrecord when empty
|
|
|
|
// set fields in mindrecord when empty
|
|
|
|
std::vector<std::pair<uint64_t, std::string>> fields = header_data->get_fields();
|
|
|
|
std::vector<std::pair<uint64_t, std::string>> fields = header_data->GetFields();
|
|
|
|
if (fields.empty()) {
|
|
|
|
if (fields.empty()) {
|
|
|
|
MS_LOG(DEBUG) << "Missing index fields by user, auto generate index fields.";
|
|
|
|
MS_LOG(DEBUG) << "Missing index fields by user, auto generate index fields.";
|
|
|
|
std::vector<std::shared_ptr<Schema>> schemas = header_data->get_schemas();
|
|
|
|
std::vector<std::shared_ptr<Schema>> schemas = header_data->GetSchemas();
|
|
|
|
for (const auto &schema : schemas) {
|
|
|
|
for (const auto &schema : schemas) {
|
|
|
|
json jsonSchema = schema->GetSchema()["schema"];
|
|
|
|
json jsonSchema = schema->GetSchema()["schema"];
|
|
|
|
for (const auto &el : jsonSchema.items()) {
|
|
|
|
for (const auto &el : jsonSchema.items()) {
|
|
|
|
@ -241,7 +241,7 @@ MSRStatus ShardWriter::SetShardHeader(std::shared_ptr<ShardHeader> header_data)
|
|
|
|
(el.value()["type"] == "int64" && el.value().find("shape") == el.value().end()) ||
|
|
|
|
(el.value()["type"] == "int64" && el.value().find("shape") == el.value().end()) ||
|
|
|
|
(el.value()["type"] == "float32" && el.value().find("shape") == el.value().end()) ||
|
|
|
|
(el.value()["type"] == "float32" && el.value().find("shape") == el.value().end()) ||
|
|
|
|
(el.value()["type"] == "float64" && el.value().find("shape") == el.value().end())) {
|
|
|
|
(el.value()["type"] == "float64" && el.value().find("shape") == el.value().end())) {
|
|
|
|
fields.emplace_back(std::make_pair(schema->get_schema_id(), el.key()));
|
|
|
|
fields.emplace_back(std::make_pair(schema->GetSchemaID(), el.key()));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
@ -256,12 +256,12 @@ MSRStatus ShardWriter::SetShardHeader(std::shared_ptr<ShardHeader> header_data)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
shard_header_ = header_data;
|
|
|
|
shard_header_ = header_data;
|
|
|
|
shard_header_->set_header_size(header_size_);
|
|
|
|
shard_header_->SetHeaderSize(header_size_);
|
|
|
|
shard_header_->set_page_size(page_size_);
|
|
|
|
shard_header_->SetPageSize(page_size_);
|
|
|
|
return SUCCESS;
|
|
|
|
return SUCCESS;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
MSRStatus ShardWriter::set_header_size(const uint64_t &header_size) {
|
|
|
|
MSRStatus ShardWriter::SetHeaderSize(const uint64_t &header_size) {
|
|
|
|
// header_size [16KB, 128MB]
|
|
|
|
// header_size [16KB, 128MB]
|
|
|
|
if (header_size < kMinHeaderSize || header_size > kMaxHeaderSize) {
|
|
|
|
if (header_size < kMinHeaderSize || header_size > kMaxHeaderSize) {
|
|
|
|
MS_LOG(ERROR) << "Header size should between 16KB and 128MB.";
|
|
|
|
MS_LOG(ERROR) << "Header size should between 16KB and 128MB.";
|
|
|
|
@ -276,7 +276,7 @@ MSRStatus ShardWriter::set_header_size(const uint64_t &header_size) {
|
|
|
|
return SUCCESS;
|
|
|
|
return SUCCESS;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
MSRStatus ShardWriter::set_page_size(const uint64_t &page_size) {
|
|
|
|
MSRStatus ShardWriter::SetPageSize(const uint64_t &page_size) {
|
|
|
|
// PageSize [32KB, 256MB]
|
|
|
|
// PageSize [32KB, 256MB]
|
|
|
|
if (page_size < kMinPageSize || page_size > kMaxPageSize) {
|
|
|
|
if (page_size < kMinPageSize || page_size > kMaxPageSize) {
|
|
|
|
MS_LOG(ERROR) << "Page size should between 16KB and 256MB.";
|
|
|
|
MS_LOG(ERROR) << "Page size should between 16KB and 256MB.";
|
|
|
|
@ -398,7 +398,7 @@ MSRStatus ShardWriter::CheckData(const std::map<uint64_t, std::vector<json>> &ra
|
|
|
|
return FAILED;
|
|
|
|
return FAILED;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
json schema = result.first->GetSchema()["schema"];
|
|
|
|
json schema = result.first->GetSchema()["schema"];
|
|
|
|
for (const auto &field : result.first->get_blob_fields()) {
|
|
|
|
for (const auto &field : result.first->GetBlobFields()) {
|
|
|
|
(void)schema.erase(field);
|
|
|
|
(void)schema.erase(field);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
std::vector<json> sub_raw_data = rawdata_iter->second;
|
|
|
|
std::vector<json> sub_raw_data = rawdata_iter->second;
|
|
|
|
@ -456,7 +456,7 @@ std::tuple<MSRStatus, int, int> ShardWriter::ValidateRawData(std::map<uint64_t,
|
|
|
|
MS_LOG(DEBUG) << "Schema count is " << schema_count_;
|
|
|
|
MS_LOG(DEBUG) << "Schema count is " << schema_count_;
|
|
|
|
|
|
|
|
|
|
|
|
// Determine if the number of schemas is the same
|
|
|
|
// Determine if the number of schemas is the same
|
|
|
|
if (shard_header_->get_schemas().size() != schema_count_) {
|
|
|
|
if (shard_header_->GetSchemas().size() != schema_count_) {
|
|
|
|
MS_LOG(ERROR) << "Data size is not equal with the schema size";
|
|
|
|
MS_LOG(ERROR) << "Data size is not equal with the schema size";
|
|
|
|
return failed;
|
|
|
|
return failed;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
@ -475,9 +475,9 @@ std::tuple<MSRStatus, int, int> ShardWriter::ValidateRawData(std::map<uint64_t,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
(void)schema_ids.insert(rawdata_iter->first);
|
|
|
|
(void)schema_ids.insert(rawdata_iter->first);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
const std::vector<std::shared_ptr<Schema>> &schemas = shard_header_->get_schemas();
|
|
|
|
const std::vector<std::shared_ptr<Schema>> &schemas = shard_header_->GetSchemas();
|
|
|
|
if (std::any_of(schemas.begin(), schemas.end(), [schema_ids](const std::shared_ptr<Schema> &schema) {
|
|
|
|
if (std::any_of(schemas.begin(), schemas.end(), [schema_ids](const std::shared_ptr<Schema> &schema) {
|
|
|
|
return schema_ids.find(schema->get_schema_id()) == schema_ids.end();
|
|
|
|
return schema_ids.find(schema->GetSchemaID()) == schema_ids.end();
|
|
|
|
})) {
|
|
|
|
})) {
|
|
|
|
// There is not enough data which is not matching the number of schema
|
|
|
|
// There is not enough data which is not matching the number of schema
|
|
|
|
MS_LOG(ERROR) << "Input rawdata schema id do not match real schema id.";
|
|
|
|
MS_LOG(ERROR) << "Input rawdata schema id do not match real schema id.";
|
|
|
|
@ -810,10 +810,10 @@ MSRStatus ShardWriter::CutRowGroup(int start_row, int end_row, const std::vector
|
|
|
|
std::vector<std::pair<int, int>> &rows_in_group,
|
|
|
|
std::vector<std::pair<int, int>> &rows_in_group,
|
|
|
|
const std::shared_ptr<Page> &last_raw_page,
|
|
|
|
const std::shared_ptr<Page> &last_raw_page,
|
|
|
|
const std::shared_ptr<Page> &last_blob_page) {
|
|
|
|
const std::shared_ptr<Page> &last_blob_page) {
|
|
|
|
auto n_byte_blob = last_blob_page ? last_blob_page->get_page_size() : 0;
|
|
|
|
auto n_byte_blob = last_blob_page ? last_blob_page->GetPageSize() : 0;
|
|
|
|
|
|
|
|
|
|
|
|
auto last_raw_page_size = last_raw_page ? last_raw_page->get_page_size() : 0;
|
|
|
|
auto last_raw_page_size = last_raw_page ? last_raw_page->GetPageSize() : 0;
|
|
|
|
auto last_raw_offset = last_raw_page ? last_raw_page->get_last_row_group_id().second : 0;
|
|
|
|
auto last_raw_offset = last_raw_page ? last_raw_page->GetLastRowGroupID().second : 0;
|
|
|
|
auto n_byte_raw = last_raw_page_size - last_raw_offset;
|
|
|
|
auto n_byte_raw = last_raw_page_size - last_raw_offset;
|
|
|
|
|
|
|
|
|
|
|
|
int page_start_row = start_row;
|
|
|
|
int page_start_row = start_row;
|
|
|
|
@ -849,8 +849,8 @@ MSRStatus ShardWriter::AppendBlobPage(const int &shard_id, const std::vector<std
|
|
|
|
if (blob_row.first == blob_row.second) return SUCCESS;
|
|
|
|
if (blob_row.first == blob_row.second) return SUCCESS;
|
|
|
|
|
|
|
|
|
|
|
|
// Write disk
|
|
|
|
// Write disk
|
|
|
|
auto page_id = last_blob_page->get_page_id();
|
|
|
|
auto page_id = last_blob_page->GetPageID();
|
|
|
|
auto bytes_page = last_blob_page->get_page_size();
|
|
|
|
auto bytes_page = last_blob_page->GetPageSize();
|
|
|
|
auto &io_seekp = file_streams_[shard_id]->seekp(page_size_ * page_id + header_size_ + bytes_page, std::ios::beg);
|
|
|
|
auto &io_seekp = file_streams_[shard_id]->seekp(page_size_ * page_id + header_size_ + bytes_page, std::ios::beg);
|
|
|
|
if (!io_seekp.good() || io_seekp.fail() || io_seekp.bad()) {
|
|
|
|
if (!io_seekp.good() || io_seekp.fail() || io_seekp.bad()) {
|
|
|
|
MS_LOG(ERROR) << "File seekp failed";
|
|
|
|
MS_LOG(ERROR) << "File seekp failed";
|
|
|
|
@ -862,9 +862,9 @@ MSRStatus ShardWriter::AppendBlobPage(const int &shard_id, const std::vector<std
|
|
|
|
|
|
|
|
|
|
|
|
// Update last blob page
|
|
|
|
// Update last blob page
|
|
|
|
bytes_page += std::accumulate(blob_data_size_.begin() + blob_row.first, blob_data_size_.begin() + blob_row.second, 0);
|
|
|
|
bytes_page += std::accumulate(blob_data_size_.begin() + blob_row.first, blob_data_size_.begin() + blob_row.second, 0);
|
|
|
|
last_blob_page->set_page_size(bytes_page);
|
|
|
|
last_blob_page->SetPageSize(bytes_page);
|
|
|
|
uint64_t end_row = last_blob_page->get_end_row_id() + blob_row.second - blob_row.first;
|
|
|
|
uint64_t end_row = last_blob_page->GetEndRowID() + blob_row.second - blob_row.first;
|
|
|
|
last_blob_page->set_end_row_id(end_row);
|
|
|
|
last_blob_page->SetEndRowID(end_row);
|
|
|
|
(void)shard_header_->SetPage(last_blob_page);
|
|
|
|
(void)shard_header_->SetPage(last_blob_page);
|
|
|
|
return SUCCESS;
|
|
|
|
return SUCCESS;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
@ -873,8 +873,8 @@ MSRStatus ShardWriter::NewBlobPage(const int &shard_id, const std::vector<std::v
|
|
|
|
const std::vector<std::pair<int, int>> &rows_in_group,
|
|
|
|
const std::vector<std::pair<int, int>> &rows_in_group,
|
|
|
|
const std::shared_ptr<Page> &last_blob_page) {
|
|
|
|
const std::shared_ptr<Page> &last_blob_page) {
|
|
|
|
auto page_id = shard_header_->GetLastPageId(shard_id);
|
|
|
|
auto page_id = shard_header_->GetLastPageId(shard_id);
|
|
|
|
auto page_type_id = last_blob_page ? last_blob_page->get_page_type_id() : -1;
|
|
|
|
auto page_type_id = last_blob_page ? last_blob_page->GetPageTypeID() : -1;
|
|
|
|
auto current_row = last_blob_page ? last_blob_page->get_end_row_id() : 0;
|
|
|
|
auto current_row = last_blob_page ? last_blob_page->GetEndRowID() : 0;
|
|
|
|
// index(0) indicate appendBlobPage
|
|
|
|
// index(0) indicate appendBlobPage
|
|
|
|
for (uint32_t i = 1; i < rows_in_group.size(); ++i) {
|
|
|
|
for (uint32_t i = 1; i < rows_in_group.size(); ++i) {
|
|
|
|
auto blob_row = rows_in_group[i];
|
|
|
|
auto blob_row = rows_in_group[i];
|
|
|
|
@ -905,15 +905,15 @@ MSRStatus ShardWriter::ShiftRawPage(const int &shard_id, const std::vector<std::
|
|
|
|
std::shared_ptr<Page> &last_raw_page) {
|
|
|
|
std::shared_ptr<Page> &last_raw_page) {
|
|
|
|
auto blob_row = rows_in_group[0];
|
|
|
|
auto blob_row = rows_in_group[0];
|
|
|
|
if (blob_row.first == blob_row.second) return SUCCESS;
|
|
|
|
if (blob_row.first == blob_row.second) return SUCCESS;
|
|
|
|
auto last_raw_page_size = last_raw_page ? last_raw_page->get_page_size() : 0;
|
|
|
|
auto last_raw_page_size = last_raw_page ? last_raw_page->GetPageSize() : 0;
|
|
|
|
if (std::accumulate(raw_data_size_.begin() + blob_row.first, raw_data_size_.begin() + blob_row.second, 0) +
|
|
|
|
if (std::accumulate(raw_data_size_.begin() + blob_row.first, raw_data_size_.begin() + blob_row.second, 0) +
|
|
|
|
last_raw_page_size <=
|
|
|
|
last_raw_page_size <=
|
|
|
|
page_size_) {
|
|
|
|
page_size_) {
|
|
|
|
return SUCCESS;
|
|
|
|
return SUCCESS;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
auto page_id = shard_header_->GetLastPageId(shard_id);
|
|
|
|
auto page_id = shard_header_->GetLastPageId(shard_id);
|
|
|
|
auto last_row_group_id_offset = last_raw_page->get_last_row_group_id().second;
|
|
|
|
auto last_row_group_id_offset = last_raw_page->GetLastRowGroupID().second;
|
|
|
|
auto last_raw_page_id = last_raw_page->get_page_id();
|
|
|
|
auto last_raw_page_id = last_raw_page->GetPageID();
|
|
|
|
auto shift_size = last_raw_page_size - last_row_group_id_offset;
|
|
|
|
auto shift_size = last_raw_page_size - last_row_group_id_offset;
|
|
|
|
|
|
|
|
|
|
|
|
std::vector<uint8_t> buf(shift_size);
|
|
|
|
std::vector<uint8_t> buf(shift_size);
|
|
|
|
@ -956,10 +956,10 @@ MSRStatus ShardWriter::ShiftRawPage(const int &shard_id, const std::vector<std::
|
|
|
|
(void)shard_header_->SetPage(last_raw_page);
|
|
|
|
(void)shard_header_->SetPage(last_raw_page);
|
|
|
|
|
|
|
|
|
|
|
|
// Refresh page info in header
|
|
|
|
// Refresh page info in header
|
|
|
|
int row_group_id = last_raw_page->get_last_row_group_id().first + 1;
|
|
|
|
int row_group_id = last_raw_page->GetLastRowGroupID().first + 1;
|
|
|
|
std::vector<std::pair<int, uint64_t>> row_group_ids;
|
|
|
|
std::vector<std::pair<int, uint64_t>> row_group_ids;
|
|
|
|
row_group_ids.emplace_back(row_group_id, 0);
|
|
|
|
row_group_ids.emplace_back(row_group_id, 0);
|
|
|
|
int page_type_id = last_raw_page->get_page_id();
|
|
|
|
int page_type_id = last_raw_page->GetPageID();
|
|
|
|
auto page = Page(++page_id, shard_id, kPageTypeRaw, ++page_type_id, 0, 0, row_group_ids, shift_size);
|
|
|
|
auto page = Page(++page_id, shard_id, kPageTypeRaw, ++page_type_id, 0, 0, row_group_ids, shift_size);
|
|
|
|
(void)shard_header_->AddPage(std::make_shared<Page>(page));
|
|
|
|
(void)shard_header_->AddPage(std::make_shared<Page>(page));
|
|
|
|
|
|
|
|
|
|
|
|
@ -971,7 +971,7 @@ MSRStatus ShardWriter::ShiftRawPage(const int &shard_id, const std::vector<std::
|
|
|
|
MSRStatus ShardWriter::WriteRawPage(const int &shard_id, const std::vector<std::pair<int, int>> &rows_in_group,
|
|
|
|
MSRStatus ShardWriter::WriteRawPage(const int &shard_id, const std::vector<std::pair<int, int>> &rows_in_group,
|
|
|
|
std::shared_ptr<Page> &last_raw_page,
|
|
|
|
std::shared_ptr<Page> &last_raw_page,
|
|
|
|
const std::vector<std::vector<uint8_t>> &bin_raw_data) {
|
|
|
|
const std::vector<std::vector<uint8_t>> &bin_raw_data) {
|
|
|
|
int last_row_group_id = last_raw_page ? last_raw_page->get_last_row_group_id().first : -1;
|
|
|
|
int last_row_group_id = last_raw_page ? last_raw_page->GetLastRowGroupID().first : -1;
|
|
|
|
for (uint32_t i = 0; i < rows_in_group.size(); ++i) {
|
|
|
|
for (uint32_t i = 0; i < rows_in_group.size(); ++i) {
|
|
|
|
const auto &blob_row = rows_in_group[i];
|
|
|
|
const auto &blob_row = rows_in_group[i];
|
|
|
|
if (blob_row.first == blob_row.second) continue;
|
|
|
|
if (blob_row.first == blob_row.second) continue;
|
|
|
|
@ -979,7 +979,7 @@ MSRStatus ShardWriter::WriteRawPage(const int &shard_id, const std::vector<std::
|
|
|
|
std::accumulate(raw_data_size_.begin() + blob_row.first, raw_data_size_.begin() + blob_row.second, 0);
|
|
|
|
std::accumulate(raw_data_size_.begin() + blob_row.first, raw_data_size_.begin() + blob_row.second, 0);
|
|
|
|
if (!last_raw_page) {
|
|
|
|
if (!last_raw_page) {
|
|
|
|
EmptyRawPage(shard_id, last_raw_page);
|
|
|
|
EmptyRawPage(shard_id, last_raw_page);
|
|
|
|
} else if (last_raw_page->get_page_size() + raw_size > page_size_) {
|
|
|
|
} else if (last_raw_page->GetPageSize() + raw_size > page_size_) {
|
|
|
|
(void)shard_header_->SetPage(last_raw_page);
|
|
|
|
(void)shard_header_->SetPage(last_raw_page);
|
|
|
|
EmptyRawPage(shard_id, last_raw_page);
|
|
|
|
EmptyRawPage(shard_id, last_raw_page);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
@ -994,7 +994,7 @@ MSRStatus ShardWriter::WriteRawPage(const int &shard_id, const std::vector<std::
|
|
|
|
void ShardWriter::EmptyRawPage(const int &shard_id, std::shared_ptr<Page> &last_raw_page) {
|
|
|
|
void ShardWriter::EmptyRawPage(const int &shard_id, std::shared_ptr<Page> &last_raw_page) {
|
|
|
|
auto row_group_ids = std::vector<std::pair<int, uint64_t>>();
|
|
|
|
auto row_group_ids = std::vector<std::pair<int, uint64_t>>();
|
|
|
|
auto page_id = shard_header_->GetLastPageId(shard_id);
|
|
|
|
auto page_id = shard_header_->GetLastPageId(shard_id);
|
|
|
|
auto page_type_id = last_raw_page ? last_raw_page->get_page_id() : -1;
|
|
|
|
auto page_type_id = last_raw_page ? last_raw_page->GetPageID() : -1;
|
|
|
|
auto page = Page(++page_id, shard_id, kPageTypeRaw, ++page_type_id, 0, 0, row_group_ids, 0);
|
|
|
|
auto page = Page(++page_id, shard_id, kPageTypeRaw, ++page_type_id, 0, 0, row_group_ids, 0);
|
|
|
|
(void)shard_header_->AddPage(std::make_shared<Page>(page));
|
|
|
|
(void)shard_header_->AddPage(std::make_shared<Page>(page));
|
|
|
|
SetLastRawPage(shard_id, last_raw_page);
|
|
|
|
SetLastRawPage(shard_id, last_raw_page);
|
|
|
|
@ -1003,9 +1003,9 @@ void ShardWriter::EmptyRawPage(const int &shard_id, std::shared_ptr<Page> &last_
|
|
|
|
MSRStatus ShardWriter::AppendRawPage(const int &shard_id, const std::vector<std::pair<int, int>> &rows_in_group,
|
|
|
|
MSRStatus ShardWriter::AppendRawPage(const int &shard_id, const std::vector<std::pair<int, int>> &rows_in_group,
|
|
|
|
const int &chunk_id, int &last_row_group_id, std::shared_ptr<Page> last_raw_page,
|
|
|
|
const int &chunk_id, int &last_row_group_id, std::shared_ptr<Page> last_raw_page,
|
|
|
|
const std::vector<std::vector<uint8_t>> &bin_raw_data) {
|
|
|
|
const std::vector<std::vector<uint8_t>> &bin_raw_data) {
|
|
|
|
std::vector<std::pair<int, uint64_t>> row_group_ids = last_raw_page->get_row_group_ids();
|
|
|
|
std::vector<std::pair<int, uint64_t>> row_group_ids = last_raw_page->GetRowGroupIds();
|
|
|
|
auto last_raw_page_id = last_raw_page->get_page_id();
|
|
|
|
auto last_raw_page_id = last_raw_page->GetPageID();
|
|
|
|
auto n_bytes = last_raw_page->get_page_size();
|
|
|
|
auto n_bytes = last_raw_page->GetPageSize();
|
|
|
|
|
|
|
|
|
|
|
|
// previous raw data page
|
|
|
|
// previous raw data page
|
|
|
|
auto &io_seekp =
|
|
|
|
auto &io_seekp =
|
|
|
|
@ -1022,8 +1022,8 @@ MSRStatus ShardWriter::AppendRawPage(const int &shard_id, const std::vector<std:
|
|
|
|
(void)FlushRawChunk(file_streams_[shard_id], rows_in_group, chunk_id, bin_raw_data);
|
|
|
|
(void)FlushRawChunk(file_streams_[shard_id], rows_in_group, chunk_id, bin_raw_data);
|
|
|
|
|
|
|
|
|
|
|
|
// Update previous raw data page
|
|
|
|
// Update previous raw data page
|
|
|
|
last_raw_page->set_page_size(n_bytes);
|
|
|
|
last_raw_page->SetPageSize(n_bytes);
|
|
|
|
last_raw_page->set_row_group_ids(row_group_ids);
|
|
|
|
last_raw_page->SetRowGroupIds(row_group_ids);
|
|
|
|
(void)shard_header_->SetPage(last_raw_page);
|
|
|
|
(void)shard_header_->SetPage(last_raw_page);
|
|
|
|
|
|
|
|
|
|
|
|
return SUCCESS;
|
|
|
|
return SUCCESS;
|
|
|
|
|
mindspore-ci-bot
5 years ago
committed by
Gitee