fix QueueDataset queue size (#19016)

* fix QueueDataset queue size,set queue size = batch size * 100, to avoid too many instances in channel when training is much slower than reading data.
padding_in_crf
jiaqi 6 years ago committed by GitHub
parent 8f53735437
commit fc038da749
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -106,6 +106,7 @@ void PrivateQueueDataFeed<T>::SetQueueSize(int queue_size) {
PADDLE_ENFORCE(queue_size > 0, "Illegal queue size: %d.", queue_size);
queue_size_ = queue_size;
queue_ = paddle::framework::MakeChannel<T>();
queue_->SetCapacity(queue_size);
}
template <typename T>
@ -301,7 +302,8 @@ void MultiSlotDataFeed::Init(
paddle::framework::MultiSlotDesc multi_slot_desc =
data_feed_desc.multi_slot_desc();
SetBatchSize(data_feed_desc.batch_size());
SetQueueSize(data_feed_desc.batch_size());
// temporarily set queue size = batch size * 100
SetQueueSize(data_feed_desc.batch_size() * 100);
size_t all_slot_num = multi_slot_desc.slots_size();
all_slots_.resize(all_slot_num);
all_slots_type_.resize(all_slot_num);

@ -194,7 +194,8 @@ std::shared_ptr<FILE> shell_popen(const std::string& cmd,
<< ", err_no[" << *err_no << "]";
}
if (wstatus == -1 && errno == ECHILD) {
LOG(WARNING) << "errno is ECHILD";
// temporarily remove this warning
// LOG(WARNING) << "errno is ECHILD";
}
}};
#endif
@ -285,7 +286,8 @@ std::pair<std::shared_ptr<FILE>, std::shared_ptr<FILE>> shell_p2open(
<< "status[" << wstatus << "], cmd[" << cmd << "]";
if (wstatus == -1 && errno == ECHILD) {
LOG(WARNING) << "errno is ECHILD";
// temporarily remove this warning
// LOG(WARNING) << "errno is ECHILD";
}
}};

Loading…
Cancel
Save