fix QueueDataset queue size (#19016)

* fix QueueDataset queue size,set queue size = batch size * 100, to avoid too many instances in channel when training is much slower than reading data.
padding_in_crf
jiaqi 6 years ago committed by GitHub
parent 8f53735437
commit fc038da749
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -106,6 +106,7 @@ void PrivateQueueDataFeed<T>::SetQueueSize(int queue_size) {
PADDLE_ENFORCE(queue_size > 0, "Illegal queue size: %d.", queue_size); PADDLE_ENFORCE(queue_size > 0, "Illegal queue size: %d.", queue_size);
queue_size_ = queue_size; queue_size_ = queue_size;
queue_ = paddle::framework::MakeChannel<T>(); queue_ = paddle::framework::MakeChannel<T>();
queue_->SetCapacity(queue_size);
} }
template <typename T> template <typename T>
@ -301,7 +302,8 @@ void MultiSlotDataFeed::Init(
paddle::framework::MultiSlotDesc multi_slot_desc = paddle::framework::MultiSlotDesc multi_slot_desc =
data_feed_desc.multi_slot_desc(); data_feed_desc.multi_slot_desc();
SetBatchSize(data_feed_desc.batch_size()); SetBatchSize(data_feed_desc.batch_size());
SetQueueSize(data_feed_desc.batch_size()); // temporarily set queue size = batch size * 100
SetQueueSize(data_feed_desc.batch_size() * 100);
size_t all_slot_num = multi_slot_desc.slots_size(); size_t all_slot_num = multi_slot_desc.slots_size();
all_slots_.resize(all_slot_num); all_slots_.resize(all_slot_num);
all_slots_type_.resize(all_slot_num); all_slots_type_.resize(all_slot_num);

@ -194,7 +194,8 @@ std::shared_ptr<FILE> shell_popen(const std::string& cmd,
<< ", err_no[" << *err_no << "]"; << ", err_no[" << *err_no << "]";
} }
if (wstatus == -1 && errno == ECHILD) { if (wstatus == -1 && errno == ECHILD) {
LOG(WARNING) << "errno is ECHILD"; // temporarily remove this warning
// LOG(WARNING) << "errno is ECHILD";
} }
}}; }};
#endif #endif
@ -285,7 +286,8 @@ std::pair<std::shared_ptr<FILE>, std::shared_ptr<FILE>> shell_p2open(
<< "status[" << wstatus << "], cmd[" << cmd << "]"; << "status[" << wstatus << "], cmd[" << cmd << "]";
if (wstatus == -1 && errno == ECHILD) { if (wstatus == -1 && errno == ECHILD) {
LOG(WARNING) << "errno is ECHILD"; // temporarily remove this warning
// LOG(WARNING) << "errno is ECHILD";
} }
}}; }};

Loading…
Cancel
Save