Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into add_huber_regression_loss_op
test=developfor_weibo
minqiyang
6 years ago
commit
65d355a72c
@ -0,0 +1,63 @@
|
||||
# Tries to find Gperftools.
|
||||
#
|
||||
# Usage of this module as follows:
|
||||
#
|
||||
# find_package(Gperftools)
|
||||
#
|
||||
# Variables used by this module, they can change the default behaviour and need
|
||||
# to be set before calling find_package:
|
||||
#
|
||||
# Gperftools_ROOT_DIR Set this variable to the root installation of
|
||||
# Gperftools if the module has problems finding
|
||||
# the proper installation path.
|
||||
#
|
||||
# Variables defined by this module:
|
||||
#
|
||||
# GPERFTOOLS_FOUND System has Gperftools libs/headers
|
||||
# GPERFTOOLS_LIBRARIES The Gperftools libraries (tcmalloc & profiler)
|
||||
# GPERFTOOLS_INCLUDE_DIR The location of Gperftools headers
|
||||
|
||||
find_library(GPERFTOOLS_TCMALLOC
|
||||
NAMES tcmalloc
|
||||
HINTS ${Gperftools_ROOT_DIR}/lib)
|
||||
|
||||
find_library(GPERFTOOLS_PROFILER
|
||||
NAMES profiler
|
||||
HINTS ${Gperftools_ROOT_DIR}/lib)
|
||||
|
||||
find_library(GPERFTOOLS_TCMALLOC_AND_PROFILER
|
||||
NAMES tcmalloc_and_profiler
|
||||
HINTS ${Gperftools_ROOT_DIR}/lib)
|
||||
|
||||
find_path(GPERFTOOLS_INCLUDE_DIR
|
||||
NAMES gperftools/heap-profiler.h
|
||||
HINTS ${Gperftools_ROOT_DIR}/include)
|
||||
|
||||
set(GPERFTOOLS_LIBRARIES ${GPERFTOOLS_TCMALLOC_AND_PROFILER})
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(
|
||||
Gperftools
|
||||
DEFAULT_MSG
|
||||
GPERFTOOLS_LIBRARIES
|
||||
GPERFTOOLS_INCLUDE_DIR)
|
||||
|
||||
mark_as_advanced(
|
||||
Gperftools_ROOT_DIR
|
||||
GPERFTOOLS_TCMALLOC
|
||||
GPERFTOOLS_PROFILER
|
||||
GPERFTOOLS_TCMALLOC_AND_PROFILER
|
||||
GPERFTOOLS_LIBRARIES
|
||||
GPERFTOOLS_INCLUDE_DIR)
|
||||
|
||||
# create IMPORTED targets
|
||||
if (Gperftools_FOUND AND NOT TARGET gperftools::tcmalloc)
|
||||
add_library(gperftools::tcmalloc UNKNOWN IMPORTED)
|
||||
set_target_properties(gperftools::tcmalloc PROPERTIES
|
||||
IMPORTED_LOCATION ${GPERFTOOLS_TCMALLOC}
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${GPERFTOOLS_INCLUDE_DIR}")
|
||||
add_library(gperftools::profiler UNKNOWN IMPORTED)
|
||||
set_target_properties(gperftools::profiler PROPERTIES
|
||||
IMPORTED_LOCATION ${GPERFTOOLS_PROFILER}
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${GPERFTOOLS_INCLUDE_DIR}")
|
||||
endif()
|
||||
@ -0,0 +1,138 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include "paddle/fluid/framework/async_executor.h"
|
||||
#include "google/protobuf/io/zero_copy_stream_impl.h"
|
||||
#include "google/protobuf/message.h"
|
||||
#include "google/protobuf/text_format.h"
|
||||
|
||||
#include "gflags/gflags.h"
|
||||
#include "paddle/fluid/framework/data_feed_factory.h"
|
||||
#include "paddle/fluid/framework/executor_thread_worker.h"
|
||||
#include "paddle/fluid/framework/feed_fetch_method.h"
|
||||
#include "paddle/fluid/framework/feed_fetch_type.h"
|
||||
#include "paddle/fluid/framework/lod_rank_table.h"
|
||||
#include "paddle/fluid/framework/lod_tensor_array.h"
|
||||
#include "paddle/fluid/framework/op_registry.h"
|
||||
#include "paddle/fluid/framework/reader.h"
|
||||
#include "paddle/fluid/inference/io.h"
|
||||
#include "paddle/fluid/platform/place.h"
|
||||
#include "paddle/fluid/pybind/pybind.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace framework {
|
||||
AsyncExecutor::AsyncExecutor(Scope* scope, const platform::Place& place)
|
||||
: root_scope_(scope), place_(place) {}
|
||||
|
||||
void AsyncExecutor::CreateThreads(
|
||||
ExecutorThreadWorker* worker, const ProgramDesc& main_program,
|
||||
const std::shared_ptr<DataFeed>& reader,
|
||||
const std::vector<std::string>& fetch_var_names, Scope* root_scope,
|
||||
const int thread_index, const bool debug) {
|
||||
worker->SetThreadId(thread_index);
|
||||
worker->SetDebug(debug);
|
||||
worker->SetRootScope(root_scope);
|
||||
worker->CreateThreadResource(main_program, place_);
|
||||
worker->SetDataFeed(reader);
|
||||
worker->SetFetchVarNames(fetch_var_names);
|
||||
worker->BindingDataFeedMemory();
|
||||
}
|
||||
|
||||
void PrepareReaders(std::vector<std::shared_ptr<DataFeed>>& readers, // NOLINT
|
||||
const int thread_num, const DataFeedDesc& data_feed_desc,
|
||||
const std::vector<std::string>& filelist) {
|
||||
readers.resize(thread_num);
|
||||
for (size_t i = 0; i < readers.size(); ++i) {
|
||||
readers[i] = DataFeedFactory::CreateDataFeed(data_feed_desc.name());
|
||||
readers[i]->Init(data_feed_desc); // set batch_size and queue_size here
|
||||
}
|
||||
readers[0]->SetFileList(filelist);
|
||||
}
|
||||
|
||||
void AsyncExecutor::RunFromFile(const ProgramDesc& main_program,
|
||||
const std::string& data_feed_desc_str,
|
||||
const std::vector<std::string>& filelist,
|
||||
const int thread_num,
|
||||
const std::vector<std::string>& fetch_var_names,
|
||||
const bool debug) {
|
||||
std::vector<std::thread> threads;
|
||||
|
||||
auto& block = main_program.Block(0);
|
||||
for (auto var_name : fetch_var_names) {
|
||||
auto var_desc = block.FindVar(var_name);
|
||||
auto shapes = var_desc->GetShape();
|
||||
PADDLE_ENFORCE(shapes[shapes.size() - 1] == 1,
|
||||
"var %s: Fetched var has wrong shape, "
|
||||
"only variables with the last dimension size 1 supported",
|
||||
var_name);
|
||||
}
|
||||
|
||||
DataFeedDesc data_feed_desc;
|
||||
google::protobuf::TextFormat::ParseFromString(data_feed_desc_str,
|
||||
&data_feed_desc);
|
||||
|
||||
int actual_thread_num = thread_num;
|
||||
int file_cnt = filelist.size();
|
||||
PADDLE_ENFORCE(file_cnt > 0, "File list cannot be empty");
|
||||
|
||||
if (actual_thread_num > file_cnt) {
|
||||
VLOG(1) << "Thread num = " << thread_num << ", file num = " << file_cnt
|
||||
<< ". Changing thread_num = " << file_cnt;
|
||||
actual_thread_num = file_cnt;
|
||||
}
|
||||
|
||||
/*
|
||||
readerDesc: protobuf description for reader initlization
|
||||
argument: class_name, batch_size, use_slot, queue_size, buffer_size,
|
||||
padding_index
|
||||
|
||||
reader:
|
||||
1) each thread has a reader, reader will read input data and
|
||||
put it into input queue
|
||||
2) each reader has a Next() iterface, that can fetch an instance
|
||||
from the input queue
|
||||
*/
|
||||
// todo: should be factory method for creating datafeed
|
||||
std::vector<std::shared_ptr<DataFeed>> readers;
|
||||
PrepareReaders(readers, actual_thread_num, data_feed_desc, filelist);
|
||||
|
||||
std::vector<std::shared_ptr<ExecutorThreadWorker>> workers;
|
||||
workers.resize(actual_thread_num);
|
||||
for (auto& worker : workers) {
|
||||
worker.reset(new ExecutorThreadWorker);
|
||||
}
|
||||
|
||||
// prepare thread resource here
|
||||
for (int thidx = 0; thidx < actual_thread_num; ++thidx) {
|
||||
CreateThreads(workers[thidx].get(), main_program, readers[thidx],
|
||||
fetch_var_names, root_scope_, thidx, debug);
|
||||
}
|
||||
|
||||
// start executing ops in multiple threads
|
||||
for (int thidx = 0; thidx < actual_thread_num; ++thidx) {
|
||||
threads.push_back(
|
||||
std::thread(&ExecutorThreadWorker::TrainFiles, workers[thidx].get()));
|
||||
}
|
||||
|
||||
for (auto& th : threads) {
|
||||
th.join();
|
||||
}
|
||||
|
||||
root_scope_->DropKids();
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
} // einit_modelnd namespace framework
|
||||
} // end namespace paddle
|
||||
@ -0,0 +1,58 @@
|
||||
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex> // NOLINT
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <thread> // NOLINT
|
||||
#include <typeinfo>
|
||||
#include <vector>
|
||||
#include "paddle/fluid/framework/data_feed.pb.h"
|
||||
#include "paddle/fluid/framework/executor.h"
|
||||
#include "paddle/fluid/framework/executor_thread_worker.h"
|
||||
#include "paddle/fluid/framework/program_desc.h"
|
||||
#include "paddle/fluid/framework/scope.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace framework {
|
||||
class AsyncExecutor {
|
||||
public:
|
||||
AsyncExecutor(Scope* scope, const platform::Place& place);
|
||||
virtual ~AsyncExecutor() {}
|
||||
void RunFromFile(const ProgramDesc& main_program,
|
||||
const std::string& data_feed_desc_str,
|
||||
const std::vector<std::string>& filelist,
|
||||
const int thread_num,
|
||||
const std::vector<std::string>& fetch_names,
|
||||
const bool debug = false);
|
||||
|
||||
private:
|
||||
void CreateThreads(ExecutorThreadWorker* worker,
|
||||
const ProgramDesc& main_program,
|
||||
const std::shared_ptr<DataFeed>& reader,
|
||||
const std::vector<std::string>& fetch_var_names,
|
||||
Scope* root_scope, const int thread_index,
|
||||
const bool debug);
|
||||
|
||||
public:
|
||||
Scope* root_scope_;
|
||||
platform::Place place_;
|
||||
};
|
||||
|
||||
} // namespace framework
|
||||
} // namespace paddle
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,240 @@
|
||||
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <fstream>
|
||||
#include <memory>
|
||||
#include <mutex> // NOLINT
|
||||
#include <string>
|
||||
#include <thread> // NOLINT
|
||||
#include <vector>
|
||||
|
||||
#include "paddle/fluid/framework/data_feed.pb.h"
|
||||
#include "paddle/fluid/framework/lod_tensor.h"
|
||||
#include "paddle/fluid/framework/reader.h"
|
||||
#include "paddle/fluid/framework/variable.h"
|
||||
#include "paddle/fluid/operators/reader/blocking_queue.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace framework {
|
||||
|
||||
// DataFeed is the base virtual class for all ohther DataFeeds.
|
||||
// It is used to read files and parse the data for subsequent trainer.
|
||||
// Example:
|
||||
// DataFeed* reader =
|
||||
// paddle::framework::DataFeedFactory::CreateDataFeed(data_feed_name);
|
||||
// reader->Init(data_feed_desc); // data_feed_desc is a protobuf object
|
||||
// reader->SetFileList(filelist);
|
||||
// const std::vector<std::string> & use_slot_alias =
|
||||
// reader->GetUseSlotAlias();
|
||||
// for (auto name: use_slot_alias){ // for binding memory
|
||||
// reader->AddFeedVar(scope->Var(name), name);
|
||||
// }
|
||||
// reader->Start();
|
||||
// while (reader->Next()) {
|
||||
// // trainer do something
|
||||
// }
|
||||
class DataFeed {
|
||||
public:
|
||||
DataFeed() {}
|
||||
virtual ~DataFeed() {}
|
||||
virtual void Init(const paddle::framework::DataFeedDesc& data_feed_desc) = 0;
|
||||
virtual bool CheckFile(const char* filename) {
|
||||
PADDLE_THROW("This function(CheckFile) is not implemented.");
|
||||
}
|
||||
// Set filelist for DataFeed.
|
||||
// Pay attention that it must init all readers before call this function.
|
||||
// Otherwise, Init() function will init finish_set_filelist_ flag.
|
||||
virtual bool SetFileList(const std::vector<std::string>& files);
|
||||
virtual bool Start() = 0;
|
||||
// The trainer calls the Next() function, and the DataFeed will load a new
|
||||
// batch to the feed_vec. The return value of this function is the batch
|
||||
// size of the current batch.
|
||||
virtual int Next() = 0;
|
||||
// Get all slots' alias which defined in protofile
|
||||
virtual const std::vector<std::string>& GetAllSlotAlias() {
|
||||
return all_slots_;
|
||||
}
|
||||
// Get used slots' alias which defined in protofile
|
||||
virtual const std::vector<std::string>& GetUseSlotAlias() {
|
||||
return use_slots_;
|
||||
}
|
||||
// This function is used for binding feed_vec memory
|
||||
virtual void AddFeedVar(Variable* var, const std::string& name);
|
||||
|
||||
protected:
|
||||
// The following three functions are used to check if it is executed in this
|
||||
// order:
|
||||
// Init() -> SetFileList() -> Start() -> Next()
|
||||
virtual void CheckInit();
|
||||
virtual void CheckSetFileList();
|
||||
virtual void CheckStart();
|
||||
virtual void SetBatchSize(
|
||||
int batch); // batch size will be set in Init() function
|
||||
// This function is used to pick one file from the global filelist(thread
|
||||
// safe).
|
||||
virtual bool PickOneFile(std::string* filename);
|
||||
|
||||
static std::vector<std::string> filelist_;
|
||||
static size_t file_idx_;
|
||||
static std::mutex mutex_for_pick_file_;
|
||||
|
||||
// the alias of used slots, and its order is determined by
|
||||
// data_feed_desc(proto object)
|
||||
std::vector<std::string> use_slots_;
|
||||
std::vector<bool> use_slots_is_dense_;
|
||||
|
||||
// the alias of all slots, and its order is determined by data_feed_desc(proto
|
||||
// object)
|
||||
std::vector<std::string> all_slots_;
|
||||
std::vector<std::string> all_slots_type_;
|
||||
std::vector<int>
|
||||
use_slots_index_; // -1: not used; >=0: the index of use_slots_
|
||||
|
||||
// The data read by DataFeed will be stored here
|
||||
std::vector<LoDTensor*> feed_vec_;
|
||||
|
||||
// the batch size defined by user
|
||||
int default_batch_size_;
|
||||
// current batch size
|
||||
int batch_size_;
|
||||
|
||||
bool finish_init_;
|
||||
static bool finish_set_filelist_;
|
||||
bool finish_start_;
|
||||
};
|
||||
|
||||
// PrivateQueueDataFeed is the base virtual class for ohther DataFeeds.
|
||||
// It use a read-thread to read file and parse data to a private-queue
|
||||
// (thread level), and get data from this queue when trainer call Next().
|
||||
template <typename T>
|
||||
class PrivateQueueDataFeed : public DataFeed {
|
||||
public:
|
||||
PrivateQueueDataFeed() {}
|
||||
virtual ~PrivateQueueDataFeed() {}
|
||||
virtual void Init(const paddle::framework::DataFeedDesc& data_feed_desc) = 0;
|
||||
virtual bool Start();
|
||||
virtual int Next();
|
||||
|
||||
protected:
|
||||
// The thread implementation function for reading file and parse.
|
||||
virtual void ReadThread();
|
||||
// This function is used to set private-queue size, and the most
|
||||
// efficient when the queue size is close to the batch size.
|
||||
virtual void SetQueueSize(int queue_size);
|
||||
// The reading and parsing method called in the ReadThread.
|
||||
virtual bool ParseOneInstance(T* instance) = 0;
|
||||
// This function is used to put instance to vec_ins
|
||||
virtual void AddInstanceToInsVec(T* vec_ins, const T& instance,
|
||||
int index) = 0;
|
||||
// This function is used to put ins_vec to feed_vec
|
||||
virtual void PutToFeedVec(const T& ins_vec) = 0;
|
||||
|
||||
// The thread for read files
|
||||
std::thread read_thread_;
|
||||
// using ifstream one line and one line parse is faster
|
||||
// than using fread one buffer and one buffer parse.
|
||||
// for a 601M real data:
|
||||
// ifstream one line and one line parse: 6034 ms
|
||||
// fread one buffer and one buffer parse: 7097 ms
|
||||
std::ifstream file_;
|
||||
size_t queue_size_;
|
||||
// The queue for store parsed data
|
||||
std::unique_ptr<paddle::operators::reader::BlockingQueue<T>> queue_;
|
||||
};
|
||||
|
||||
// This class define the data type of instance(ins_vec) in MultiSlotDataFeed
|
||||
class MultiSlotType {
|
||||
public:
|
||||
MultiSlotType() {}
|
||||
~MultiSlotType() {}
|
||||
void Init(const std::string& type) {
|
||||
CheckType(type);
|
||||
if (type_[0] == 'f') {
|
||||
float_feasign_.clear();
|
||||
} else if (type_[0] == 'u') {
|
||||
uint64_feasign_.clear();
|
||||
}
|
||||
type_ = type;
|
||||
}
|
||||
void InitOffset() {
|
||||
offset_.resize(1);
|
||||
// LoDTensor' lod is counted from 0, the size of lod
|
||||
// is one size larger than the size of data.
|
||||
offset_[0] = 0;
|
||||
}
|
||||
const std::vector<size_t>& GetOffset() const { return offset_; }
|
||||
void AddValue(const float v) {
|
||||
CheckFloat();
|
||||
float_feasign_.push_back(v);
|
||||
}
|
||||
void AddValue(const uint64_t v) {
|
||||
CheckUint64();
|
||||
uint64_feasign_.push_back(v);
|
||||
}
|
||||
void AddIns(const MultiSlotType& ins) {
|
||||
if (ins.GetType()[0] == 'f') { // float
|
||||
CheckFloat();
|
||||
auto& vec = ins.GetFloatData();
|
||||
offset_.push_back(offset_.back() + vec.size());
|
||||
float_feasign_.insert(float_feasign_.end(), vec.begin(), vec.end());
|
||||
} else if (ins.GetType()[0] == 'u') { // uint64
|
||||
CheckUint64();
|
||||
auto& vec = ins.GetUint64Data();
|
||||
offset_.push_back(offset_.back() + vec.size());
|
||||
uint64_feasign_.insert(uint64_feasign_.end(), vec.begin(), vec.end());
|
||||
}
|
||||
}
|
||||
const std::vector<float>& GetFloatData() const { return float_feasign_; }
|
||||
const std::vector<uint64_t>& GetUint64Data() const { return uint64_feasign_; }
|
||||
const std::string& GetType() const { return type_; }
|
||||
|
||||
private:
|
||||
void CheckType(const std::string& type) const {
|
||||
PADDLE_ENFORCE((type == "uint64") || (type == "float"),
|
||||
"There is no this type<%s>.", type);
|
||||
}
|
||||
void CheckFloat() const {
|
||||
PADDLE_ENFORCE(type_[0] == 'f', "Add %s value to float slot.", type_);
|
||||
}
|
||||
void CheckUint64() const {
|
||||
PADDLE_ENFORCE(type_[0] == 'u', "Add %s value to uint64 slot.", type_);
|
||||
}
|
||||
std::vector<float> float_feasign_;
|
||||
std::vector<uint64_t> uint64_feasign_;
|
||||
std::string type_;
|
||||
std::vector<size_t> offset_;
|
||||
};
|
||||
|
||||
// This DataFeed is used to feed multi-slot type data.
|
||||
// The format of multi-slot type data:
|
||||
// [n feasign_0 feasign_1 ... feasign_n]*
|
||||
class MultiSlotDataFeed
|
||||
: public PrivateQueueDataFeed<std::vector<MultiSlotType>> {
|
||||
public:
|
||||
MultiSlotDataFeed() {}
|
||||
virtual ~MultiSlotDataFeed() {}
|
||||
virtual void Init(const paddle::framework::DataFeedDesc& data_feed_desc);
|
||||
virtual bool CheckFile(const char* filename);
|
||||
|
||||
protected:
|
||||
virtual void AddInstanceToInsVec(std::vector<MultiSlotType>* vec_ins,
|
||||
const std::vector<MultiSlotType>& instance,
|
||||
int index);
|
||||
virtual bool ParseOneInstance(std::vector<MultiSlotType>* instance);
|
||||
virtual void PutToFeedVec(const std::vector<MultiSlotType>& ins_vec);
|
||||
};
|
||||
} // namespace framework
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,30 @@
|
||||
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
syntax = "proto2";
|
||||
package paddle.framework;
|
||||
|
||||
message Slot {
|
||||
required string name = 1;
|
||||
required string type = 2;
|
||||
optional bool is_dense = 3 [ default = false ];
|
||||
optional bool is_used = 4 [ default = false ];
|
||||
}
|
||||
|
||||
message MultiSlotDesc { repeated Slot slots = 1; }
|
||||
|
||||
message DataFeedDesc {
|
||||
optional string name = 1;
|
||||
optional int32 batch_size = 2 [ default = 32 ];
|
||||
optional MultiSlotDesc multi_slot_desc = 3;
|
||||
}
|
||||
@ -0,0 +1,64 @@
|
||||
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include "paddle/fluid/framework/data_feed_factory.h"
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "paddle/fluid/framework/data_feed.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace framework {
|
||||
typedef std::shared_ptr<DataFeed> (*Createdata_feedFunction)();
|
||||
typedef std::unordered_map<std::string, Createdata_feedFunction> data_feedMap;
|
||||
data_feedMap g_data_feed_map;
|
||||
|
||||
#define REGISTER_DATAFEED_CLASS(data_feed_class) \
|
||||
namespace { \
|
||||
std::shared_ptr<DataFeed> Creator_##data_feed_class() { \
|
||||
return std::shared_ptr<DataFeed>(new data_feed_class); \
|
||||
} \
|
||||
class __Registerer_##data_feed_class { \
|
||||
public: \
|
||||
__Registerer_##data_feed_class() { \
|
||||
g_data_feed_map[#data_feed_class] = &Creator_##data_feed_class; \
|
||||
} \
|
||||
}; \
|
||||
__Registerer_##data_feed_class g_registerer_##data_feed_class; \
|
||||
} // namespace
|
||||
|
||||
std::string DataFeedFactory::DataFeedTypeList() {
|
||||
std::string data_feed_types;
|
||||
for (auto iter = g_data_feed_map.begin(); iter != g_data_feed_map.end();
|
||||
++iter) {
|
||||
if (iter != g_data_feed_map.begin()) {
|
||||
data_feed_types += ", ";
|
||||
}
|
||||
data_feed_types += iter->first;
|
||||
}
|
||||
return data_feed_types;
|
||||
}
|
||||
|
||||
std::shared_ptr<DataFeed> DataFeedFactory::CreateDataFeed(
|
||||
std::string data_feed_class) {
|
||||
if (g_data_feed_map.count(data_feed_class) < 1) {
|
||||
exit(-1);
|
||||
}
|
||||
return g_data_feed_map[data_feed_class]();
|
||||
}
|
||||
|
||||
REGISTER_DATAFEED_CLASS(MultiSlotDataFeed);
|
||||
} // namespace framework
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,29 @@
|
||||
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include "paddle/fluid/framework/data_feed.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace framework {
|
||||
class DataFeedFactory {
|
||||
public:
|
||||
static std::string DataFeedTypeList();
|
||||
static std::shared_ptr<DataFeed> CreateDataFeed(std::string data_feed_class);
|
||||
};
|
||||
} // namespace framework
|
||||
} // namespace paddle
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue