Python trainer api (#193)
* Python trainer API and demo * Adding missing PaddleAPIPrivate.h * Adding api_train.sh * More comments * Bump up patch version to 0b3avx_docs
emailweixu
8 years ago
committed by
Yu Yang
parent
46bd5f53e3
commit
cbe734b396
@ -0,0 +1,114 @@
|
|||||||
|
# Copyright (c) 2016 Baidu, Inc. All Rights Reserved
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import itertools
|
||||||
|
import random
|
||||||
|
|
||||||
|
from paddle.trainer.config_parser import parse_config
|
||||||
|
from py_paddle import swig_paddle as api
|
||||||
|
from py_paddle import DataProviderConverter
|
||||||
|
from paddle.trainer.PyDataProvider2 \
|
||||||
|
import integer_value, integer_value_sequence, sparse_binary_vector
|
||||||
|
|
||||||
|
def parse_arguments():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--train_data",
|
||||||
|
type=str, required=False, help="train data file")
|
||||||
|
parser.add_argument("--test_data", type=str, help="test data file")
|
||||||
|
parser.add_argument("--config",
|
||||||
|
type=str, required=True, help="config file name")
|
||||||
|
parser.add_argument("--dict_file", required=True, help="dictionary file")
|
||||||
|
parser.add_argument("--seq",
|
||||||
|
default=1, type=int,
|
||||||
|
help="whether use sequence training")
|
||||||
|
parser.add_argument("--use_gpu", default=0, type=int,
|
||||||
|
help="whether use GPU for training")
|
||||||
|
parser.add_argument("--trainer_count", default=1, type=int,
|
||||||
|
help="Number of threads for training")
|
||||||
|
parser.add_argument("--num_passes", default=5, type=int,
|
||||||
|
help="Number of training passes")
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
UNK_IDX = 0
|
||||||
|
|
||||||
|
def load_data(file_name, word_dict):
|
||||||
|
with open(file_name, 'r') as f:
|
||||||
|
for line in f:
|
||||||
|
label, comment = line.strip().split('\t')
|
||||||
|
words = comment.split()
|
||||||
|
word_slot = [word_dict.get(w, UNK_IDX) for w in words]
|
||||||
|
yield word_slot, int(label)
|
||||||
|
|
||||||
|
def load_dict(dict_file):
|
||||||
|
word_dict = dict()
|
||||||
|
with open(dict_file, 'r') as f:
|
||||||
|
for i, line in enumerate(f):
|
||||||
|
w = line.strip().split()[0]
|
||||||
|
word_dict[w] = i
|
||||||
|
return word_dict
|
||||||
|
|
||||||
|
def main():
|
||||||
|
options = parse_arguments()
|
||||||
|
api.initPaddle("--use_gpu=%s" % options.use_gpu,
|
||||||
|
"--trainer_count=%s" % options.trainer_count)
|
||||||
|
|
||||||
|
word_dict = load_dict(options.dict_file)
|
||||||
|
train_dataset = list(load_data(options.train_data, word_dict))
|
||||||
|
if options.test_data:
|
||||||
|
test_dataset = list(load_data(options.test_data, word_dict))
|
||||||
|
else:
|
||||||
|
test_dataset = None
|
||||||
|
|
||||||
|
trainer_config = parse_config(options.config,
|
||||||
|
"dict_file=%s" % options.dict_file)
|
||||||
|
# No need to have data provider for trainer
|
||||||
|
trainer_config.ClearField('data_config')
|
||||||
|
trainer_config.ClearField('test_data_config')
|
||||||
|
|
||||||
|
# create a GradientMachine from the model configuratin
|
||||||
|
model = api.GradientMachine.createFromConfigProto(
|
||||||
|
trainer_config.model_config)
|
||||||
|
# create a trainer for the gradient machine
|
||||||
|
trainer = api.Trainer.create(trainer_config, model)
|
||||||
|
|
||||||
|
# create a data converter which converts data to PaddlePaddle
|
||||||
|
# internal format
|
||||||
|
input_types = [
|
||||||
|
integer_value_sequence(len(word_dict)) if options.seq
|
||||||
|
else sparse_binary_vector(len(word_dict)),
|
||||||
|
integer_value(2)]
|
||||||
|
converter = DataProviderConverter(input_types)
|
||||||
|
|
||||||
|
batch_size = trainer_config.opt_config.batch_size
|
||||||
|
trainer.startTrain()
|
||||||
|
for train_pass in xrange(options.num_passes):
|
||||||
|
trainer.startTrainPass()
|
||||||
|
random.shuffle(train_dataset)
|
||||||
|
for pos in xrange(0, len(train_dataset), batch_size):
|
||||||
|
batch = itertools.islice(train_dataset, pos, pos + batch_size)
|
||||||
|
size = min(batch_size, len(train_dataset) - pos)
|
||||||
|
trainer.trainOneDataBatch(size, converter(batch))
|
||||||
|
trainer.finishTrainPass()
|
||||||
|
if test_dataset:
|
||||||
|
trainer.startTestPeriod();
|
||||||
|
for pos in xrange(0, len(test_dataset), batch_size):
|
||||||
|
batch = itertools.islice(test_dataset, pos, pos + batch_size)
|
||||||
|
size = min(batch_size, len(test_dataset) - pos)
|
||||||
|
trainer.testOneDataBatch(size, converter(batch))
|
||||||
|
trainer.finishTestPeriod()
|
||||||
|
trainer.finishTrain()
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@ -0,0 +1,29 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Copyright (c) 2016 Baidu, Inc. All Rights Reserved
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Note: if using trainer_config.emb.py, trainer_config.cnn.py
|
||||||
|
# or trainer_config.lstm.py, you need to change --seq to --seq=1
|
||||||
|
# because they are sequence models.
|
||||||
|
python api_train.py \
|
||||||
|
--config=trainer_config.lr.py \
|
||||||
|
--trainer_count=2 \
|
||||||
|
--num_passes=15 \
|
||||||
|
--use_gpu=0 \
|
||||||
|
--seq=0 \
|
||||||
|
--train_data=data/train.txt \
|
||||||
|
--test_data=data/test.txt \
|
||||||
|
--dict_file=data/dict.txt \
|
||||||
|
2>&1 | tee 'train.log'
|
||||||
@ -0,0 +1,68 @@
|
|||||||
|
/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#include "paddle/gserver/gradientmachines/GradientMachine.h"
|
||||||
|
#include "paddle/trainer/TrainerConfigHelper.h"
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
struct GradientMachinePrivate {
|
||||||
|
std::shared_ptr<paddle::GradientMachine> machine;
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
|
inline T& cast(void* ptr) {
|
||||||
|
return *(T*)(ptr);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
struct OptimizationConfigPrivate {
|
||||||
|
std::shared_ptr<paddle::TrainerConfigHelper> trainer_config;
|
||||||
|
paddle::OptimizationConfig config;
|
||||||
|
|
||||||
|
const paddle::OptimizationConfig& getConfig() {
|
||||||
|
if (trainer_config != nullptr) {
|
||||||
|
return trainer_config->getOptConfig();
|
||||||
|
} else {
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
struct TrainerConfigPrivate {
|
||||||
|
std::shared_ptr<paddle::TrainerConfigHelper> conf;
|
||||||
|
TrainerConfigPrivate() {}
|
||||||
|
};
|
||||||
|
|
||||||
|
struct ModelConfigPrivate {
|
||||||
|
std::shared_ptr<paddle::TrainerConfigHelper> conf;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct ArgumentsPrivate {
|
||||||
|
std::vector<paddle::Argument> outputs;
|
||||||
|
|
||||||
|
inline paddle::Argument& getArg(size_t idx) throw(RangeError) {
|
||||||
|
if (idx < outputs.size()) {
|
||||||
|
return outputs[idx];
|
||||||
|
} else {
|
||||||
|
RangeError e;
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
|
std::shared_ptr<T>& cast(void* rawPtr) const {
|
||||||
|
return *(std::shared_ptr<T>*)(rawPtr);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
@ -0,0 +1,63 @@
|
|||||||
|
# Copyright (c) 2016 Baidu, Inc. All Rights Reserved
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from paddle.trainer.config_parser import parse_config
|
||||||
|
from paddle.trainer.config_parser import logger
|
||||||
|
from py_paddle import swig_paddle
|
||||||
|
import util
|
||||||
|
|
||||||
|
def main():
|
||||||
|
trainer_config = parse_config(
|
||||||
|
"./testTrainConfig.py", "")
|
||||||
|
model = swig_paddle.GradientMachine.createFromConfigProto(
|
||||||
|
trainer_config.model_config)
|
||||||
|
trainer = swig_paddle.Trainer.create(trainer_config, model)
|
||||||
|
trainer.startTrain()
|
||||||
|
for train_pass in xrange(2):
|
||||||
|
trainer.startTrainPass()
|
||||||
|
num = 0
|
||||||
|
cost = 0
|
||||||
|
while True: # Train one batch
|
||||||
|
batch_size = 1000
|
||||||
|
data, atEnd = util.loadMNISTTrainData(batch_size)
|
||||||
|
if atEnd:
|
||||||
|
break
|
||||||
|
trainer.trainOneDataBatch(batch_size, data)
|
||||||
|
outs = trainer.getForwardOutput()
|
||||||
|
cost += sum(outs[0]['value'])
|
||||||
|
num += batch_size
|
||||||
|
trainer.finishTrainPass()
|
||||||
|
logger.info('train cost=%f' % (cost / num))
|
||||||
|
|
||||||
|
trainer.startTestPeriod()
|
||||||
|
num = 0
|
||||||
|
cost = 0
|
||||||
|
while True: # Test one batch
|
||||||
|
batch_size = 1000
|
||||||
|
data, atEnd = util.loadMNISTTrainData(batch_size)
|
||||||
|
if atEnd:
|
||||||
|
break
|
||||||
|
trainer.testOneDataBatch(batch_size, data)
|
||||||
|
outs = trainer.getForwardOutput()
|
||||||
|
cost += sum(outs[0]['value'])
|
||||||
|
num += batch_size
|
||||||
|
trainer.finishTestPeriod()
|
||||||
|
logger.info('test cost=%f' % (cost / num))
|
||||||
|
|
||||||
|
trainer.finishTrain()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
swig_paddle.initPaddle("--use_gpu=0", "--trainer_count=1")
|
||||||
|
main()
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue