Tao Luo
7 years ago
committed by
GitHub
commit
144854d2e8
@ -1,5 +0,0 @@
|
|||||||
The examples in v1_api_demo are using v1_api currently, and will be upgraded to v2_api later.
|
|
||||||
Thus, v1_api_demo is a temporary directory. We decide not to maintain it and will delete it in future.
|
|
||||||
|
|
||||||
Please go to [PaddlePaddle/book](https://github.com/PaddlePaddle/book) and
|
|
||||||
[PaddlePaddle/models](https://github.com/PaddlePaddle/models) to learn PaddlePaddle.
|
|
||||||
@ -1,11 +0,0 @@
|
|||||||
output/
|
|
||||||
uniform_params/
|
|
||||||
cifar_params/
|
|
||||||
mnist_params/
|
|
||||||
*.png
|
|
||||||
.pydevproject
|
|
||||||
.project
|
|
||||||
*.log
|
|
||||||
*.pyc
|
|
||||||
data/mnist_data/
|
|
||||||
data/cifar-10-batches-py/
|
|
||||||
@ -1,13 +0,0 @@
|
|||||||
# Generative Adversarial Networks (GAN)
|
|
||||||
|
|
||||||
This demo implements GAN training described in the original GAN paper (https://arxiv.org/abs/1406.2661) and DCGAN (https://arxiv.org/abs/1511.06434).
|
|
||||||
|
|
||||||
The general training procedures are implemented in gan_trainer.py. The neural network configurations are specified in gan_conf.py (for synthetic data) and gan_conf_image.py (for image data).
|
|
||||||
|
|
||||||
In order to run the model, first download the corresponding data by running the shell script in ./data.
|
|
||||||
Then you can run the command below. The flag -d specifies the training data (cifar, mnist or uniform) and flag --useGpu specifies whether to use gpu for training (0 is cpu, 1 is gpu).
|
|
||||||
|
|
||||||
$python gan_trainer.py -d cifar --use_gpu 1
|
|
||||||
|
|
||||||
The generated images will be stored in ./cifar_samples/
|
|
||||||
The corresponding models will be stored in ./cifar_params/
|
|
||||||
@ -1,18 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
set -e
|
|
||||||
wget https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz
|
|
||||||
tar zxf cifar-10-python.tar.gz
|
|
||||||
rm cifar-10-python.tar.gz
|
|
||||||
@ -1,17 +0,0 @@
|
|||||||
#!/usr/bin/env sh
|
|
||||||
# This script downloads the mnist data and unzips it.
|
|
||||||
set -e
|
|
||||||
DIR="$( cd "$(dirname "$0")" ; pwd -P )"
|
|
||||||
rm -rf "$DIR/mnist_data"
|
|
||||||
mkdir "$DIR/mnist_data"
|
|
||||||
cd "$DIR/mnist_data"
|
|
||||||
|
|
||||||
echo "Downloading..."
|
|
||||||
|
|
||||||
for fname in train-images-idx3-ubyte train-labels-idx1-ubyte t10k-images-idx3-ubyte t10k-labels-idx1-ubyte
|
|
||||||
do
|
|
||||||
if [ ! -e $fname ]; then
|
|
||||||
wget --no-check-certificate http://yann.lecun.com/exdb/mnist/${fname}.gz
|
|
||||||
gunzip ${fname}.gz
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
@ -1,151 +0,0 @@
|
|||||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
from paddle.trainer_config_helpers import *
|
|
||||||
|
|
||||||
mode = get_config_arg("mode", str, "generator")
|
|
||||||
assert mode in set([
|
|
||||||
"generator", "discriminator", "generator_training", "discriminator_training"
|
|
||||||
])
|
|
||||||
|
|
||||||
is_generator_training = mode == "generator_training"
|
|
||||||
is_discriminator_training = mode == "discriminator_training"
|
|
||||||
is_generator = mode == "generator"
|
|
||||||
is_discriminator = mode == "discriminator"
|
|
||||||
|
|
||||||
# The network structure below follows the ref https://arxiv.org/abs/1406.2661
|
|
||||||
# Here we used two hidden layers and batch_norm
|
|
||||||
|
|
||||||
print('mode=%s' % mode)
|
|
||||||
# the dim of the noise (z) as the input of the generator network
|
|
||||||
noise_dim = 10
|
|
||||||
# the dim of the hidden layer
|
|
||||||
hidden_dim = 10
|
|
||||||
# the dim of the generated sample
|
|
||||||
sample_dim = 2
|
|
||||||
|
|
||||||
settings(
|
|
||||||
batch_size=128,
|
|
||||||
learning_rate=1e-4,
|
|
||||||
learning_method=AdamOptimizer(beta1=0.5))
|
|
||||||
|
|
||||||
|
|
||||||
def discriminator(sample):
|
|
||||||
"""
|
|
||||||
discriminator ouputs the probablity of a sample is from generator
|
|
||||||
or real data.
|
|
||||||
The output has two dimenstional: dimension 0 is the probablity
|
|
||||||
of the sample is from generator and dimension 1 is the probabblity
|
|
||||||
of the sample is from real data.
|
|
||||||
"""
|
|
||||||
param_attr = ParamAttr(is_static=is_generator_training)
|
|
||||||
bias_attr = ParamAttr(
|
|
||||||
is_static=is_generator_training, initial_mean=1.0, initial_std=0)
|
|
||||||
|
|
||||||
hidden = fc_layer(
|
|
||||||
input=sample,
|
|
||||||
name="dis_hidden",
|
|
||||||
size=hidden_dim,
|
|
||||||
bias_attr=bias_attr,
|
|
||||||
param_attr=param_attr,
|
|
||||||
act=ReluActivation())
|
|
||||||
|
|
||||||
hidden2 = fc_layer(
|
|
||||||
input=hidden,
|
|
||||||
name="dis_hidden2",
|
|
||||||
size=hidden_dim,
|
|
||||||
bias_attr=bias_attr,
|
|
||||||
param_attr=param_attr,
|
|
||||||
act=LinearActivation())
|
|
||||||
|
|
||||||
hidden_bn = batch_norm_layer(
|
|
||||||
hidden2,
|
|
||||||
act=ReluActivation(),
|
|
||||||
name="dis_hidden_bn",
|
|
||||||
bias_attr=bias_attr,
|
|
||||||
param_attr=ParamAttr(
|
|
||||||
is_static=is_generator_training, initial_mean=1.0,
|
|
||||||
initial_std=0.02),
|
|
||||||
use_global_stats=False)
|
|
||||||
|
|
||||||
return fc_layer(
|
|
||||||
input=hidden_bn,
|
|
||||||
name="dis_prob",
|
|
||||||
size=2,
|
|
||||||
bias_attr=bias_attr,
|
|
||||||
param_attr=param_attr,
|
|
||||||
act=SoftmaxActivation())
|
|
||||||
|
|
||||||
|
|
||||||
def generator(noise):
|
|
||||||
"""
|
|
||||||
generator generates a sample given noise
|
|
||||||
"""
|
|
||||||
param_attr = ParamAttr(is_static=is_discriminator_training)
|
|
||||||
bias_attr = ParamAttr(
|
|
||||||
is_static=is_discriminator_training, initial_mean=1.0, initial_std=0)
|
|
||||||
|
|
||||||
hidden = fc_layer(
|
|
||||||
input=noise,
|
|
||||||
name="gen_layer_hidden",
|
|
||||||
size=hidden_dim,
|
|
||||||
bias_attr=bias_attr,
|
|
||||||
param_attr=param_attr,
|
|
||||||
act=ReluActivation())
|
|
||||||
|
|
||||||
hidden2 = fc_layer(
|
|
||||||
input=hidden,
|
|
||||||
name="gen_hidden2",
|
|
||||||
size=hidden_dim,
|
|
||||||
bias_attr=bias_attr,
|
|
||||||
param_attr=param_attr,
|
|
||||||
act=LinearActivation())
|
|
||||||
|
|
||||||
hidden_bn = batch_norm_layer(
|
|
||||||
hidden2,
|
|
||||||
act=ReluActivation(),
|
|
||||||
name="gen_layer_hidden_bn",
|
|
||||||
bias_attr=bias_attr,
|
|
||||||
param_attr=ParamAttr(
|
|
||||||
is_static=is_discriminator_training,
|
|
||||||
initial_mean=1.0,
|
|
||||||
initial_std=0.02),
|
|
||||||
use_global_stats=False)
|
|
||||||
|
|
||||||
return fc_layer(
|
|
||||||
input=hidden_bn,
|
|
||||||
name="gen_layer1",
|
|
||||||
size=sample_dim,
|
|
||||||
bias_attr=bias_attr,
|
|
||||||
param_attr=param_attr,
|
|
||||||
act=LinearActivation())
|
|
||||||
|
|
||||||
|
|
||||||
if is_generator_training:
|
|
||||||
noise = data_layer(name="noise", size=noise_dim)
|
|
||||||
sample = generator(noise)
|
|
||||||
|
|
||||||
if is_discriminator_training:
|
|
||||||
sample = data_layer(name="sample", size=sample_dim)
|
|
||||||
|
|
||||||
if is_generator_training or is_discriminator_training:
|
|
||||||
label = data_layer(name="label", size=1)
|
|
||||||
prob = discriminator(sample)
|
|
||||||
cost = cross_entropy(input=prob, label=label)
|
|
||||||
classification_error_evaluator(
|
|
||||||
input=prob, label=label, name=mode + '_error')
|
|
||||||
outputs(cost)
|
|
||||||
|
|
||||||
if is_generator:
|
|
||||||
noise = data_layer(name="noise", size=noise_dim)
|
|
||||||
outputs(generator(noise))
|
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,10 +0,0 @@
|
|||||||
data/raw_data
|
|
||||||
data/*.list
|
|
||||||
mnist_vgg_model
|
|
||||||
plot.png
|
|
||||||
train.log
|
|
||||||
*pyc
|
|
||||||
.ipynb_checkpoints
|
|
||||||
params.pkl
|
|
||||||
params.tar
|
|
||||||
params.tar.gz
|
|
||||||
@ -1,209 +0,0 @@
|
|||||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve.
|
|
||||||
#
|
|
||||||
#Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
#you may not use this file except in compliance with the License.
|
|
||||||
#You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
#Unless required by applicable law or agreed to in writing, software
|
|
||||||
#distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
#See the License for the specific language governing permissions and
|
|
||||||
#limitations under the License.
|
|
||||||
"""
|
|
||||||
A very basic example for how to use current Raw SWIG API to train mnist network.
|
|
||||||
|
|
||||||
Current implementation uses Raw SWIG, which means the API call is directly \
|
|
||||||
passed to C++ side of Paddle.
|
|
||||||
|
|
||||||
The user api could be simpler and carefully designed.
|
|
||||||
"""
|
|
||||||
import random
|
|
||||||
|
|
||||||
import numpy as np
|
|
||||||
import paddle.v2 as paddle_v2
|
|
||||||
import py_paddle.swig_paddle as api
|
|
||||||
from paddle.trainer_config_helpers import *
|
|
||||||
from py_paddle import DataProviderConverter
|
|
||||||
|
|
||||||
from mnist_util import read_from_mnist
|
|
||||||
|
|
||||||
|
|
||||||
def init_parameter(network):
|
|
||||||
assert isinstance(network, api.GradientMachine)
|
|
||||||
for each_param in network.getParameters():
|
|
||||||
assert isinstance(each_param, api.Parameter)
|
|
||||||
array_size = len(each_param)
|
|
||||||
array = np.random.uniform(-1.0, 1.0, array_size).astype('float32')
|
|
||||||
each_param.getBuf(api.PARAMETER_VALUE).copyFromNumpyArray(array)
|
|
||||||
|
|
||||||
|
|
||||||
def generator_to_batch(generator, batch_size):
|
|
||||||
ret_val = list()
|
|
||||||
for each_item in generator:
|
|
||||||
ret_val.append(each_item)
|
|
||||||
if len(ret_val) == batch_size:
|
|
||||||
yield ret_val
|
|
||||||
ret_val = list()
|
|
||||||
if len(ret_val) != 0:
|
|
||||||
yield ret_val
|
|
||||||
|
|
||||||
|
|
||||||
class BatchPool(object):
|
|
||||||
def __init__(self, generator, batch_size):
|
|
||||||
self.data = list(generator)
|
|
||||||
self.batch_size = batch_size
|
|
||||||
|
|
||||||
def __call__(self):
|
|
||||||
random.shuffle(self.data)
|
|
||||||
for offset in xrange(0, len(self.data), self.batch_size):
|
|
||||||
limit = min(offset + self.batch_size, len(self.data))
|
|
||||||
yield self.data[offset:limit]
|
|
||||||
|
|
||||||
|
|
||||||
def input_order_converter(generator):
|
|
||||||
for each_item in generator:
|
|
||||||
yield each_item['pixel'], each_item['label']
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
api.initPaddle("-use_gpu=false", "-trainer_count=4") # use 4 cpu cores
|
|
||||||
|
|
||||||
optimizer = paddle_v2.optimizer.Adam(
|
|
||||||
learning_rate=1e-4,
|
|
||||||
batch_size=1000,
|
|
||||||
model_average=ModelAverage(average_window=0.5),
|
|
||||||
regularization=L2Regularization(rate=0.5))
|
|
||||||
|
|
||||||
# Create Local Updater. Local means not run in cluster.
|
|
||||||
# For a cluster training, here we can change to createRemoteUpdater
|
|
||||||
# in future.
|
|
||||||
updater = optimizer.create_local_updater()
|
|
||||||
assert isinstance(updater, api.ParameterUpdater)
|
|
||||||
|
|
||||||
# define network
|
|
||||||
images = paddle_v2.layer.data(
|
|
||||||
name='pixel', type=paddle_v2.data_type.dense_vector(784))
|
|
||||||
label = paddle_v2.layer.data(
|
|
||||||
name='label', type=paddle_v2.data_type.integer_value(10))
|
|
||||||
hidden1 = paddle_v2.layer.fc(input=images, size=200)
|
|
||||||
hidden2 = paddle_v2.layer.fc(input=hidden1, size=200)
|
|
||||||
inference = paddle_v2.layer.fc(input=hidden2,
|
|
||||||
size=10,
|
|
||||||
act=paddle_v2.activation.Softmax())
|
|
||||||
cost = paddle_v2.layer.classification_cost(input=inference, label=label)
|
|
||||||
|
|
||||||
# Create Simple Gradient Machine.
|
|
||||||
model_config = paddle_v2.layer.parse_network(cost)
|
|
||||||
m = api.GradientMachine.createFromConfigProto(model_config,
|
|
||||||
api.CREATE_MODE_NORMAL,
|
|
||||||
optimizer.enable_types())
|
|
||||||
|
|
||||||
# This type check is not useful. Only enable type hint in IDE.
|
|
||||||
# Such as PyCharm
|
|
||||||
assert isinstance(m, api.GradientMachine)
|
|
||||||
|
|
||||||
# Initialize Parameter by numpy.
|
|
||||||
init_parameter(network=m)
|
|
||||||
|
|
||||||
# Initialize ParameterUpdater.
|
|
||||||
updater.init(m)
|
|
||||||
|
|
||||||
# DataProvider Converter is a utility convert Python Object to Paddle C++
|
|
||||||
# Input. The input format is as same as Paddle's DataProvider.
|
|
||||||
converter = DataProviderConverter(input_types=[images.type, label.type])
|
|
||||||
|
|
||||||
train_file = './data/raw_data/train'
|
|
||||||
test_file = './data/raw_data/t10k'
|
|
||||||
|
|
||||||
# start gradient machine.
|
|
||||||
# the gradient machine must be started before invoke forward/backward.
|
|
||||||
# not just for training, but also for inference.
|
|
||||||
m.start()
|
|
||||||
|
|
||||||
# evaluator can print error rate, etc. It is a C++ class.
|
|
||||||
batch_evaluator = m.makeEvaluator()
|
|
||||||
test_evaluator = m.makeEvaluator()
|
|
||||||
|
|
||||||
# Get Train Data.
|
|
||||||
# TrainData will stored in a data pool. Currently implementation is not care
|
|
||||||
# about memory, speed. Just a very naive implementation.
|
|
||||||
train_data_generator = input_order_converter(read_from_mnist(train_file))
|
|
||||||
train_data = BatchPool(train_data_generator, 512)
|
|
||||||
|
|
||||||
# outArgs is Neural Network forward result. Here is not useful, just passed
|
|
||||||
# to gradient_machine.forward
|
|
||||||
outArgs = api.Arguments.createArguments(0)
|
|
||||||
|
|
||||||
for pass_id in xrange(2): # we train 2 passes.
|
|
||||||
updater.startPass()
|
|
||||||
|
|
||||||
for batch_id, data_batch in enumerate(train_data()):
|
|
||||||
# data_batch is input images.
|
|
||||||
# here, for online learning, we could get data_batch from network.
|
|
||||||
|
|
||||||
# Start update one batch.
|
|
||||||
pass_type = updater.startBatch(len(data_batch))
|
|
||||||
|
|
||||||
# Start BatchEvaluator.
|
|
||||||
# batch_evaluator can be used between start/finish.
|
|
||||||
batch_evaluator.start()
|
|
||||||
|
|
||||||
# forwardBackward is a shortcut for forward and backward.
|
|
||||||
# It is sometimes faster than invoke forward/backward separately,
|
|
||||||
# because in GradientMachine, it may be async.
|
|
||||||
m.forwardBackward(converter(data_batch), outArgs, pass_type)
|
|
||||||
|
|
||||||
for each_param in m.getParameters():
|
|
||||||
updater.update(each_param)
|
|
||||||
|
|
||||||
# Get cost. We use numpy to calculate total cost for this batch.
|
|
||||||
cost_vec = outArgs.getSlotValue(0)
|
|
||||||
cost_vec = cost_vec.copyToNumpyMat()
|
|
||||||
cost = cost_vec.sum() / len(data_batch)
|
|
||||||
|
|
||||||
# Make evaluator works.
|
|
||||||
m.eval(batch_evaluator)
|
|
||||||
|
|
||||||
# Print logs.
|
|
||||||
print 'Pass id', pass_id, 'Batch id', batch_id, 'with cost=', \
|
|
||||||
cost, batch_evaluator
|
|
||||||
|
|
||||||
batch_evaluator.finish()
|
|
||||||
# Finish batch.
|
|
||||||
# * will clear gradient.
|
|
||||||
# * ensure all values should be updated.
|
|
||||||
updater.finishBatch(cost)
|
|
||||||
|
|
||||||
# testing stage. use test data set to test current network.
|
|
||||||
updater.apply()
|
|
||||||
test_evaluator.start()
|
|
||||||
test_data_generator = input_order_converter(read_from_mnist(test_file))
|
|
||||||
for data_batch in generator_to_batch(test_data_generator, 512):
|
|
||||||
# in testing stage, only forward is needed.
|
|
||||||
m.forward(converter(data_batch), outArgs, api.PASS_TEST)
|
|
||||||
m.eval(test_evaluator)
|
|
||||||
|
|
||||||
# print error rate for test data set
|
|
||||||
print 'Pass', pass_id, ' test evaluator: ', test_evaluator
|
|
||||||
test_evaluator.finish()
|
|
||||||
updater.restore()
|
|
||||||
|
|
||||||
updater.catchUpWith()
|
|
||||||
params = m.getParameters()
|
|
||||||
for each_param in params:
|
|
||||||
assert isinstance(each_param, api.Parameter)
|
|
||||||
value = each_param.getBuf(api.PARAMETER_VALUE)
|
|
||||||
value = value.copyToNumpyArray()
|
|
||||||
|
|
||||||
# Here, we could save parameter to every where you want
|
|
||||||
print each_param.getName(), value
|
|
||||||
|
|
||||||
updater.finishPass()
|
|
||||||
|
|
||||||
m.finish()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@ -1,21 +0,0 @@
|
|||||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
o = open("./" + "train.list", "w")
|
|
||||||
o.write("./data/raw_data/train" + "\n")
|
|
||||||
o.close()
|
|
||||||
|
|
||||||
o = open("./" + "test.list", "w")
|
|
||||||
o.write("./data/raw_data/t10k" + "\n")
|
|
||||||
o.close()
|
|
||||||
@ -1,21 +0,0 @@
|
|||||||
#!/usr/bin/env sh
|
|
||||||
# This scripts downloads the mnist data and unzips it.
|
|
||||||
set -e
|
|
||||||
DIR="$( cd "$(dirname "$0")" ; pwd -P )"
|
|
||||||
rm -rf "$DIR/raw_data"
|
|
||||||
mkdir "$DIR/raw_data"
|
|
||||||
cd "$DIR/raw_data"
|
|
||||||
|
|
||||||
echo "Downloading..."
|
|
||||||
|
|
||||||
for fname in train-images-idx3-ubyte train-labels-idx1-ubyte t10k-images-idx3-ubyte t10k-labels-idx1-ubyte
|
|
||||||
do
|
|
||||||
if [ ! -e $fname ]; then
|
|
||||||
wget --no-check-certificate http://yann.lecun.com/exdb/mnist/${fname}.gz
|
|
||||||
gunzip ${fname}.gz
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
cd $DIR
|
|
||||||
rm -f *.list
|
|
||||||
python generate_list.py
|
|
||||||
@ -1,79 +0,0 @@
|
|||||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
from paddle.trainer_config_helpers import *
|
|
||||||
|
|
||||||
is_predict = get_config_arg("is_predict", bool, False)
|
|
||||||
|
|
||||||
####################Data Configuration ##################
|
|
||||||
|
|
||||||
if not is_predict:
|
|
||||||
data_dir = './data/'
|
|
||||||
define_py_data_sources2(
|
|
||||||
train_list=data_dir + 'train.list',
|
|
||||||
test_list=data_dir + 'test.list',
|
|
||||||
module='mnist_provider',
|
|
||||||
obj='process')
|
|
||||||
|
|
||||||
######################Algorithm Configuration #############
|
|
||||||
settings(batch_size=50, learning_rate=0.001, learning_method=AdamOptimizer())
|
|
||||||
|
|
||||||
#######################Network Configuration #############
|
|
||||||
|
|
||||||
data_size = 1 * 28 * 28
|
|
||||||
label_size = 10
|
|
||||||
img = data_layer(name='pixel', size=data_size)
|
|
||||||
|
|
||||||
|
|
||||||
# light cnn
|
|
||||||
# A shallower cnn model: [CNN, BN, ReLU, Max-Pooling] x4 + FC x1
|
|
||||||
# Easier to train for mnist dataset and quite efficient
|
|
||||||
# Final performance is close to deeper ones on tasks such as digital and character classification
|
|
||||||
def light_cnn(input_image, num_channels, num_classes):
|
|
||||||
def __light__(ipt,
|
|
||||||
num_filter=128,
|
|
||||||
times=1,
|
|
||||||
conv_filter_size=3,
|
|
||||||
dropouts=0,
|
|
||||||
num_channels_=None):
|
|
||||||
return img_conv_group(
|
|
||||||
input=ipt,
|
|
||||||
num_channels=num_channels_,
|
|
||||||
pool_size=2,
|
|
||||||
pool_stride=2,
|
|
||||||
conv_padding=0,
|
|
||||||
conv_num_filter=[num_filter] * times,
|
|
||||||
conv_filter_size=conv_filter_size,
|
|
||||||
conv_act=ReluActivation(),
|
|
||||||
conv_with_batchnorm=True,
|
|
||||||
conv_batchnorm_drop_rate=dropouts,
|
|
||||||
pool_type=MaxPooling())
|
|
||||||
|
|
||||||
tmp = __light__(input_image, num_filter=128, num_channels_=num_channels)
|
|
||||||
tmp = __light__(tmp, num_filter=128)
|
|
||||||
tmp = __light__(tmp, num_filter=128)
|
|
||||||
tmp = __light__(tmp, num_filter=128, conv_filter_size=1)
|
|
||||||
|
|
||||||
tmp = fc_layer(input=tmp, size=num_classes, act=SoftmaxActivation())
|
|
||||||
return tmp
|
|
||||||
|
|
||||||
|
|
||||||
predict = light_cnn(input_image=img, num_channels=1, num_classes=label_size)
|
|
||||||
|
|
||||||
if not is_predict:
|
|
||||||
lbl = data_layer(name="label", size=label_size)
|
|
||||||
inputs(img, lbl)
|
|
||||||
outputs(classification_cost(input=predict, label=lbl))
|
|
||||||
else:
|
|
||||||
outputs(predict)
|
|
||||||
@ -1,25 +0,0 @@
|
|||||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve.
|
|
||||||
#
|
|
||||||
#Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
#you may not use this file except in compliance with the License.
|
|
||||||
#You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
#Unless required by applicable law or agreed to in writing, software
|
|
||||||
#distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
#See the License for the specific language governing permissions and
|
|
||||||
#limitations under the License.
|
|
||||||
from paddle.trainer.PyDataProvider2 import *
|
|
||||||
from mnist_util import read_from_mnist
|
|
||||||
|
|
||||||
|
|
||||||
# Define a py data provider
|
|
||||||
@provider(
|
|
||||||
input_types={'pixel': dense_vector(28 * 28),
|
|
||||||
'label': integer_value(10)},
|
|
||||||
cache=CacheType.CACHE_PASS_IN_MEM)
|
|
||||||
def process(settings, filename): # settings is not used currently.
|
|
||||||
for each in read_from_mnist(filename):
|
|
||||||
yield each
|
|
||||||
@ -1,30 +0,0 @@
|
|||||||
import numpy
|
|
||||||
|
|
||||||
__all__ = ['read_from_mnist']
|
|
||||||
|
|
||||||
|
|
||||||
def read_from_mnist(filename):
|
|
||||||
imgf = filename + "-images-idx3-ubyte"
|
|
||||||
labelf = filename + "-labels-idx1-ubyte"
|
|
||||||
f = open(imgf, "rb")
|
|
||||||
l = open(labelf, "rb")
|
|
||||||
|
|
||||||
f.read(16)
|
|
||||||
l.read(8)
|
|
||||||
|
|
||||||
# Define number of samples for train/test
|
|
||||||
if "train" in filename:
|
|
||||||
n = 60000
|
|
||||||
else:
|
|
||||||
n = 10000
|
|
||||||
|
|
||||||
images = numpy.fromfile(
|
|
||||||
f, 'ubyte', count=n * 28 * 28).reshape((n, 28 * 28)).astype('float32')
|
|
||||||
images = images / 255.0 * 2.0 - 1.0
|
|
||||||
labels = numpy.fromfile(l, 'ubyte', count=n).astype("int")
|
|
||||||
|
|
||||||
for i in xrange(n):
|
|
||||||
yield {"pixel": images[i, :], 'label': labels[i]}
|
|
||||||
|
|
||||||
f.close()
|
|
||||||
l.close()
|
|
||||||
@ -1,32 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
set -e
|
|
||||||
config=vgg_16_mnist.py
|
|
||||||
output=./mnist_vgg_model
|
|
||||||
log=train.log
|
|
||||||
|
|
||||||
paddle train \
|
|
||||||
--config=$config \
|
|
||||||
--dot_period=10 \
|
|
||||||
--log_period=100 \
|
|
||||||
--test_all_data_in_one_period=1 \
|
|
||||||
--use_gpu=0 \
|
|
||||||
--trainer_count=1 \
|
|
||||||
--num_passes=100 \
|
|
||||||
--save_dir=$output \
|
|
||||||
2>&1 | tee $log
|
|
||||||
paddle usage -l $log -e $? -n "mnist_train" >/dev/null 2>&1
|
|
||||||
|
|
||||||
python -m paddle.utils.plotcurve -i $log > plot.png
|
|
||||||
@ -1,50 +0,0 @@
|
|||||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
from paddle.trainer_config_helpers import *
|
|
||||||
|
|
||||||
is_predict = get_config_arg("is_predict", bool, False)
|
|
||||||
|
|
||||||
####################Data Configuration ##################
|
|
||||||
|
|
||||||
if not is_predict:
|
|
||||||
data_dir = './data/'
|
|
||||||
define_py_data_sources2(
|
|
||||||
train_list=data_dir + 'train.list',
|
|
||||||
test_list=data_dir + 'test.list',
|
|
||||||
module='mnist_provider',
|
|
||||||
obj='process')
|
|
||||||
|
|
||||||
######################Algorithm Configuration #############
|
|
||||||
settings(
|
|
||||||
batch_size=128,
|
|
||||||
learning_rate=0.1 / 128.0,
|
|
||||||
learning_method=MomentumOptimizer(0.9),
|
|
||||||
regularization=L2Regularization(0.0005 * 128))
|
|
||||||
|
|
||||||
#######################Network Configuration #############
|
|
||||||
|
|
||||||
data_size = 1 * 28 * 28
|
|
||||||
label_size = 10
|
|
||||||
img = data_layer(name='pixel', size=data_size)
|
|
||||||
|
|
||||||
# small_vgg is predined in trainer_config_helpers.network
|
|
||||||
predict = small_vgg(input_image=img, num_channels=1, num_classes=label_size)
|
|
||||||
|
|
||||||
if not is_predict:
|
|
||||||
lbl = data_layer(name="label", size=label_size)
|
|
||||||
inputs(img, lbl)
|
|
||||||
outputs(classification_cost(input=predict, label=lbl))
|
|
||||||
else:
|
|
||||||
outputs(predict)
|
|
||||||
@ -1,2 +0,0 @@
|
|||||||
baidu.dict
|
|
||||||
model_*.emb
|
|
||||||
@ -1,113 +0,0 @@
|
|||||||
#!/bin/env python
|
|
||||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
"""
|
|
||||||
Example:
|
|
||||||
python extract_para.py --preModel PREMODEL --preDict PREDICT \
|
|
||||||
--usrModel USRMODEL --usrDict USRDICT -d DIM
|
|
||||||
|
|
||||||
Options:
|
|
||||||
-h, --help show this help message and exit
|
|
||||||
--preModel PREMODEL the name of pretrained embedding model
|
|
||||||
--preDict PREDICT the name of pretrained dictionary
|
|
||||||
--usrModel usrModel the name of output usr embedding model
|
|
||||||
--usrDict usrDict the name of user specified dictionary
|
|
||||||
-d DIM dimension of parameter
|
|
||||||
"""
|
|
||||||
from optparse import OptionParser
|
|
||||||
import struct
|
|
||||||
|
|
||||||
|
|
||||||
def get_row_index(preDict, usrDict):
|
|
||||||
"""
|
|
||||||
Get the row positions for all words in user dictionary from pre-trained dictionary.
|
|
||||||
return: a list of row positions
|
|
||||||
Example: preDict='a\nb\nc\n', usrDict='a\nc\n', then return [0,2]
|
|
||||||
"""
|
|
||||||
pos = []
|
|
||||||
index = dict()
|
|
||||||
with open(preDict, "r") as f:
|
|
||||||
for line_index, line in enumerate(f):
|
|
||||||
word = line.strip().split()[0]
|
|
||||||
index[word] = line_index
|
|
||||||
with open(usrDict, "r") as f:
|
|
||||||
for line in f:
|
|
||||||
word = line.strip().split()[0]
|
|
||||||
pos.append(index[word])
|
|
||||||
return pos
|
|
||||||
|
|
||||||
|
|
||||||
def extract_parameters_by_usrDict(preModel, preDict, usrModel, usrDict,
|
|
||||||
paraDim):
|
|
||||||
"""
|
|
||||||
Extract desired parameters from a pretrained embedding model based on user dictionary
|
|
||||||
"""
|
|
||||||
if paraDim not in [32, 64, 128, 256]:
|
|
||||||
raise RuntimeError("We only support 32, 64, 128, 256 dimensions now")
|
|
||||||
|
|
||||||
fi = open(preModel, "rb")
|
|
||||||
fo = open(usrModel, "wb")
|
|
||||||
|
|
||||||
# write filehead
|
|
||||||
rowIndex = get_row_index(preDict, usrDict)
|
|
||||||
newHead = struct.pack("iil", 0, 4, len(rowIndex) * paraDim)
|
|
||||||
fo.write(newHead)
|
|
||||||
bytes = 4 * paraDim
|
|
||||||
for i in range(0, len(rowIndex)):
|
|
||||||
# find the absolute position of input file
|
|
||||||
fi.seek(rowIndex[i] * bytes + 16, 0)
|
|
||||||
fo.write(fi.read(bytes))
|
|
||||||
|
|
||||||
print "extract parameters finish, total", len(rowIndex), "lines"
|
|
||||||
fi.close()
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
"""
|
|
||||||
Main entry for running paraconvert.py
|
|
||||||
"""
|
|
||||||
usage = "usage: \n" \
|
|
||||||
"python %prog --preModel PREMODEL --preDict PREDICT" \
|
|
||||||
" --usrModel USRMODEL --usrDict USRDICT -d DIM"
|
|
||||||
parser = OptionParser(usage)
|
|
||||||
parser.add_option(
|
|
||||||
"--preModel",
|
|
||||||
action="store",
|
|
||||||
dest="preModel",
|
|
||||||
help="the name of pretrained embedding model")
|
|
||||||
parser.add_option(
|
|
||||||
"--preDict",
|
|
||||||
action="store",
|
|
||||||
dest="preDict",
|
|
||||||
help="the name of pretrained dictionary")
|
|
||||||
parser.add_option(
|
|
||||||
"--usrModel",
|
|
||||||
action="store",
|
|
||||||
dest="usrModel",
|
|
||||||
help="the name of output usr embedding model")
|
|
||||||
parser.add_option(
|
|
||||||
"--usrDict",
|
|
||||||
action="store",
|
|
||||||
dest="usrDict",
|
|
||||||
help="the name of user specified dictionary")
|
|
||||||
parser.add_option(
|
|
||||||
"-d", action="store", dest="dim", help="dimension of parameter")
|
|
||||||
(options, args) = parser.parse_args()
|
|
||||||
extract_parameters_by_usrDict(options.preModel, options.preDict,
|
|
||||||
options.usrModel, options.usrDict,
|
|
||||||
int(options.dim))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@ -1,159 +0,0 @@
|
|||||||
#!/bin/env python
|
|
||||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
"""
|
|
||||||
Example:
|
|
||||||
python paraconvert.py --b2t -i INPUT -o OUTPUT -d DIM
|
|
||||||
python paraconvert.py --t2b -i INPUT -o OUTPUT
|
|
||||||
|
|
||||||
Options:
|
|
||||||
-h, --help show this help message and exit
|
|
||||||
--b2t convert parameter file of embedding model from binary to text
|
|
||||||
--t2b convert parameter file of embedding model from text to binary
|
|
||||||
-i INPUT input parameter file name
|
|
||||||
-o OUTPUT output parameter file name
|
|
||||||
-d DIM dimension of parameter
|
|
||||||
"""
|
|
||||||
from optparse import OptionParser
|
|
||||||
import struct
|
|
||||||
|
|
||||||
|
|
||||||
def binary2text(input, output, paraDim):
|
|
||||||
"""
|
|
||||||
Convert a binary parameter file of embedding model to be a text file.
|
|
||||||
input: the name of input binary parameter file, the format is:
|
|
||||||
1) the first 16 bytes is filehead:
|
|
||||||
version(4 bytes): version of paddle, default = 0
|
|
||||||
floatSize(4 bytes): sizeof(float) = 4
|
|
||||||
paraCount(8 bytes): total number of parameter
|
|
||||||
2) the next (paraCount * 4) bytes is parameters, each has 4 bytes
|
|
||||||
output: the name of output text parameter file, for example:
|
|
||||||
0,4,32156096
|
|
||||||
-0.7845433,1.1937413,-0.1704215,...
|
|
||||||
0.0000909,0.0009465,-0.0008813,...
|
|
||||||
...
|
|
||||||
the format is:
|
|
||||||
1) the first line is filehead:
|
|
||||||
version=0, floatSize=4, paraCount=32156096
|
|
||||||
2) other lines print the paramters
|
|
||||||
a) each line prints paraDim paramters splitted by ','
|
|
||||||
b) there is paraCount/paraDim lines (embedding words)
|
|
||||||
paraDim: dimension of parameters
|
|
||||||
"""
|
|
||||||
fi = open(input, "rb")
|
|
||||||
fo = open(output, "w")
|
|
||||||
"""
|
|
||||||
"""
|
|
||||||
version, floatSize, paraCount = struct.unpack("iil", fi.read(16))
|
|
||||||
newHead = ','.join([str(version), str(floatSize), str(paraCount)])
|
|
||||||
print >> fo, newHead
|
|
||||||
|
|
||||||
bytes = 4 * int(paraDim)
|
|
||||||
format = "%df" % int(paraDim)
|
|
||||||
context = fi.read(bytes)
|
|
||||||
line = 0
|
|
||||||
|
|
||||||
while context:
|
|
||||||
numbers = struct.unpack(format, context)
|
|
||||||
lst = []
|
|
||||||
for i in numbers:
|
|
||||||
lst.append('%8.7f' % i)
|
|
||||||
print >> fo, ','.join(lst)
|
|
||||||
context = fi.read(bytes)
|
|
||||||
line += 1
|
|
||||||
fi.close()
|
|
||||||
fo.close()
|
|
||||||
print "binary2text finish, total", line, "lines"
|
|
||||||
|
|
||||||
|
|
||||||
def get_para_count(input):
|
|
||||||
"""
|
|
||||||
Compute the total number of embedding parameters in input text file.
|
|
||||||
input: the name of input text file
|
|
||||||
"""
|
|
||||||
numRows = 1
|
|
||||||
paraDim = 0
|
|
||||||
with open(input) as f:
|
|
||||||
line = f.readline()
|
|
||||||
paraDim = len(line.split(","))
|
|
||||||
for line in f:
|
|
||||||
numRows += 1
|
|
||||||
return numRows * paraDim
|
|
||||||
|
|
||||||
|
|
||||||
def text2binary(input, output, paddle_head=True):
|
|
||||||
"""
|
|
||||||
Convert a text parameter file of embedding model to be a binary file.
|
|
||||||
input: the name of input text parameter file, for example:
|
|
||||||
-0.7845433,1.1937413,-0.1704215,...
|
|
||||||
0.0000909,0.0009465,-0.0008813,...
|
|
||||||
...
|
|
||||||
the format is:
|
|
||||||
1) it doesn't have filehead
|
|
||||||
2) each line stores the same dimension of parameters,
|
|
||||||
the separator is commas ','
|
|
||||||
output: the name of output binary parameter file, the format is:
|
|
||||||
1) the first 16 bytes is filehead:
|
|
||||||
version(4 bytes), floatSize(4 bytes), paraCount(8 bytes)
|
|
||||||
2) the next (paraCount * 4) bytes is parameters, each has 4 bytes
|
|
||||||
"""
|
|
||||||
fi = open(input, "r")
|
|
||||||
fo = open(output, "wb")
|
|
||||||
|
|
||||||
newHead = struct.pack("iil", 0, 4, get_para_count(input))
|
|
||||||
fo.write(newHead)
|
|
||||||
|
|
||||||
count = 0
|
|
||||||
for line in fi:
|
|
||||||
line = line.strip().split(",")
|
|
||||||
for i in range(0, len(line)):
|
|
||||||
binary_data = struct.pack("f", float(line[i]))
|
|
||||||
fo.write(binary_data)
|
|
||||||
count += 1
|
|
||||||
fi.close()
|
|
||||||
fo.close()
|
|
||||||
print "text2binary finish, total", count, "lines"
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
"""
|
|
||||||
Main entry for running paraconvert.py
|
|
||||||
"""
|
|
||||||
usage = "usage: \n" \
|
|
||||||
"python %prog --b2t -i INPUT -o OUTPUT -d DIM \n" \
|
|
||||||
"python %prog --t2b -i INPUT -o OUTPUT"
|
|
||||||
parser = OptionParser(usage)
|
|
||||||
parser.add_option(
|
|
||||||
"--b2t",
|
|
||||||
action="store_true",
|
|
||||||
help="convert parameter file of embedding model from binary to text")
|
|
||||||
parser.add_option(
|
|
||||||
"--t2b",
|
|
||||||
action="store_true",
|
|
||||||
help="convert parameter file of embedding model from text to binary")
|
|
||||||
parser.add_option(
|
|
||||||
"-i", action="store", dest="input", help="input parameter file name")
|
|
||||||
parser.add_option(
|
|
||||||
"-o", action="store", dest="output", help="output parameter file name")
|
|
||||||
parser.add_option(
|
|
||||||
"-d", action="store", dest="dim", help="dimension of parameter")
|
|
||||||
(options, args) = parser.parse_args()
|
|
||||||
if options.b2t:
|
|
||||||
binary2text(options.input, options.output, options.dim)
|
|
||||||
if options.t2b:
|
|
||||||
text2binary(options.input, options.output)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@ -1,32 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
set -e
|
|
||||||
set -x
|
|
||||||
BASE_URL='http://paddlepaddle.cdn.bcebos.com/model_zoo/embedding'
|
|
||||||
|
|
||||||
DOWNLOAD_ITEMS=(baidu.dict model_32.emb model_64.emb model_128.emb model_256.emb)
|
|
||||||
ITEM_MD5=(fa03a12321eaab6c30a8fcc9442eaea3
|
|
||||||
f88c8325ee6da6187f1080e8fe66c1cd
|
|
||||||
927cf70f27f860aff1a5703ebf7f1584
|
|
||||||
a52e43655cd25d279777ed509a1ae27b
|
|
||||||
b92c67fe9ff70fea53596080e351ac80)
|
|
||||||
|
|
||||||
for ((i=0; i<${#ITEM_MD5[@]}; i++))
|
|
||||||
do
|
|
||||||
FILENAME=${DOWNLOAD_ITEMS[${i}]}
|
|
||||||
REAL_MD5=`wget ${BASE_URL}/${FILENAME} -O - | tee ${FILENAME} | md5sum | cut -d ' ' -f 1`
|
|
||||||
EXPECTED_MD5=${ITEM_MD5[${i}]}
|
|
||||||
[ "${EXPECTED_MD5}" = "${REAL_MD5}" ]
|
|
||||||
done
|
|
||||||
@ -1,5 +0,0 @@
|
|||||||
fea_output/
|
|
||||||
features/
|
|
||||||
model.list
|
|
||||||
ResNet_50.dot
|
|
||||||
ResNet_50.png
|
|
||||||
File diff suppressed because it is too large
Load Diff
@ -1 +0,0 @@
|
|||||||
*image_list_provider_copy_1.py
|
|
||||||
@ -1,13 +0,0 @@
|
|||||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue