You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
240 lines
8.6 KiB
240 lines
8.6 KiB
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from __future__ import print_function
|
|
|
|
import os
|
|
import collections
|
|
from ..framework import Variable, default_main_program
|
|
import pickle
|
|
from . import learning_rate_scheduler
|
|
import warnings
|
|
|
|
__all__ = ['save_persistables', 'load_persistables']
|
|
|
|
|
|
def save_persistables(model_dict, dirname='save_dir', optimizers=None):
|
|
"""
|
|
This function filters out all variables in layer.parameters from the give `layer`, and optimizer's learning rate decay.
|
|
And then trys to save these variables to the folder `dirname`.
|
|
|
|
Use the `dirname` to specify the folder where persistable variables were
|
|
saved.
|
|
|
|
Args:
|
|
model_dict(dict of Parameters): The parameters will
|
|
be saved. If it is None, nothing
|
|
will be deal.
|
|
dirname(str): The directory path.
|
|
optimizers(fluid.Optimizer|list(fluid.Optimizer)|None): The optimizers to be saved
|
|
|
|
Returns:
|
|
None
|
|
|
|
Examples:
|
|
|
|
.. code-block:: python
|
|
|
|
ptb_model = PtbModel(
|
|
hidden_size=hidden_size,
|
|
vocab_size=vocab_size,
|
|
num_layers=num_layers,
|
|
num_steps=num_steps,
|
|
init_scale=init_scale)
|
|
sgd = fluid.optimizer.SGD(learning_rate=0.01)
|
|
x_data = np.arange(12).reshape(4, 3).astype('int64')
|
|
y_data = np.arange(1, 13).reshape(4, 3).astype('int64')
|
|
x_data = x_data.reshape((-1, num_steps, 1))
|
|
y_data = y_data.reshape((-1, 1))
|
|
init_hidden_data = np.zeros(
|
|
(num_layers, batch_size, hidden_size), dtype='float32')
|
|
init_cell_data = np.zeros(
|
|
(num_layers, batch_size, hidden_size), dtype='float32')
|
|
x = to_variable(x_data)
|
|
y = to_variable(y_data)
|
|
init_hidden = to_variable(init_hidden_data)
|
|
init_cell = to_variable(init_cell_data)
|
|
dy_loss, last_hidden, last_cell = ptb_model(x, y, init_hidden,
|
|
init_cell)
|
|
dy_loss.backward()
|
|
sgd.minimize(dy_loss)
|
|
ptb_model.clear_gradient()
|
|
param_path = "./my_paddle_model"
|
|
fluid.dygraph.save_persistables(ptb_model.state_dict(), dirname=param_path, sgd)
|
|
"""
|
|
if isinstance(model_dict, collections.OrderedDict):
|
|
_save_var_to_file(model_dict, optimizers, dirname, None)
|
|
|
|
|
|
def load_persistables(dirname='save_dir'):
|
|
"""
|
|
This function trys to load persistable variables and optimizer's learning rate decay from the folder `dirname`.
|
|
And return the restored values in a dictionary way, respectively.
|
|
|
|
Use the `dirname` to specify the folder where persistable variables were
|
|
saved.
|
|
|
|
Args:
|
|
dirname(str): The directory path. default is save_dir
|
|
|
|
Returns:
|
|
layer_dict: The parameter-dict resumed from file
|
|
optimizer: The optimizer
|
|
|
|
Examples:
|
|
|
|
.. code-block:: python
|
|
|
|
my_layer = layer(fluid.Layer)
|
|
param_path = "./my_paddle_model"
|
|
sgd = SGDOptimizer(learning_rate=1e-3)
|
|
param_dict, optimizer_dict = fluid.dygraph.load_persistables(my_layer.parameters(), param_path)
|
|
param_1 = param_dict['PtbModel_0.w_1']
|
|
sgd.load(optimizer_dict)
|
|
|
|
"""
|
|
return _load_var_from_file(dirname)
|
|
|
|
|
|
def _save_var_to_file(stat_dict, optimizers, file_dir, file_name):
|
|
save_block = default_main_program().global_block()
|
|
save_var_map = {}
|
|
for var_key, each_var in stat_dict.items():
|
|
save_var_map[each_var.name] = each_var
|
|
if file_name is None:
|
|
save_block.append_op(
|
|
type='save',
|
|
inputs={'X': [each_var]},
|
|
outputs={},
|
|
attrs={
|
|
'file_path': os.path.join(file_dir,
|
|
os.path.normpath(each_var.name))
|
|
})
|
|
|
|
if optimizers is not None:
|
|
if isinstance(optimizers, (list, tuple)):
|
|
optimizers = optimizers
|
|
else:
|
|
optimizers = [optimizers]
|
|
if os.path.exists(
|
|
os.path.join(file_dir, os.path.normpath("optimizers"))):
|
|
pass
|
|
else:
|
|
os.mkdir(os.path.join(file_dir, os.path.normpath("optimizers")))
|
|
for optimizer in optimizers:
|
|
if isinstance(optimizer._learning_rate,
|
|
learning_rate_scheduler.LearningRateDecay):
|
|
try:
|
|
f = open(
|
|
os.path.join(file_dir, "optimizers",
|
|
os.path.normpath(str(optimizer._name))),
|
|
"wb")
|
|
pickle.dump(optimizer._learning_rate, f, 2)
|
|
f.close()
|
|
except ():
|
|
raise IOError("Can't load %s",
|
|
os.path.join(
|
|
file_dir, "optimizers",
|
|
os.path.normpath(str(optimizer._name))))
|
|
else:
|
|
warnings.warn(
|
|
"Optimizer not saved, Only optimizer with 'LearningRateDecay' under DyGraph mode need to be saved"
|
|
)
|
|
else:
|
|
pass
|
|
|
|
if file_name is not None:
|
|
save_var_list = []
|
|
for name in sorted(save_var_map.keys()):
|
|
save_var_list.append(save_var_map[name])
|
|
|
|
save_block.append_op(
|
|
type='save_combine',
|
|
inputs={'X': save_var_list},
|
|
outputs={},
|
|
attrs={
|
|
'file_path': os.path.join(file_dir, os.path.normpath(file_name))
|
|
})
|
|
|
|
|
|
def _load_var_from_file(file_dir):
|
|
if not os.path.exists(file_dir):
|
|
raise IOError("{} not exist".format(file_dir))
|
|
|
|
def walk_filename(file_dir):
|
|
base_path = os.path.join(file_dir)
|
|
var_name_list = []
|
|
if os.path.exists(base_path):
|
|
for dirpath, dirnames, filenames in os.walk(base_path):
|
|
if "optimizers" in dirpath:
|
|
continue
|
|
pt = dirpath.replace(base_path, "", 1)
|
|
if pt.startswith("/") or pt.startswith("\\"):
|
|
pt = pt[1:]
|
|
for fth_name in filenames:
|
|
if fth_name[0] != '.':
|
|
name_path = os.path.join(pt, fth_name)
|
|
if "\\" in name_path:
|
|
name_path = name_path.replace("\\", "/")
|
|
var_name_list.append(name_path)
|
|
|
|
return var_name_list
|
|
|
|
load_block = default_main_program().global_block()
|
|
load_var_map = {}
|
|
load_optimizer_map = {}
|
|
file_var_list = walk_filename(file_dir)
|
|
for var_name in file_var_list:
|
|
new_var = Variable(block=load_block, name=var_name)
|
|
load_block.append_op(
|
|
type='load',
|
|
inputs={},
|
|
outputs={'Out': [new_var]},
|
|
attrs={
|
|
'file_path': os.path.join(file_dir,
|
|
os.path.normpath(new_var.name))
|
|
})
|
|
|
|
load_var_map[new_var.name] = new_var
|
|
opt_path = os.path.join(file_dir, "optimizers")
|
|
for _, _, optimizers in os.walk(opt_path):
|
|
for optimizer in optimizers:
|
|
try:
|
|
f = open(os.path.join(opt_path, optimizer), "rb")
|
|
load_optimizer_map[optimizer] = pickle.load(f)
|
|
f.close()
|
|
except IOError:
|
|
raise IOError("Can't load %s",
|
|
os.path.join(
|
|
file_dir, "optimizers",
|
|
os.path.normpath(str(optimizer._name))))
|
|
if len(load_optimizer_map) == 0:
|
|
print(
|
|
"No optimizer loaded. If you didn't save optimizer, please ignore this. The program can still work with new optimizer. "
|
|
)
|
|
pass
|
|
|
|
return load_var_map, load_optimizer_map
|
|
|
|
|
|
def _clone_var_in_block_(block, var):
|
|
assert isinstance(var, Variable)
|
|
return block.create_var(
|
|
name=var.name,
|
|
shape=var.shape,
|
|
dtype=var.dtype,
|
|
type=var.type,
|
|
lod_level=0,
|
|
persistable=True)
|