|
|
|
@ -9,6 +9,7 @@ import (
|
|
|
|
"encoding/json"
|
|
|
|
"encoding/json"
|
|
|
|
"errors"
|
|
|
|
"errors"
|
|
|
|
"fmt"
|
|
|
|
"fmt"
|
|
|
|
|
|
|
|
"io/ioutil"
|
|
|
|
"os"
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
"path/filepath"
|
|
|
|
"strconv"
|
|
|
|
"strconv"
|
|
|
|
@ -21,14 +22,14 @@ import (
|
|
|
|
// ElementType is the type of elements of a Parameter.
|
|
|
|
// ElementType is the type of elements of a Parameter.
|
|
|
|
type ElementType int
|
|
|
|
type ElementType int
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// RPC error message.
|
|
|
|
const (
|
|
|
|
const (
|
|
|
|
// AlreadyInitialized is true if pserver is initialized
|
|
|
|
AlreadyInitialized = "pserver already initialized"
|
|
|
|
AlreadyInitialized = "pserver already initialized"
|
|
|
|
Uninitialized = "pserver not fully initialized"
|
|
|
|
// Uninitialized is true if pserver not fully initialized
|
|
|
|
CheckpointMD5Failed = "checkpoint file MD5 validation failed"
|
|
|
|
Uninitialized = "pserver not fully initialized"
|
|
|
|
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
// Supported element types
|
|
|
|
// Supported element types.
|
|
|
|
const (
|
|
|
|
const (
|
|
|
|
Int32 ElementType = iota
|
|
|
|
Int32 ElementType = iota
|
|
|
|
UInt32
|
|
|
|
UInt32
|
|
|
|
@ -51,21 +52,15 @@ type ParameterWithConfig struct {
|
|
|
|
Config []byte // parameter configuration in Proto Buffer format
|
|
|
|
Config []byte // parameter configuration in Proto Buffer format
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// ParameterCheckpoint is Parameter and State checkpoint
|
|
|
|
// checkpointMeta saves checkpoint metadata
|
|
|
|
type ParameterCheckpoint struct {
|
|
|
|
|
|
|
|
ParamConfig ParameterWithConfig
|
|
|
|
|
|
|
|
State []byte
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// checkpoint signature
|
|
|
|
|
|
|
|
type checkpointMeta struct {
|
|
|
|
type checkpointMeta struct {
|
|
|
|
UUID string `json:"uuid"`
|
|
|
|
UUID string `json:"uuid"`
|
|
|
|
Md5sum string `json:"md5sum"`
|
|
|
|
MD5 string `json:"md5"`
|
|
|
|
Timestamp string `json:"timestamp"`
|
|
|
|
Timestamp int64 `json:"timestamp"`
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Checkpoint is the pserver shard persist in file
|
|
|
|
// Checkpoint is the pserver shard persist in file
|
|
|
|
type Checkpoint []ParameterCheckpoint
|
|
|
|
type Checkpoint []parameterCheckpoint
|
|
|
|
|
|
|
|
|
|
|
|
// Gradient is the gradient of the parameter.
|
|
|
|
// Gradient is the gradient of the parameter.
|
|
|
|
type Gradient Parameter
|
|
|
|
type Gradient Parameter
|
|
|
|
@ -81,12 +76,53 @@ type Service struct {
|
|
|
|
optMap map[string]*optimizer
|
|
|
|
optMap map[string]*optimizer
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// parameterCheckpoint saves parameter checkpoint
|
|
|
|
|
|
|
|
type parameterCheckpoint struct {
|
|
|
|
|
|
|
|
ParameterWithConfig
|
|
|
|
|
|
|
|
State []byte
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// NewCheckpointFromFile loads parameters and state from checkpoint file
|
|
|
|
|
|
|
|
func NewCheckpointFromFile(cpPath string, idx int, e *EtcdClient) (Checkpoint, error) {
|
|
|
|
|
|
|
|
v, err := e.GetKey(PsPath+string(idx), 3*time.Second)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return nil, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
var cpMeta checkpointMeta
|
|
|
|
|
|
|
|
if err = json.Unmarshal(v, &cpMeta); err != nil {
|
|
|
|
|
|
|
|
return nil, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fn := filepath.Join(cpPath, cpMeta.UUID)
|
|
|
|
|
|
|
|
if _, err = os.Stat(fn); os.IsNotExist(err) {
|
|
|
|
|
|
|
|
return nil, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
content, err := ioutil.ReadFile(fn)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return nil, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
h := md5.New()
|
|
|
|
|
|
|
|
md5 := hex.EncodeToString(h.Sum(content))
|
|
|
|
|
|
|
|
if md5 != cpMeta.MD5 {
|
|
|
|
|
|
|
|
return nil, errors.New(CheckpointMD5Failed)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
dec := gob.NewDecoder(bytes.NewReader(content))
|
|
|
|
|
|
|
|
cp := Checkpoint{}
|
|
|
|
|
|
|
|
if err = dec.Decode(cp); err != nil {
|
|
|
|
|
|
|
|
return nil, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
return cp, nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// NewService creates a new service, will bypass etcd registration if no
|
|
|
|
// NewService creates a new service, will bypass etcd registration if no
|
|
|
|
// endpoints specified.
|
|
|
|
// endpoints specified. It will recovery from checkpoint file if a exists a specified checkpoint.
|
|
|
|
func NewService(idx int, seconds int, path string, client *EtcdClient, cp Checkpoint) (*Service, error) {
|
|
|
|
func NewService(idx int, interval time.Duration, path string, client *EtcdClient, cp Checkpoint) (*Service, error) {
|
|
|
|
s := &Service{
|
|
|
|
s := &Service{
|
|
|
|
idx: idx,
|
|
|
|
idx: idx,
|
|
|
|
checkpointInterval: time.Second * time.Duration(seconds),
|
|
|
|
checkpointInterval: interval,
|
|
|
|
checkpointPath: path,
|
|
|
|
checkpointPath: path,
|
|
|
|
client: client,
|
|
|
|
client: client,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
@ -95,9 +131,11 @@ func NewService(idx int, seconds int, path string, client *EtcdClient, cp Checkp
|
|
|
|
|
|
|
|
|
|
|
|
if cp != nil {
|
|
|
|
if cp != nil {
|
|
|
|
for _, item := range cp {
|
|
|
|
for _, item := range cp {
|
|
|
|
p := item.ParamConfig
|
|
|
|
p := ParameterWithConfig{
|
|
|
|
st := item.State
|
|
|
|
Param: item.Param,
|
|
|
|
s.optMap[p.Param.Name] = newOptimizer(p, st)
|
|
|
|
Config: item.Config,
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
s.optMap[p.Param.Name] = newOptimizer(p, item.State)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return s, nil
|
|
|
|
return s, nil
|
|
|
|
@ -186,13 +224,13 @@ func (s *Service) doCheckpoint() error {
|
|
|
|
s.mu.Lock()
|
|
|
|
s.mu.Lock()
|
|
|
|
defer s.mu.Unlock()
|
|
|
|
defer s.mu.Unlock()
|
|
|
|
|
|
|
|
|
|
|
|
cp := make([]ParameterCheckpoint, 0, len(s.optMap))
|
|
|
|
cp := make([]parameterCheckpoint, len(s.optMap))
|
|
|
|
index := 0
|
|
|
|
index := 0
|
|
|
|
for name, opt := range s.optMap {
|
|
|
|
for name, opt := range s.optMap {
|
|
|
|
var pc ParameterCheckpoint
|
|
|
|
var pc parameterCheckpoint
|
|
|
|
pc.ParamConfig.Param.Name = name
|
|
|
|
pc.Param.Name = name
|
|
|
|
pc.ParamConfig.Param.ElementType = opt.elementType
|
|
|
|
pc.Param.ElementType = opt.elementType
|
|
|
|
pc.ParamConfig.Param.Content = opt.GetWeights()
|
|
|
|
pc.Param.Content = opt.GetWeights()
|
|
|
|
pc.State = opt.GetStates()
|
|
|
|
pc.State = opt.GetStates()
|
|
|
|
cp[index] = pc
|
|
|
|
cp[index] = pc
|
|
|
|
index++
|
|
|
|
index++
|
|
|
|
@ -206,12 +244,12 @@ func (s *Service) doCheckpoint() error {
|
|
|
|
|
|
|
|
|
|
|
|
cpMeta := checkpointMeta{}
|
|
|
|
cpMeta := checkpointMeta{}
|
|
|
|
cpMeta.UUID = s.checkpointPath + strconv.Itoa(s.idx)
|
|
|
|
cpMeta.UUID = s.checkpointPath + strconv.Itoa(s.idx)
|
|
|
|
cpMeta.Timestamp = time.Now().String()
|
|
|
|
cpMeta.Timestamp = time.Now().UnixNano()
|
|
|
|
h := md5.New()
|
|
|
|
h := md5.New()
|
|
|
|
cpMeta.Md5sum = hex.EncodeToString(h.Sum(buf.Bytes()))
|
|
|
|
cpMeta.MD5 = hex.EncodeToString(h.Sum(buf.Bytes()))
|
|
|
|
|
|
|
|
|
|
|
|
cpMetajson, _ := json.Marshal(cpMeta)
|
|
|
|
cpMetajson, _ := json.Marshal(cpMeta)
|
|
|
|
err = s.client.PutKey(filepath.Join(PsCheckpoint, strconv.Itoa(s.idx)), cpMetajson, 3)
|
|
|
|
err = s.client.PutKey(filepath.Join(PsCheckpoint, strconv.Itoa(s.idx)), cpMetajson, 3*time.Second)
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
@ -219,7 +257,11 @@ func (s *Service) doCheckpoint() error {
|
|
|
|
log.Info("checkpoint does not exists.")
|
|
|
|
log.Info("checkpoint does not exists.")
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
err = os.Remove(cpMeta.UUID)
|
|
|
|
err = os.Remove(cpMeta.UUID)
|
|
|
|
log.Infof("checkpoint %s already exsits, removing ", cpMeta.UUID)
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
log.Infof("Removing checkpoint %s failed", cpMeta.UUID)
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
log.Infof("checkpoint %s already exsits, removing ", cpMeta.UUID)
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
f, err := os.Create(cpMeta.UUID)
|
|
|
|
f, err := os.Create(cpMeta.UUID)
|
|
|
|
defer f.Close()
|
|
|
|
defer f.Close()
|
|
|
|
|
fengjiayi
8 years ago