You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
310 lines
9.6 KiB
310 lines
9.6 KiB
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License. */
|
|
|
|
#pragma once
|
|
#include "ModelConfig.pb.h"
|
|
#include "paddle/gserver/layers/DataLayer.h"
|
|
#include "paddle/trainer/Trainer.h"
|
|
|
|
#include "paddle/testing/TestUtil.h"
|
|
using namespace std; // NOLINT
|
|
|
|
namespace paddle {
|
|
enum InputType {
|
|
INPUT_DATA, // dense vector
|
|
INPUT_LABEL, // id
|
|
INPUT_DATA_TARGET, // dense vector, but no gradient
|
|
INPUT_SEQUENCE_DATA,
|
|
INPUT_HASSUB_SEQUENCE_DATA, // sequence has sub-sequence
|
|
INPUT_SEQUENCE_MDIM_DATA,
|
|
INPUT_SEQUENCE_LABEL,
|
|
INPUT_SPARSE_NON_VALUE_DATA,
|
|
INPUT_SPARSE_FLOAT_VALUE_DATA,
|
|
INPUT_DENSE_DIM_DATA, // using sequence length to init dense data
|
|
INPUT_SELF_DEFINE_DATA, // support customizing for input value
|
|
};
|
|
|
|
struct ParaSparse {
|
|
bool sparse;
|
|
string format;
|
|
// if equalNnzPerSample is set true,
|
|
// every row of the sparse matrix in a format of CSR has a same
|
|
// number of nnz values. Currently, this flag is only used for
|
|
// selective_fc layer
|
|
bool equalNnzPerSample;
|
|
ParaSparse(const string& formatIn = "") { // NOLINT
|
|
if (formatIn == "") {
|
|
sparse = false;
|
|
} else {
|
|
sparse = true;
|
|
}
|
|
equalNnzPerSample = false;
|
|
}
|
|
ParaSparse(const string& formatIn, bool equalNnz) {
|
|
format = formatIn;
|
|
sparse = true;
|
|
equalNnzPerSample = equalNnz;
|
|
}
|
|
};
|
|
|
|
struct InputDef {
|
|
InputType inputType;
|
|
string name;
|
|
size_t dim;
|
|
size_t paraSize;
|
|
ParaSparse sparse;
|
|
bool isStatic;
|
|
std::vector<int> labelInitValue;
|
|
std::vector<int> labelSeqStartPositions;
|
|
MatrixPtr selfDefinedData;
|
|
|
|
InputDef(InputType type, string nameIn, size_t dimIn, size_t sizeIn) {
|
|
inputType = type;
|
|
name = nameIn;
|
|
dim = dimIn;
|
|
paraSize = sizeIn;
|
|
sparse = {""};
|
|
isStatic = false;
|
|
}
|
|
|
|
InputDef(InputType type,
|
|
string nameIn,
|
|
MatrixPtr selfDefinedData,
|
|
std::vector<int> selfDefinedSeqStartPos = {})
|
|
: labelSeqStartPositions(selfDefinedSeqStartPos),
|
|
selfDefinedData(selfDefinedData) {
|
|
inputType = type;
|
|
name = nameIn;
|
|
dim = 0;
|
|
sparse = {""};
|
|
paraSize = 0;
|
|
isStatic = false;
|
|
}
|
|
|
|
InputDef(InputType type,
|
|
string nameIn,
|
|
size_t dimIn,
|
|
size_t sizeIn,
|
|
const std::vector<int>& labelInitValue,
|
|
const std::vector<int>& labelSeqStartPositions)
|
|
: labelInitValue(labelInitValue),
|
|
labelSeqStartPositions(labelSeqStartPositions) {
|
|
inputType = type;
|
|
name = nameIn;
|
|
dim = dimIn;
|
|
paraSize = sizeIn;
|
|
sparse = {""};
|
|
isStatic = false;
|
|
}
|
|
|
|
InputDef(InputType type,
|
|
string nameIn,
|
|
size_t dimIn,
|
|
size_t sizeIn,
|
|
ParaSparse sparseIn) {
|
|
inputType = type;
|
|
name = nameIn;
|
|
dim = dimIn;
|
|
paraSize = sizeIn;
|
|
sparse = sparseIn;
|
|
}
|
|
};
|
|
|
|
struct TestConfig {
|
|
LayerConfig layerConfig;
|
|
std::vector<InputDef> inputDefs;
|
|
size_t biasSize;
|
|
real paramInitialMean;
|
|
real paramInitialStd;
|
|
bool testAccumulate;
|
|
bool testState;
|
|
bool staticBias;
|
|
bool testBatchState;
|
|
TestConfig()
|
|
: biasSize(0),
|
|
paramInitialMean(0.0),
|
|
paramInitialStd(1.0),
|
|
testAccumulate(true),
|
|
testState(false),
|
|
staticBias(false),
|
|
testBatchState(false) {}
|
|
};
|
|
|
|
real getCostSum(ParameterPtr& parameter,
|
|
CpuVector& cpuPara,
|
|
LayerPtr& testLayer,
|
|
MatrixPtr weights = nullptr);
|
|
|
|
real getDiffAndPrint(real newCost1,
|
|
real newCost2,
|
|
real callbackCount,
|
|
char fill,
|
|
string testLayerName,
|
|
string name,
|
|
real step,
|
|
real delta);
|
|
|
|
/**
|
|
* @brief verify that sequentially running forward() one timestamp at one time
|
|
* has same result as running forward() with one whole sequence
|
|
*
|
|
* @param testLayer[in/out] testLayer
|
|
* @param dataLayers[in/out] dataLayers
|
|
* @param datas[in/out] data of dataLayers
|
|
*/
|
|
void testState(LayerPtr testLayer,
|
|
vector<DataLayerPtr>& dataLayers,
|
|
vector<Argument>& datas);
|
|
|
|
/**
|
|
* @brief verify that sequentially running forward() with short sequences one
|
|
* time has same result as running forward() with long sequences.
|
|
*
|
|
* @param testLayer[in/out] testLayer
|
|
* @param dataLayers[in/out] dataLayers
|
|
* @param datas[in/out] data of dataLayers
|
|
*/
|
|
void testBatchState(LayerPtr testLayer,
|
|
vector<DataLayerPtr>& dataLayers,
|
|
vector<Argument>& datas);
|
|
|
|
/**
|
|
* @brief Generate a perturbation so that it is roughly aligned with the
|
|
* gradient direction. This is to make sure that change along this
|
|
* direction will make cost increase (or decrease) in a meaningful
|
|
* way so that the finite difference can be used to approximate the
|
|
* directional dirivative well.
|
|
*
|
|
* @param oldGrad[in] input gradient
|
|
* newGrad[out] output gradient
|
|
* dim dimension of oldGrad/newGrad
|
|
*
|
|
* @return sum_i(oldGrad[i] * newGrad[i])
|
|
*/
|
|
double genPerturbation(const real* oldGrad, real* newGrad, size_t dim);
|
|
|
|
void initWeight(MatrixPtr& weights);
|
|
|
|
void initBatchState(LayerPtr dataLayer,
|
|
LayerPtr testLayer,
|
|
LayerStatePtr state,
|
|
bool useGpu);
|
|
|
|
/**
|
|
* @brief initialize the dataLayer by its inputType
|
|
*
|
|
* @param testConf[in] test config
|
|
* dataLayers[out] dataLayers
|
|
* datas[out] initialized data of dataLayers
|
|
* layerMap[out] layerMap
|
|
*/
|
|
void initDataLayer(TestConfig testConf,
|
|
std::vector<DataLayerPtr>* dataLayers,
|
|
vector<Argument>* datas,
|
|
LayerMap* layerMap,
|
|
string testLayerName,
|
|
size_t batchSize,
|
|
bool trans,
|
|
bool useGpu);
|
|
|
|
/**
|
|
* @brief initialize the parameter of testLayer
|
|
*
|
|
* @param testConf[in/out] test config
|
|
* layerMap[out] layerMap
|
|
* parameters[out] parameters of testLayer
|
|
* testLayer[out] testLayer
|
|
*/
|
|
void initTestLayer(TestConfig testConf,
|
|
LayerMap* layerMap,
|
|
std::vector<ParameterPtr>* parameters,
|
|
LayerPtr* testLayer);
|
|
|
|
/**
|
|
* @brief Test whether the layer's forward calculation is stable by adding
|
|
* perturbation to its parameters
|
|
*
|
|
* @param testConf[in] test config
|
|
* weights[in] weights of testLayer
|
|
* state[in] state of testLayer
|
|
* cost[in] input cost
|
|
* callbackCount[in] number of done callback
|
|
* maxDiff[in/out] max of all previous diff
|
|
* testLayer[in/out] testLayer
|
|
* parameters[in/out] parameters of testLayer
|
|
*/
|
|
void testPerturbParameter(TestConfig testConf,
|
|
const MatrixPtr weights,
|
|
const LayerStatePtr state,
|
|
real cost,
|
|
real callbackCount,
|
|
real* maxDiff,
|
|
LayerPtr testLayer,
|
|
std::vector<ParameterPtr>* parameters);
|
|
|
|
/**
|
|
* @brief Test whether the layer's forward calculation is stable by adding
|
|
* perturbation to its input layers
|
|
*
|
|
* @param testConf[in] test config
|
|
* weights[in] weights of testLayer
|
|
* state[in] state of testLayer
|
|
* cost[in] input cost
|
|
* callbackCount[in] number of done callback
|
|
* maxDiff[in/out] max of all previous diff
|
|
* testLayer[in/out] testLayer
|
|
* dataLayers[in/out] dataLayers
|
|
*/
|
|
void testPerturbInput(TestConfig testConf,
|
|
const MatrixPtr weights,
|
|
const LayerStatePtr state,
|
|
real cost,
|
|
real callbackCount,
|
|
real* maxDiff,
|
|
LayerPtr testLayer,
|
|
std::vector<DataLayerPtr> dataLayers);
|
|
|
|
void testLayerGradKernel(TestConfig testConf,
|
|
string testLayerName,
|
|
size_t batchSize,
|
|
bool trans,
|
|
bool useGpu,
|
|
bool useWeight = false,
|
|
float epsilon = 0.02);
|
|
|
|
void testLayerGrad(TestConfig testConf,
|
|
string testLayerName,
|
|
size_t batchSize,
|
|
bool trans,
|
|
bool useGpu,
|
|
bool useWeight = false,
|
|
float epsilon = 0.02);
|
|
|
|
void testProjectionGrad(ProjectionConfig conf,
|
|
InputType inputType,
|
|
size_t parameterSize,
|
|
size_t batchSize,
|
|
bool useGpu,
|
|
bool testState = false,
|
|
int biasSize = 0,
|
|
bool sharedBias = false);
|
|
|
|
void testOperatorGrad(TestConfig& config,
|
|
OperatorConfig& operatorConf,
|
|
size_t batchSize,
|
|
bool useGpu,
|
|
bool testState = false);
|
|
|
|
} // namespace paddle
|