|
|
|
@ -23,10 +23,11 @@ limitations under the License. */
|
|
|
|
namespace paddle {
|
|
|
|
namespace paddle {
|
|
|
|
|
|
|
|
|
|
|
|
enum BufferType {
|
|
|
|
enum BufferType {
|
|
|
|
TENSOR_NORMAL = 0,
|
|
|
|
TENSOR_UNKNOWN = 0,
|
|
|
|
TENSOR_SEQUENCE_ID = 1,
|
|
|
|
TENSOR_NORMAL = 1,
|
|
|
|
TENSOR_SEQUENCE_DATA = 2,
|
|
|
|
TENSOR_SEQUENCE_ID = 2,
|
|
|
|
TENSOR_SPARSE = 3
|
|
|
|
TENSOR_SEQUENCE_DATA = 3,
|
|
|
|
|
|
|
|
TENSOR_SPARSE = 4
|
|
|
|
};
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
enum SparseDataType {
|
|
|
|
enum SparseDataType {
|
|
|
|
@ -39,7 +40,6 @@ enum SparseDataFormat { SPARSE_CSR_FORMAT = 0, SPARSE_CSC_FORMAT = 1 };
|
|
|
|
class BufferArg;
|
|
|
|
class BufferArg;
|
|
|
|
class SequenceArg;
|
|
|
|
class SequenceArg;
|
|
|
|
class SparseMatrixArg;
|
|
|
|
class SparseMatrixArg;
|
|
|
|
typedef std::shared_ptr<BufferArg> BufferArgPtr;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
/**
|
|
|
|
* \brief BufferArg used as the argument type of Function.
|
|
|
|
* \brief BufferArg used as the argument type of Function.
|
|
|
|
@ -50,6 +50,11 @@ typedef std::shared_ptr<BufferArg> BufferArgPtr;
|
|
|
|
* 3. SequenceArg for a Buffer of sequence data.
|
|
|
|
* 3. SequenceArg for a Buffer of sequence data.
|
|
|
|
* 4. SparseMatrixArg for a Buffer of sparse matrix.
|
|
|
|
* 4. SparseMatrixArg for a Buffer of sparse matrix.
|
|
|
|
*
|
|
|
|
*
|
|
|
|
|
|
|
|
* Buffer shape
|
|
|
|
|
|
|
|
* For most buffers, the first dimension `shape()[0]` represents
|
|
|
|
|
|
|
|
* the size of the mini-batch.
|
|
|
|
|
|
|
|
*
|
|
|
|
|
|
|
|
* Buffer argType
|
|
|
|
* There is an ArgType property for the BufferArg used as Function Output.
|
|
|
|
* There is an ArgType property for the BufferArg used as Function Output.
|
|
|
|
* Whether the result of the Function calculation is assigned to the
|
|
|
|
* Whether the result of the Function calculation is assigned to the
|
|
|
|
* output Buffer or added to the output Buffer is determined by the
|
|
|
|
* output Buffer or added to the output Buffer is determined by the
|
|
|
|
@ -71,6 +76,14 @@ public:
|
|
|
|
ArgType getArgType() const { return argType_; }
|
|
|
|
ArgType getArgType() const { return argType_; }
|
|
|
|
|
|
|
|
|
|
|
|
public:
|
|
|
|
public:
|
|
|
|
|
|
|
|
BufferArg(ValueType valueType,
|
|
|
|
|
|
|
|
const TensorShape& shape,
|
|
|
|
|
|
|
|
ArgType argType = UNSPECIFIED)
|
|
|
|
|
|
|
|
: buf_(nullptr),
|
|
|
|
|
|
|
|
valueType_(valueType),
|
|
|
|
|
|
|
|
shape_(shape),
|
|
|
|
|
|
|
|
argType_(argType) {}
|
|
|
|
|
|
|
|
|
|
|
|
BufferArg(void* buf,
|
|
|
|
BufferArg(void* buf,
|
|
|
|
ValueType valueType,
|
|
|
|
ValueType valueType,
|
|
|
|
const TensorShape& shape,
|
|
|
|
const TensorShape& shape,
|
|
|
|
@ -86,6 +99,7 @@ public:
|
|
|
|
valueType_(DataType<real>::value),
|
|
|
|
valueType_(DataType<real>::value),
|
|
|
|
shape_(2),
|
|
|
|
shape_(2),
|
|
|
|
argType_(argType) {
|
|
|
|
argType_(argType) {
|
|
|
|
|
|
|
|
bufferType_ = TENSOR_NORMAL;
|
|
|
|
shape_.setDim(0, matrix.getHeight());
|
|
|
|
shape_.setDim(0, matrix.getHeight());
|
|
|
|
shape_.setDim(1, matrix.getWidth());
|
|
|
|
shape_.setDim(1, matrix.getWidth());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
@ -98,6 +112,7 @@ public:
|
|
|
|
valueType_(DataType<real>::value),
|
|
|
|
valueType_(DataType<real>::value),
|
|
|
|
shape_(shape),
|
|
|
|
shape_(shape),
|
|
|
|
argType_(argType) {
|
|
|
|
argType_(argType) {
|
|
|
|
|
|
|
|
bufferType_ = TENSOR_NORMAL;
|
|
|
|
CHECK_EQ(matrix.getElementCnt(), shape.getElements());
|
|
|
|
CHECK_EQ(matrix.getElementCnt(), shape.getElements());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@ -107,6 +122,7 @@ public:
|
|
|
|
valueType_(DataType<real>::value),
|
|
|
|
valueType_(DataType<real>::value),
|
|
|
|
shape_(1),
|
|
|
|
shape_(1),
|
|
|
|
argType_(argType) {
|
|
|
|
argType_(argType) {
|
|
|
|
|
|
|
|
bufferType_ = TENSOR_NORMAL;
|
|
|
|
shape_.setDim(0, vector.getSize());
|
|
|
|
shape_.setDim(0, vector.getSize());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@ -116,6 +132,7 @@ public:
|
|
|
|
valueType_(VALUE_TYPE_INT32),
|
|
|
|
valueType_(VALUE_TYPE_INT32),
|
|
|
|
shape_(1),
|
|
|
|
shape_(1),
|
|
|
|
argType_(argType) {
|
|
|
|
argType_(argType) {
|
|
|
|
|
|
|
|
bufferType_ = TENSOR_NORMAL;
|
|
|
|
shape_.setDim(0, vector.getSize());
|
|
|
|
shape_.setDim(0, vector.getSize());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@ -150,6 +167,8 @@ public:
|
|
|
|
ValueType valueType() const { return valueType_; }
|
|
|
|
ValueType valueType() const { return valueType_; }
|
|
|
|
BufferType bufferType() const { return bufferType_; }
|
|
|
|
BufferType bufferType() const { return bufferType_; }
|
|
|
|
const TensorShape& shape() const { return shape_; }
|
|
|
|
const TensorShape& shape() const { return shape_; }
|
|
|
|
|
|
|
|
bool isSparse() const { return (TENSOR_SPARSE == bufferType_); }
|
|
|
|
|
|
|
|
bool isSequenceArg() const { return TENSOR_SEQUENCE_DATA == bufferType_; }
|
|
|
|
|
|
|
|
|
|
|
|
const SequenceArg& sequence() const;
|
|
|
|
const SequenceArg& sequence() const;
|
|
|
|
const SparseMatrixArg& sparse() const;
|
|
|
|
const SparseMatrixArg& sparse() const;
|
|
|
|
@ -158,8 +177,8 @@ protected:
|
|
|
|
void* buf_;
|
|
|
|
void* buf_;
|
|
|
|
ValueType valueType_;
|
|
|
|
ValueType valueType_;
|
|
|
|
TensorShape shape_;
|
|
|
|
TensorShape shape_;
|
|
|
|
BufferType bufferType_;
|
|
|
|
BufferType bufferType_{TENSOR_UNKNOWN};
|
|
|
|
ArgType argType_ = UNSPECIFIED;
|
|
|
|
ArgType argType_{UNSPECIFIED};
|
|
|
|
// leading dimensions. The size is dims_.size()
|
|
|
|
// leading dimensions. The size is dims_.size()
|
|
|
|
// Dims lds_;
|
|
|
|
// Dims lds_;
|
|
|
|
};
|
|
|
|
};
|
|
|
|
@ -170,15 +189,24 @@ protected:
|
|
|
|
// if a < b then value_.buf_[a] < value_.buf_[b]
|
|
|
|
// if a < b then value_.buf_[a] < value_.buf_[b]
|
|
|
|
class SequenceIdArg : public BufferArg {
|
|
|
|
class SequenceIdArg : public BufferArg {
|
|
|
|
public:
|
|
|
|
public:
|
|
|
|
|
|
|
|
SequenceIdArg(const TensorShape& shape, ArgType argType = UNSPECIFIED)
|
|
|
|
|
|
|
|
: BufferArg(VALUE_TYPE_INT32, shape, argType) {
|
|
|
|
|
|
|
|
CHECK_EQ(shape_.ndims(), (size_t)1);
|
|
|
|
|
|
|
|
CHECK_GT(shape_[0], 1);
|
|
|
|
|
|
|
|
numSeqs_ = shape_[0] - 1;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
SequenceIdArg(void* buf,
|
|
|
|
SequenceIdArg(void* buf,
|
|
|
|
const TensorShape& shape,
|
|
|
|
const TensorShape& shape,
|
|
|
|
ArgType argType = UNSPECIFIED)
|
|
|
|
ArgType argType = UNSPECIFIED)
|
|
|
|
: BufferArg(buf, VALUE_TYPE_INT32, shape, argType) {
|
|
|
|
: BufferArg(buf, VALUE_TYPE_INT32, shape, argType) {
|
|
|
|
|
|
|
|
bufferType_ = TENSOR_SEQUENCE_ID;
|
|
|
|
CHECK_EQ(shape_.ndims(), (size_t)1);
|
|
|
|
CHECK_EQ(shape_.ndims(), (size_t)1);
|
|
|
|
numSeqs_ = shape_[0] - 1;
|
|
|
|
numSeqs_ = shape_[0] - 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
SequenceIdArg(const IVector& vector) : BufferArg(vector) {
|
|
|
|
SequenceIdArg(const IVector& vector) : BufferArg(vector) {
|
|
|
|
|
|
|
|
bufferType_ = TENSOR_SEQUENCE_ID;
|
|
|
|
numSeqs_ = shape_[0] - 1;
|
|
|
|
numSeqs_ = shape_[0] - 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@ -190,26 +218,41 @@ private:
|
|
|
|
size_t numSeqs_;
|
|
|
|
size_t numSeqs_;
|
|
|
|
};
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
// sequence data
|
|
|
|
// sequences data
|
|
|
|
|
|
|
|
// For mini-batch calculate,
|
|
|
|
|
|
|
|
// one batch can contain more than one sequence of data.
|
|
|
|
|
|
|
|
// SequenceArg can be used to represent sequences that contain multiple
|
|
|
|
|
|
|
|
// unequal lengths.
|
|
|
|
class SequenceArg : public BufferArg {
|
|
|
|
class SequenceArg : public BufferArg {
|
|
|
|
public:
|
|
|
|
public:
|
|
|
|
|
|
|
|
SequenceArg(ValueType valueType,
|
|
|
|
|
|
|
|
const TensorShape& shape,
|
|
|
|
|
|
|
|
ArgType argType = UNSPECIFIED)
|
|
|
|
|
|
|
|
: BufferArg(valueType, shape, argType), startPositions_(TensorShape()) {}
|
|
|
|
|
|
|
|
|
|
|
|
SequenceArg(void* buf,
|
|
|
|
SequenceArg(void* buf,
|
|
|
|
ValueType valueType,
|
|
|
|
ValueType valueType,
|
|
|
|
const TensorShape& shape,
|
|
|
|
const TensorShape& shape,
|
|
|
|
const SequenceIdArg& startPositions,
|
|
|
|
const SequenceIdArg& startPositions,
|
|
|
|
ArgType argType = UNSPECIFIED)
|
|
|
|
ArgType argType = UNSPECIFIED)
|
|
|
|
: BufferArg(buf, valueType, shape, argType),
|
|
|
|
: BufferArg(buf, valueType, shape, argType),
|
|
|
|
startPositions_(startPositions) {}
|
|
|
|
startPositions_(startPositions) {
|
|
|
|
|
|
|
|
bufferType_ = TENSOR_SEQUENCE_DATA;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
SequenceArg(const Matrix& matrix,
|
|
|
|
SequenceArg(const Matrix& matrix,
|
|
|
|
const IVector& vector,
|
|
|
|
const IVector& vector,
|
|
|
|
ArgType argType = UNSPECIFIED)
|
|
|
|
ArgType argType = UNSPECIFIED)
|
|
|
|
: BufferArg(matrix, argType), startPositions_(vector) {}
|
|
|
|
: BufferArg(matrix, argType), startPositions_(vector) {
|
|
|
|
|
|
|
|
bufferType_ = TENSOR_SEQUENCE_DATA;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
~SequenceArg() {}
|
|
|
|
~SequenceArg() {}
|
|
|
|
|
|
|
|
|
|
|
|
void* getIdBuf() const { return startPositions_.data(); }
|
|
|
|
void* getIdBuf() const { return startPositions_.data(); }
|
|
|
|
size_t numSeqs() const { return startPositions_.numSeqs(); }
|
|
|
|
size_t numSeqs() const { return startPositions_.numSeqs(); }
|
|
|
|
|
|
|
|
SequenceIdArg& getSequenceId() { return startPositions_; }
|
|
|
|
|
|
|
|
const SequenceIdArg& getSequenceId() const { return startPositions_; }
|
|
|
|
|
|
|
|
|
|
|
|
private:
|
|
|
|
private:
|
|
|
|
SequenceIdArg startPositions_;
|
|
|
|
SequenceIdArg startPositions_;
|
|
|
|
@ -235,6 +278,7 @@ public:
|
|
|
|
nnz_(nnz),
|
|
|
|
nnz_(nnz),
|
|
|
|
format_(format),
|
|
|
|
format_(format),
|
|
|
|
type_(type) {
|
|
|
|
type_(type) {
|
|
|
|
|
|
|
|
bufferType_ = TENSOR_SPARSE;
|
|
|
|
CHECK((valueType == VALUE_TYPE_FLOAT) || (valueType == VALUE_TYPE_DOUBLE));
|
|
|
|
CHECK((valueType == VALUE_TYPE_FLOAT) || (valueType == VALUE_TYPE_DOUBLE));
|
|
|
|
CHECK_EQ(shape_.ndims(), (size_t)2);
|
|
|
|
CHECK_EQ(shape_.ndims(), (size_t)2);
|
|
|
|
CHECK_EQ(row_.shape().ndims(), (size_t)1);
|
|
|
|
CHECK_EQ(row_.shape().ndims(), (size_t)1);
|
|
|
|
|
dangqingqing
8 years ago
