|
|
@ -52,7 +52,7 @@ def lstm(
|
|
|
|
g = np.dot(h_pre, w_h) # 1 x 4D
|
|
|
|
g = np.dot(h_pre, w_h) # 1 x 4D
|
|
|
|
g = g + x
|
|
|
|
g = g + x
|
|
|
|
g = np.reshape(g, (1, g.size))
|
|
|
|
g = np.reshape(g, (1, g.size))
|
|
|
|
c_tmp, g_i, g_f, g_o = np.split(g, 4, axis=1)
|
|
|
|
c, g_i, g_f, g_o = np.split(g, 4, axis=1)
|
|
|
|
if w_c is None:
|
|
|
|
if w_c is None:
|
|
|
|
g_i = act_gate(g_i) # 1 x D
|
|
|
|
g_i = act_gate(g_i) # 1 x D
|
|
|
|
g_f = act_gate(g_f) # 1 x D
|
|
|
|
g_f = act_gate(g_f) # 1 x D
|
|
|
@ -60,7 +60,7 @@ def lstm(
|
|
|
|
w_ic, w_fc, w_oc = np.split(w_c, 3, axis=1)
|
|
|
|
w_ic, w_fc, w_oc = np.split(w_c, 3, axis=1)
|
|
|
|
g_i = act_gate(g_i + w_ic * c_pre) # 1 x D
|
|
|
|
g_i = act_gate(g_i + w_ic * c_pre) # 1 x D
|
|
|
|
g_f = act_gate(g_f + w_fc * c_pre) # 1 x D
|
|
|
|
g_f = act_gate(g_f + w_fc * c_pre) # 1 x D
|
|
|
|
c = g_f * c_pre + g_i * act_cand(c_tmp) # 1 x D
|
|
|
|
c = g_f * c_pre + g_i * act_cand(c) # 1 x D
|
|
|
|
|
|
|
|
|
|
|
|
if w_c is None:
|
|
|
|
if w_c is None:
|
|
|
|
g_o = act_gate(g_o) # 1 x D
|
|
|
|
g_o = act_gate(g_o) # 1 x D
|
|
|
@ -68,8 +68,7 @@ def lstm(
|
|
|
|
_, _, w_oc = np.split(w_c, 3, axis=1)
|
|
|
|
_, _, w_oc = np.split(w_c, 3, axis=1)
|
|
|
|
g_o = act_gate(g_o + w_oc * c) # 1 x D
|
|
|
|
g_o = act_gate(g_o + w_oc * c) # 1 x D
|
|
|
|
h = g_o * act_cell(c)
|
|
|
|
h = g_o * act_cell(c)
|
|
|
|
bg = np.concatenate((act_cand(c_tmp), g_i, g_f, g_o), axis=1)
|
|
|
|
return h, c
|
|
|
|
return h, c, bg
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _reverse(x, lod):
|
|
|
|
def _reverse(x, lod):
|
|
|
|
y = np.zeros_like(x)
|
|
|
|
y = np.zeros_like(x)
|
|
|
@ -82,7 +81,6 @@ def lstm(
|
|
|
|
batch_size = len(offset) - 1
|
|
|
|
batch_size = len(offset) - 1
|
|
|
|
hidden = []
|
|
|
|
hidden = []
|
|
|
|
cell = []
|
|
|
|
cell = []
|
|
|
|
gate = []
|
|
|
|
|
|
|
|
input = _reverse(input, offset) if is_reverse else input
|
|
|
|
input = _reverse(input, offset) if is_reverse else input
|
|
|
|
if w_b is not None:
|
|
|
|
if w_b is not None:
|
|
|
|
input = input + np.tile(w_b, (offset[-1], 1))
|
|
|
|
input = input + np.tile(w_b, (offset[-1], 1))
|
|
|
@ -94,30 +92,26 @@ def lstm(
|
|
|
|
c_pre = c0[i] # 1 x D
|
|
|
|
c_pre = c0[i] # 1 x D
|
|
|
|
for j in range(seq_len):
|
|
|
|
for j in range(seq_len):
|
|
|
|
# compute one step
|
|
|
|
# compute one step
|
|
|
|
h_pre, c_pre, g_pre = _step(x[j], w_h, w_c, h_pre, c_pre, act_gate,
|
|
|
|
h_pre, c_pre = _step(x[j], w_h, w_c, h_pre, c_pre, act_gate,
|
|
|
|
act_cell, act_cand)
|
|
|
|
act_cell, act_cand)
|
|
|
|
hidden.append(h_pre.flatten())
|
|
|
|
hidden.append(h_pre.flatten())
|
|
|
|
cell.append(c_pre.flatten())
|
|
|
|
cell.append(c_pre.flatten())
|
|
|
|
gate.append(g_pre.flatten())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
hidden = np.array(hidden).astype('float64')
|
|
|
|
hidden = np.array(hidden).astype('float64')
|
|
|
|
cell = np.array(cell).astype('float64')
|
|
|
|
cell = np.array(cell).astype('float64')
|
|
|
|
gate = np.array(gate).astype('float64')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
hidden = _reverse(hidden, offset) if is_reverse else hidden
|
|
|
|
hidden = _reverse(hidden, offset) if is_reverse else hidden
|
|
|
|
cell = _reverse(cell, offset) if is_reverse else cell
|
|
|
|
cell = _reverse(cell, offset) if is_reverse else cell
|
|
|
|
|
|
|
|
|
|
|
|
assert gate.shape == input.shape
|
|
|
|
|
|
|
|
assert hidden.shape == (input.shape[0], input.shape[1] / 4)
|
|
|
|
assert hidden.shape == (input.shape[0], input.shape[1] / 4)
|
|
|
|
assert cell.shape == (input.shape[0], input.shape[1] / 4)
|
|
|
|
assert cell.shape == (input.shape[0], input.shape[1] / 4)
|
|
|
|
return hidden, cell, gate
|
|
|
|
return hidden, cell
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestLstmOp(OpTest):
|
|
|
|
class TestLstmOp(OpTest):
|
|
|
|
def set_argument(self):
|
|
|
|
def set_argument(self):
|
|
|
|
self.lod = [[0, 2, 6, 9]]
|
|
|
|
self.lod = [[0, 2, 6]]
|
|
|
|
self.D = 16
|
|
|
|
self.D = 16
|
|
|
|
self.sort_idx = [2, 6, 0, 3, 7, 1, 4, 8, 5]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
self.act_gate = 'sigmoid'
|
|
|
|
self.act_gate = 'sigmoid'
|
|
|
|
self.act_cell = 'tanh'
|
|
|
|
self.act_cell = 'tanh'
|
|
|
@ -141,22 +135,18 @@ class TestLstmOp(OpTest):
|
|
|
|
|
|
|
|
|
|
|
|
w_b = b[:, 0:4 * self.D]
|
|
|
|
w_b = b[:, 0:4 * self.D]
|
|
|
|
w_c = b[:, 4 * self.D:]
|
|
|
|
w_c = b[:, 4 * self.D:]
|
|
|
|
h, c, g = lstm(x, self.lod, h0, c0, w, w_b, w_c, self.is_reverse,
|
|
|
|
h, c = lstm(x, self.lod, h0, c0, w, w_b, w_c, self.is_reverse,
|
|
|
|
ACTVATION[self.act_gate], ACTVATION[self.act_cell],
|
|
|
|
ACTVATION[self.act_gate], ACTVATION[self.act_cell],
|
|
|
|
ACTVATION[self.act_cand])
|
|
|
|
ACTVATION[self.act_cand])
|
|
|
|
|
|
|
|
|
|
|
|
g_sort = np.zeros_like(x)
|
|
|
|
|
|
|
|
for i, j in enumerate(self.sort_idx):
|
|
|
|
|
|
|
|
g_sort[i, :] = g[j, :]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
self.inputs = {'Input': (x, self.lod), 'Weight': w, 'Bias': b}
|
|
|
|
self.inputs = {'Input': (x, self.lod), 'Weight': w, 'Bias': b}
|
|
|
|
self.inputs['H0'] = h0
|
|
|
|
if self.has_initial_state:
|
|
|
|
self.inputs['C0'] = c0
|
|
|
|
self.inputs['H0'] = h0
|
|
|
|
|
|
|
|
self.inputs['C0'] = c0
|
|
|
|
|
|
|
|
|
|
|
|
self.outputs = {
|
|
|
|
self.outputs = {
|
|
|
|
'Hidden': (h, self.lod),
|
|
|
|
'Hidden': (h, self.lod),
|
|
|
|
'Cell': (c, self.lod),
|
|
|
|
'Cell': (c, self.lod),
|
|
|
|
'BatchGate': g_sort,
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
self.attrs = {
|
|
|
|
self.attrs = {
|
|
|
|
'usePeepholes': True,
|
|
|
|
'usePeepholes': True,
|
|
|
@ -179,9 +169,8 @@ class TestLstmOp(OpTest):
|
|
|
|
|
|
|
|
|
|
|
|
class TestLstmOpHasNoInitial(TestLstmOp):
|
|
|
|
class TestLstmOpHasNoInitial(TestLstmOp):
|
|
|
|
def set_argument(self):
|
|
|
|
def set_argument(self):
|
|
|
|
self.lod = [[0, 2, 6, 9]]
|
|
|
|
self.lod = [[0, 2, 6]]
|
|
|
|
self.D = 64
|
|
|
|
self.D = 16
|
|
|
|
self.sort_idx = [2, 6, 0, 3, 7, 1, 4, 8, 5]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
self.act_gate = 'sigmoid'
|
|
|
|
self.act_gate = 'sigmoid'
|
|
|
|
self.act_cell = 'tanh'
|
|
|
|
self.act_cell = 'tanh'
|
|
|
@ -193,9 +182,8 @@ class TestLstmOpHasNoInitial(TestLstmOp):
|
|
|
|
|
|
|
|
|
|
|
|
class TestLstmOpRerverse(TestLstmOp):
|
|
|
|
class TestLstmOpRerverse(TestLstmOp):
|
|
|
|
def set_argument(self):
|
|
|
|
def set_argument(self):
|
|
|
|
self.lod = [[0, 2, 6, 9]]
|
|
|
|
self.lod = [[0, 2, 6]]
|
|
|
|
self.D = 64
|
|
|
|
self.D = 16
|
|
|
|
self.sort_idx = [2, 6, 0, 3, 7, 1, 4, 8, 5]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
self.act_gate = 'sigmoid'
|
|
|
|
self.act_gate = 'sigmoid'
|
|
|
|
self.act_cell = 'tanh'
|
|
|
|
self.act_cell = 'tanh'
|
|
|
|