LSTM RNN反向传播

上面的两个传递都是以大小的块来完成的len(inputs),这对应于展开的RNN的大小.您可能希望将其设置得更大,以便在输入中捕获更长的依赖项,但您可以通过存储每个单元格的所有输出和渐变来为此付费.

LSTM有什么不同

LSTM图片和公式看起来令人生畏,但是一旦你编写了简单的vanilla RNN,LSTM的实现就差不多了.例如,这是向后传递:

# Loop over all cells, like before
d_h_next_t = np.zeros((N, H))
d_c_next_t = np.zeros((N, H))
for t in reversed(xrange(T)):
  d_x_t, d_h_prev_t, d_c_prev_t, d_Wx_t, d_Wh_t, d_b_t = lstm_step_backward(d_h_next_t + d_h[:,t,:], d_c_next_t, cache[t])
  d_c_next_t = d_c_prev_t
  d_h_next_t = d_h_prev_t

  d_x[:,t,:] = d_x_t
  d_h0 = d_h_prev_t
  d_Wx += d_Wx_t
  d_Wh += d_Wh_t
  d_b += d_b_t

# The step in each cell
# Captures all LSTM complexity in few formulas.
def lstm_step_backward(d_next_h, d_next_c, cache):
  """
  Backward pass for a single timestep of an LSTM.

  Inputs:
  - dnext_h: Gradients of next hidden state, of shape (N, H)
  - dnext_c: Gradients of next cell state, of shape (N, H)
  - cache: Values from the forward pass

  Returns a tuple of:
  - dx: Gradient of input data, of shape (N, D)
  - dprev_h: Gradient of previous hidden state, of shape (N, H)
  - dprev_c: Gradient of previous cell state, of shape (N, H)
  - dWx: Gradient of input-to-hidden weights, of shape (D, 4H)
  - dWh: Gradient of hidden-to-hidden weights, of shape (H, 4H)
  - db: Gradient of biases, of shape (4H,)
  """
  x, prev_h, prev_c, Wx, Wh, a, i, f, o, g, next_c, z, next_h = cache

  d_z = o * d_next_h
  d_o = z * d_next_h
  d_next_c += (1 - z * z) * d_z

  d_f = d_next_c * prev_c
  d_prev_c = d_next_c * f
  d_i = d_next_c * g
  d_g = d_next_c * i

  d_a_g = (1 - g * g) * d_g
  d_a_o = o * (1 - o) * d_o
  d_a_f = f * (1 - f) * d_f
  d_a_i = i * (1 - i) * d_i
  d_a = np.concatenate((d_a_i, d_a_f, d_a_o, d_a_g), axis=1)

  d_prev_h = d_a.dot(Wh.T)
  d_Wh = prev_h.T.dot(d_a)

  d_x = d_a.dot(Wx.T)
  d_Wx = x.T.dot(d_a)

  d_b = np.sum(d_a, axis=0)

  return d_x, d_prev_h, d_prev_c, d_Wx, d_Wh, d_b

# remember that ps is the probability distribution from the forward pass
dy = np.copy(ps[t])  
dy[targets[t]] -= 1