BYR Achieve · 镜像论坛

Re: 关于attention的分类

2022/2/27镜像同步0 回复

from tensorflow.keras.layers import Dense, Lambda, Dot, Activation, Concatenate from tensorflow.keras.layers import Layer class Attention1(Layer): def __init__(self, units=128, **kwargs): self.units = units super().__init__(**kwargs) def __call__(self, inputs): """ Many-to-one attention mechanism for Keras. @param inputs: 3D tensor with shape (batch_size, time_steps, input_dim). @return: 2D tensor with shape (batch_size, 128) @author: felixhao28, philipperemy. """ hidden_states = inputs hidden_size = int(hidden_states.shape[2]) # Inside dense layer # hidden_states dot W => score_first_part # (batch_size, time_steps, hidden_size) dot (hidden_size, hidden_size) => (batch_size, time_steps, hidden_size) # W is the trainable weight matrix of attention Luong's multiplicative style score score_first_part = Dense(hidden_size, use_bias=False)(hidden_states) # score_first_part dot last_hidden_state => attention_weights # (batch_size, time_steps, hidden_size) dot (batch_size, hidden_size) => (batch_size, time_steps) h_t = Lambda(lambda x: x[:, -1, :], output_shape=(hidden_size,))(hidden_states) score = Dot(axes=[1, 2])([h_t, score_first_part]) attention_weights = Activation('softmax')(score) # (batch_size, time_steps, hidden_size) dot (batch_size, time_steps) => (batch_size, hidden_size) context_vector = Dot(axes=[1, 1])([hidden_states, attention_weights]) pre_activation = Concatenate()([context_vector, h_t]) attention_vector = Dense(self.units, use_bias=False, activation='tanh')(pre_activation) # attention_vector = Dense(self.units, use_bias=False, activation='tanh')(pre_activation) return attention_vector def get_config(self): return { 'units': self.units} @classmethod def from_config(cls, config): return cls(**config)

订阅后，新回复会通过你的通知中心匿名送达。