在 CNTK 中创建自定义错误函数

Question

这是我当前使用 CNTK 模块在 python 中进行 NN 训练的 python 代码的一部分

batch_axis = C.Axis.default_batch_axis()
input_seq_axis = C.Axis.default_dynamic_axis()

input_dynamic_axes = [batch_axis, input_seq_axis]
input_dynamic_axes2 = [batch_axis, input_seq_axis]

input = C.input_variable(n_ins, dynamic_axes=input_dynamic_axes, dtype=numpy.float32)
output = C.input_variable(n_outs, dynamic_axes=input_dynamic_axes2, dtype=numpy.float32)

dnn_model = cntk_model.create_model(input, hidden_layer_type, hidden_layer_size, n_outs)

loss = C.squared_error(dnn_model, output)
error = C.squared_error(dnn_model, output)

lr_schedule = C.learning_rate_schedule(current_finetune_lr, C.UnitType.minibatch)
            momentum_schedule = C.momentum_schedule(current_momentum)

learner = C.adam(dnn_model.parameters, lr_schedule, momentum_schedule, unit_gain = False, l1_regularization_weight=l1_reg, l2_regularization_weight= l2_reg)    

trainer = C.Trainer(dnn_model, (loss, error), [learner])  

这里是创建神经网络模型的代码

def create_model(features, hidden_layer_type, hidden_layer_size, n_out):
    logger.debug('Creating cntk model')
    assert len(hidden_layer_size) == len(hidden_layer_type)

    n_layers = len(hidden_layer_size)

    my_layers = list()
    for i in xrange(n_layers):
        if(hidden_layer_type[i] == 'TANH'):
            my_layers.append(C.layers.Dense(hidden_layer_size[i], activation=C.tanh, init=C.layers.glorot_uniform()))
        elif (hidden_layer_type[i] == 'LSTM'):
            my_layers.append(C.layers.Recurrence(C.layers.LSTM(hidden_layer_size[i])))
        else:
            raise Exception('Unknown hidden layer type')

    my_layers.append(C.layers.Dense(n_out, activation=None))

    my_model = C.layers.Sequential([my_layers])
    my_model = my_model(features)

    return my_model

现在，我想改变一个反向传播，所以在计算误差时不是直接使用网络输出，而是经过一些额外计算后的输出。我试图定义这样的东西

 def create_error_function(self, prediction, target):

    prediction_denorm = C.element_times(prediction, self.std_vector)
    prediction_denorm = C.plus(prediction_denorm, self.mean_vector)
    prediction_denorm_rounded = C.round(C.element_times(prediction_denorm[0:5], C.round(prediction_denorm[5])))
    prediction_denorm_rounded = C.element_divide(prediction_denorm_rounded, C.round(prediction_denorm[5]))

    prediction_norm = C.minus(prediction_denorm_rounded, self.mean_vector[0:5])
    prediction_norm = C.element_divide(prediction_norm, self.std_vector[0:5])

    first =  C.squared_error(prediction_norm, target[0:5])
    second = C.minus(C.round(prediction_denorm[5]), self.mean_vector[5])
    second = C.element_divide(second, self.std_vector[5])

    return C.plus(first, C.squared_error(second, target[5]))

并使用它来代替标准squared_error。 以及神经网络训练部分

dnn_model = cntk_model.create_model(input, hidden_layer_type, hidden_layer_size, n_outs)
 error_function = cntk_model.ErrorFunction(cmp_mean_vector, cmp_std_vector)
 loss = error_function.create_error_function(dnn_model, output)
 error = error_function.create_error_function(dnn_model, output)
 lr_schedule = C.learning_rate_schedule(current_finetune_lr, C.UnitType.minibatch)
 momentum_schedule = C.momentum_schedule(current_momentum)

 learner = C.adam(dnn_model.parameters, lr_schedule, momentum_schedule, unit_gain = False, l1_regularization_weight=l1_reg,
                                 l2_regularization_weight= l2_reg)    

 trainer = C.Trainer(dnn_model, (loss, error), [learner])  
 trainer.train_minibatch({input: temp_train_x, output: temp_train_y}) 

但是在两个 epoch 之后，我开始得到相同的平均损失，因为我的网络没有学习

Answer 1

每次你想改变反向传播的工作方式时，你都需要使用stop_gradient。这是唯一一个梯度与前向传播操作的梯度不同的函数。在前向传递中，stop_gradient 充当身份。在反向传播中，它阻止了梯度传播。

要在前向传递中对某些x 执行操作f(x) 并假装它在反向传递中从未发生过，您需要执行以下操作： C.stop_gradient(f(x) - x) + x。在你的情况下，那将是

norm_features = C.stop_gradient(features/normalization - features) + features