TensorFlow 中的块激活函数实现

Question

我试图重现一个 DNN，其中使用了称为 BlockRelu 的块激活函数。它被定义为

BlockRelu

我尝试根据一些自定义激活函数的示例代码来编写这个函数，但是这些函数都是标量函数，而 BlockRelu 将块作为一个整体来处理。由于 numpy 数组和张量的区别，这里不能使用 numpy 函数。我想知道是否有人可以提供帮助。谢谢。这是我的代码：

import tensorflow as tf
import numpy as np
from tensorflow.python.framework import ops

def block_relu(x):
    for i in range(x.shape[0]):
        if x[i] > 0:
            return x
    return x * 0


def grad_block_relu(x):
    for i in range(x.shape[0]):
        if x[i] > 0:
             return np.ones(x.shape[0])
    return x * 0


# transferring a common function into a numpy function, not needed here
'''
block_relu_np = np.vectorize(block_relu)
grad_block_relu_np = np.vectorize(grad_block_relu)
'''
# numpy uses float64 but tensorflow uses float32
block_relu_np32 = lambda x: block_relu(x).astype(np.float32)
grad_block_relu_np32 = lambda x: grad_block_relu(x).astype(np.float32)


def grad_block_relu_tf(x, name=None):
    with ops.name_scope(name, "grad_block_relu_tf", [x]) as name:
        y = tf.py_func(grad_block_relu_np32, [x], [tf.float32], False, name)
    return y[0]


def my_py_func(func, inp, Tout, stateful=False, name=None, my_grad_func=None):
    # a unique name is required to avoid duplicates:
    random_name = "PyFuncGrad" + str(np.random.randint(0, 1E+8))
    tf.RegisterGradient(random_name)(my_grad_func)
    g = tf.get_default_graph()
    with g.gradient_override_map({"PyFunc": random_name, "PyFuncStateless": random_name}):
        return tf.py_func(func, inp, Tout, stateful=stateful, name=name)


# The gradient function we need to pass to the above my_py_func function takes a special form:
# It needs to take in (an operation, the previous gradients before the operation)
# and propagate(i.e., return) the gradients backward after the operation.
def _block_relu_grad(op, pre_grad):
    x = op.inputs[0]
    cur_grad = grad_block_relu(x)
    next_grad = pre_grad * cur_grad
    return next_grad


def block_relu_tf(x, name=None):
    with ops.name_scope(name, "block_relu_tf", [x]) as name:
        y = my_py_func(block_relu_np32, [x], [tf.float32], stateful=False, name=name, my_grad_func=_block_relu_grad)
    return y[0]


with tf.Session() as sess:
    x = tf.constant([-0.3, 0.005, 0.08, 0.12])
    y = block_relu_tf(x)
    tf.global_variables_initializer().run()
    print(x.eval())
    print(y.eval())
    print(tf.gradients(y, [x])[0].eval())

会报错： TypeError：不允许将tf.Tensor 用作Python bool。使用if t is not None: 而不是if t: 来测试是否定义了张量，并使用TensorFlow ops（例如tf.cond）来执行以张量值为条件的子图。

Answer 1

我很确定您可以使用标准的 Tensorflow 函数来实现它：

# input: x
y = tf.scalar_mul( tf.sign( tf.reduce_max( tf.nn.relu(x))), x)