深入理解 Deep Learning 前端框架 (I)

# logical-like ，用于表示概率分布
tanh(x)
sigmoid(x) # 1/(1+exp(-x))
hard_sigmoid(x) # faster sigmoid
softmax(x) # X_i <- exp(X_i)/sum(exp(X[])), sigmoid的延伸

# ReLU-like，线性修正单元
relu(x, alpha=0.0, max_value=None) # 即 min(max(alpha*x,x)),max_value)，(注：alpha<1.0)
softplus(x) # ReLU连续近似

max(x, axis=None, keepdims=False)
min(x, axis=None, keepdims=False)
sum(x, axis=None, keepdims=False)
prod(x, axis=None, keepdims=False)
var(x, axis=None, keepdims=False)
std(x, axis=None, keepdims=False)
mean(x, axis=None, keepdims=False)
any(x, axis=None, keepdims=False) # logical OR
all(x, axis=None, keepdims=False) # logical AND
argmax(x, axis=-1) # Returns the index of the maximum value along a tensor axis.
argmin(x, axis=-1) # Returns the index of the minimum value along a tensor axis.

zeros(shape, dtype='float32', name=None)
ones(shape, dtype='float32', name=None)
eye(size, dtype='float32', name=None)
# 这两个“模仿”张量的形状
zeros_like(x, name=None)
ones_like(x, name=None)
variable(value, dtype='float32', name=None) # numpy array -> tensor

# 描述了输入张量的“接口形状”，相当于定义了“张量类型”
placeholder(shape=None, ndim=None, dtype='float32', name=None) 

eval(x),get_value(x),batch_get_value(xs[]) # Tensor -> numpy array
set_value(x, value),batch_set_value(tuples(x,value)[]) # numpy array->Tensor
cast(x, dtype) # Casts a tensor to a different dtype.

int_shape(x) # 返回张量的形状，以int-tuple形式呈现
shape(x) # 返回张量的形状，以符号形式呈现
count_params(x) # 张量元素个数
dtype(x) # 张量数据类型
ndim(x) # 张量的维数（阶数）

# 包括张量间逐元素的 +, -, /, *, +=, -=, *=, /=
transpose(x) # Transpose a matrix
gather(reference, indices) # return reference[indices]

# 对于向量是内积运算，矩阵是标准矩阵乘法，
# 更高维度的是沿着x.shape[-1]和y.shape[-2]这两个轴求积之和
dot(x, y)


# 沿着某个轴展开，里面内容分别进行dot运算（向量化的dot）
batch_dot(x, y, axes=None)

# 改变张量的形状为shape。前后两个shape元素个数必须一样，
#   利用这个约束可以放入至多一个-1来自动计算该维度的尺寸
reshape(x, shape) 
flatten(x) # 把张量变成向量，相当于reshape(x, [-1])。是reshape高效具体实现
# 除了第一个维度外，剩余的都flatten作为第一个维度的元素, 最终构成一个矩阵。
#    是reshape高效具体实现
batch_flatten(x) 

permute_dimensions(x, pattern) # 将张量轴按照序号pattern重新排序


concatenate(tensors, axis=-1) # 沿轴拼接张量
# 层叠一个矩阵，形状(samples, dim) -> (samples, n, dim)。
repeat(x, n) 
# 沿轴重叠元素，不改变形状。If x has shape (s1, s2, s3) and axis=1, 
#    the output will have shape (s1, s2 * rep, s3)
#    是concatenate高效具体实现
repeat_elements(x, rep, axis)


# 加入一个空的维度，例如2->[2]; [4,5,6]->[[4,5,6]]或者[[4],[5],[6]]
expand_dims(x, dim=-1) 
# 去掉指定的size为1的轴(意味着没有)，比如 [[0]]->0 。和expand_dims互为逆操作
squeeze(x, axis)

# 这些损失以交叉熵形式为主，在信息学上有信服的解释。后面文章会解释这一点。

categorical_crossentropy(output, target, from_logits=False) # Categorical crossentropy between an output tensor and a target tensor, where the target is a tensor of the same shape as the output.

sparse_categorical_crossentropy(output, target, from_logits=False) # Categorical crossentropy between an output tensor and a target tensor, where the target is an integer tensor.

binary_crossentropy(output, target, from_logits=False)# Binary crossentropy between an output tensor and a target tensor.

conv2d(x, kernel, strides=(1, 1), border_mode='valid', dim_ordering='th', image_shape=None, filter_shape=None)
pool2d(x, pool_size, strides=(1, 1), border_mode='valid', dim_ordering='th', pool_mode='max')

# If condition then_expression else else_expression, 形成计算图中的分支
switch(condition, then_expression, else_expression):

def in_train_phase(x, alt):
    x = switch(_LEARNING_PHASE, x, alt)
    x._uses_learning_phase = True
    return x

def in_test_phase(x, alt):
    x = switch(_LEARNING_PHASE, alt, x)
    x._uses_learning_phase = True
    return x


# 表示一个计算图中的变换
'''Instantiates a Keras function.
inputs: list of placeholder/variable tensors.
outputs: list of output tensors.
updates: list of update tuples (old_tensor, new_tensor).
'''
function(inputs, outputs, updates=[], **kwargs)

# 获取变量相对损失的梯度
gradients(loss, variables)

# 循环网络，在图中相当于一个环
def rnn(step_function, inputs, initial_states,
        go_backwards=False, mask=None, constants=None,
        unroll=False, input_length=None):
    '''Iterates over the time dimension of a tensor.

    # Arguments
        inputs: tensor of temporal data of shape (samples, time, ...)
            (at least 3D).
        step_function:
            Parameters:
                input: tensor with shape (samples, ...) (no time dimension),
                    representing input for the batch of samples at a certain
                    time step.
                states: list of tensors.
            Returns:
                output: tensor with shape (samples, ...) (no time dimension),
                new_states: list of tensors, same length and shapes
                    as 'states'.
        initial_states: tensor with shape (samples, ...) (no time dimension),
            containing the initial values for the states used in
            the step function.
        go_backwards: boolean. If True, do the iteration over
            the time dimension in reverse order.
        mask: binary tensor with shape (samples, time, 1),
            with a zero for every element that is masked.
        constants: a list of constant values passed at each step.
        unroll: with TensorFlow the RNN is always unrolled, but with Theano you
            can use this boolean flag to unroll the RNN.
        input_length: not relevant in the TensorFlow implementation.
            Must be specified if using unrolling with Theano.

    # Returns
        A tuple (last_output, outputs, new_states).

        last_output: the latest output of the rnn, of shape (samples, ...)
        outputs: tensor with shape (samples, time, ...) where each
            entry outputs[s, t] is the output of the step function
            at time t for sample s.
        new_states: list of tensors, latest states returned by
            the step function, of shape (samples, ...).
    '''

# Sets entries in x to zero at random, while scaling the entire tensor.
dropout(x, level, seed=None) 

# 沿着一个轴除以L2范数，即x/T.sqrt(T.sum(T.square(x), axis=axis, keepdims=True))
l2_normalize(x, axis)

# 调整图像尺寸和格式
# Resizes the images contained in a 4D tensor of shape 
# - [batch, channels, height, width] (for 'th' dim_ordering) 
# - [batch, height, width, channels] (for 'tf' dim_ordering) 
# by a factor of (height_factor, width_factor). 
# Both factors should be positive integers.
resize_images(X, height_factor, width_factor, dim_ordering)


# 沿着一个3D Tensor 中间的轴镶边
temporal_padding(x, padding=1)

# 给图像集(4D Tensor)镶边
spatial_2d_padding(x, padding=(1, 1), dim_ordering='th')