1 #include "caffe2/operators/selu_op.h" 3 #include "caffe2/utils/math.h" 8 bool SeluOp<float, CPUContext>::RunOnDevice() {
13 ConstEigenVectorArrayMap<float> Xvec(X.data<
float>(), X.size());
14 EigenVectorArrayMap<float> Yvec(Y->mutable_data<
float>(), Y->size());
15 Yvec = lambda_ * (Xvec > 0).select(Xvec, (alpha_ * Xvec.exp() - alpha_));
20 bool SeluGradientOp<float, CPUContext>::RunOnDevice() {
24 CAFFE_ENFORCE_EQ(dY.size(), Y.size());
27 ConstEigenVectorArrayMap<float> Yvec(Y.data<
float>(), Y.size());
28 ConstEigenVectorArrayMap<float> dYvec(dY.data<
float>(), dY.size());
29 EigenVectorArrayMap<float> dXvec(dX->mutable_data<
float>(), dX->size());
31 const float la = lambda_ * alpha_;
32 dXvec = (Yvec > 0).select(lambda_ * dYvec, dYvec * (Yvec + la));
36 REGISTER_CPU_OPERATOR(Selu, SeluOp<float, CPUContext>);
37 REGISTER_CPU_OPERATOR(SeluGradient, SeluGradientOp<float, CPUContext>);
43 .AllowInplace({{0, 0}})
44 .IdenticalTypeAndShape()
46 Selu takes one input data (Tensor<T>) and produces one output data 47 (Tensor<T>) where the function, y = scale*(alpha_*e^x-alpha_ if x < 0 else x), 48 is applied to the tensor elementwise. 52 "(float) default to 1.6732~; affects the activation function itself. " 53 "This should go with the weight initialization in the paper. " 54 " See https://arxiv.org/abs/1706.02515 ")
57 "(float) default to 1.0507~; affects the activation function itself.")
58 .Input(0,
"X",
"input tensor")
59 .Output(0,
"Y",
"input tensor")
60 .InheritOnnxSchema(
"Selu");
63 OPERATOR_SCHEMA(SeluGradient)
66 .AllowInplace({{1, 0}})
68 SeluGradient takes both Y and dY and uses this to update dX according to the 69 chain rule and derivatives of the selu function. 73 "(float) default to 1.6732~; affects the activation function itself." 74 "This should go with the weight initialization in the paper. " 75 " See https://arxiv.org/abs/1706.02515 ")
78 "(float) default to 1.0507~; affects the activation function itself.")
79 .Input(0,
"Y",
"input tensor")
80 .Input(1,
"dY",
"input tensor");
83 using GradientMakerBase::GradientMakerBase;
84 vector<OperatorDef> GetGradientDefs()
override {
86 def_.type() +
"Gradient",
88 vector<string>{O(0), GO(0)},
89 vector<string>{GI(0)});
A global dictionary that holds information about what Caffe2 modules have been loaded in the current ...
static vector< OperatorDef > SingleGradientDef(const Args &...args)
a helper function to allow one to create one single operator def, which is usually the case for many ...