piiswrong commented on a change in pull request #8302: Refactor operators &
MKLDNN
URL: https://github.com/apache/incubator-mxnet/pull/8302#discussion_r156265784
##########
File path: src/operator/nn/activation.cc
##########
@@ -21,65 +21,128 @@
* Copyright (c) 2015 by Contributors
* \file activation.cc
* \brief activation op
- * \author Bing Xu
+ * \author Bing Xu, Da Zheng
*/
#include "./activation-inl.h"
#include "../mshadow_op.h"
-#if MXNET_USE_MKL2017 == 1
-#include <mkl_memory.h>
-#include "../mkl/mkl_memory-inl.h"
-#include "../mkl/mkl_relu-inl.h"
-#endif // MXNET_USE_MKL2017
+#include "../tensor/elemwise_unary_op.h"
+#if MXNET_USE_MKLDNN == 1
+#include "./mkldnn/mkldnn_base-inl.h"
+#include "./mkldnn/mkldnn_act-inl.h"
+#endif // MXNET_USE_MKLDNN
namespace mxnet {
namespace op {
-template<>
-Operator *CreateOp<cpu>(ActivationParam param, int dtype, const TShape&
dshape) {
- Operator *op = NULL;
-#if MXNET_USE_MKL2017 == 1
- if (param.act_type == activation::kReLU && dshape.ndim() <= 4) {
- switch (dtype) {
- case mshadow::kFloat32:
- return new MKLReluOp<cpu, float>();
- case mshadow::kFloat64:
- return new MKLReluOp<cpu, double>();
- default:
- break;
- }
+
+DMLC_REGISTER_PARAMETER(ActivationParam);
+
+// This will determine the order of the inputs for backward computation.
+struct ActivationGrad {
+ const char *op_name;
+ std::vector<nnvm::NodeEntry> operator()(const nnvm::NodePtr& n,
+ const std::vector<nnvm::NodeEntry>&
ograds) const {
+ std::vector<nnvm::NodeEntry> heads(ograds.begin(), ograds.end());
+ heads.emplace_back(nnvm::NodeEntry{n, activation::kOut, 0});
+#if MXNET_USE_CUDNN == 1
+ heads.push_back(n->inputs[activation::kData]);
+#endif
+ return MakeGradNode(op_name, n, heads, n->attrs.dict);
+ }
+};
+
+static void ActivationComputeEx_CPU(const nnvm::NodeAttrs& attrs,
+ const OpContext& ctx,
+ const std::vector<NDArray>& inputs,
+ const std::vector<OpReqType>& req,
+ const std::vector<NDArray>& outputs) {
+ const ActivationParam& param = nnvm::get<ActivationParam>(attrs.parsed);
+ CHECK_EQ(inputs.size(), 1U);
+ CHECK_EQ(outputs.size(), 1U);
+#if MXNET_USE_MKLDNN == 1
+ if (SupportMKLDNN(inputs[0])) {
+ MKLDNNAct_Forward<float>(ctx, param, inputs[0], req[0], outputs[0]);
+ return;
}
- if (enableMKLWarnGenerated())
- LOG(INFO) << MKLReluOp<cpu, float>::getName() << " Skip MKL optimization";
#endif
- MSHADOW_REAL_TYPE_SWITCH(dtype, DType, {
- switch (param.act_type) {
- case activation::kReLU:
- op = new ActivationOp<cpu, mshadow_op::relu, mshadow_op::relu_grad,
DType>();
- break;
- case activation::kSigmoid:
- op = new ActivationOp<cpu, mshadow_op::sigmoid,
mshadow_op::sigmoid_grad, DType>();
- break;
- case activation::kTanh:
- op = new ActivationOp<cpu, mshadow_op::tanh, mshadow_op::tanh_grad,
DType>();
- break;
- case activation::kSoftReLU:
- op = new ActivationOp<cpu, mshadow_op::softrelu,
mshadow_op::softrelu_grad, DType>();
- break;
- default:
- LOG(FATAL) << "unknown activation type";
- }
- })
- return op;
+ _ActivationCompute<cpu>(param, ctx, inputs[0].data(), req[0],
outputs[0].data());
}
-// DO_BIND_DISPATCH comes from operator_common.h
-Operator *ActivationProp::CreateOperatorEx(Context ctx, std::vector<TShape>
*in_shape,
- std::vector<int> *in_type) const {
- DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0], (*in_shape)[0]);
+void ActivationGradComputeEx_CPU(const nnvm::NodeAttrs& attrs,
+ const OpContext& ctx,
+ const std::vector<NDArray>& inputs,
+ const std::vector<OpReqType>& req,
+ const std::vector<NDArray>& outputs) {
+#if MXNET_USE_CUDNN == 1
+ CHECK_EQ(inputs.size(), 3U);
+#else
+ CHECK_EQ(inputs.size(), 2U);
+#endif
+ const ActivationParam& param = nnvm::get<ActivationParam>(attrs.parsed);
+#if MXNET_USE_MKLDNN == 1
+ if (SupportMKLDNN(inputs[0])) {
+ MKLDNNAct_Backward<float>(ctx, param, inputs[0], inputs[1], req[0],
+ outputs[0]);
+ return;
+ }
+#endif
+ _ActivationGradCompute<cpu>(param, ctx, inputs[0].data(), inputs[1].data(),
+ req[0], outputs[0].data());
}
-DMLC_REGISTER_PARAMETER(ActivationParam);
+inline static bool ActivationStorageType(const nnvm::NodeAttrs& attrs,
+ const int dev_mask,
+ DispatchMode* dispatch_mode,
+ std::vector<int> *in_attrs,
+ std::vector<int> *out_attrs) {
+ CHECK_EQ(in_attrs->size(), 1);
+ CHECK_EQ(out_attrs->size(), 1);
+ const ActivationParam& param = nnvm::get<ActivationParam>(attrs.parsed);
+#if MXNET_USE_MKLDNN == 1
+ if (dev_mask == mshadow::cpu::kDevMask && SupportMKLDNNAct(param)
+ // There is no reason to use MKLDNN activation if the input isn't in
+ // MKLDNN format.
+ && in_attrs->at(0) == kMKLDNNStorage) {
+ *dispatch_mode = DispatchMode::kFComputeEx;
+ (*out_attrs)[0] = kMKLDNNStorage;
+ return true;
+ }
+#endif
+ return ElemwiseStorageType<1, 1, false, false, false>(attrs, dev_mask,
+ dispatch_mode, in_attrs, out_attrs);
+}
+
+inline static bool backward_ActStorageType(const nnvm::NodeAttrs& attrs,
Review comment:
BackwardActivationStorageType
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
