This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 186e4990b0 [SYSTEMDS-3917] New built-in SELU activiation function
186e4990b0 is described below
commit 186e4990b0011fe4698b5a55e959fdee8552b88b
Author: e-strauss <[email protected]>
AuthorDate: Sun Sep 21 16:14:34 2025 +0200
[SYSTEMDS-3917] New built-in SELU activiation function
Closes #2328.
---
scripts/nn/layers/selu.dml | 66 +++++++++++++++++++++
.../test/applications/nn/NNComponentTest.java | 5 ++
.../scripts/applications/nn/component/selu.dml | 68 ++++++++++++++++++++++
3 files changed, 139 insertions(+)
diff --git a/scripts/nn/layers/selu.dml b/scripts/nn/layers/selu.dml
new file mode 100644
index 0000000000..83350b9b7a
--- /dev/null
+++ b/scripts/nn/layers/selu.dml
@@ -0,0 +1,66 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+/*
+ * SeLU (Scaled Exponential Linear Unit) nonlinearity layer.
+ */
+
+forward = function(matrix[double] X)
+ return (matrix[double] out) {
+ /*
+ * Computes the forward pass for a SeLU nonlinearity layer.
+ *
+ * selu(x) = lambda * ( x if x > 0
+ * α * (exp(x) - 1) if x <= 0 )
+ *
+ * Inputs:
+ * - X: Inputs, of shape (any, any).
+ *
+ * Outputs:
+ * - out: Outputs, of same shape as `X`.
+ */
+ alpha = 1.6732632423543772
+ lambda = 1.0507009873554805
+
+ out = (X > 0) * (lambda * X) +
+ (X <= 0) * (lambda * alpha * (exp(X) - 1))
+}
+
+backward = function(matrix[double] dout, matrix[double] X)
+ return (matrix[double] dX) {
+ /*
+ * Computes the backward pass for a SeLU nonlinearity layer.
+ *
+ * Inputs:
+ * - dout: Gradient wrt `out` from upstream, of same shape as `X`.
+ * - X: Inputs, of shape (any, any).
+ *
+ * Outputs:
+ * - dX: Gradient wrt `X`, of same shape as `X`.
+ */
+ alpha = 1.6732632423543772
+ lambda = 1.0507009873554805
+
+ dselu = (X > 0) * lambda +
+ (X <= 0) * (lambda * alpha * exp(X))
+
+ dX = dselu * dout
+}
diff --git
a/src/test/java/org/apache/sysds/test/applications/nn/NNComponentTest.java
b/src/test/java/org/apache/sysds/test/applications/nn/NNComponentTest.java
index 55d322a9b3..75f2799ffd 100644
--- a/src/test/java/org/apache/sysds/test/applications/nn/NNComponentTest.java
+++ b/src/test/java/org/apache/sysds/test/applications/nn/NNComponentTest.java
@@ -129,6 +129,11 @@ public class NNComponentTest extends TestFolder {
run("gelu.dml");
}
+ @Test
+ public void selu() {
+ run("selu.dml");
+ }
+
@Test
public void embedding() {
run("embedding.dml");
diff --git a/src/test/scripts/applications/nn/component/selu.dml
b/src/test/scripts/applications/nn/component/selu.dml
new file mode 100644
index 0000000000..93f0218d30
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/selu.dml
@@ -0,0 +1,68 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("nn/layers/selu.dml") as selu
+source("src/test/scripts/applications/nn/util.dml") as test_util
+
+selu_test1 = function() {
+ print("Testing SeLU, test 1")
+
+ X = matrix("1. -0.5
+ 0. 2.", rows=2, cols=2)
+ dout = matrix("1 1
+ 1 1", rows=2, cols=2)
+
+ # Reference from PyTorch nn.SELU
+ out_expected = matrix("1.050701 -0.69175816
+ 0. 2.101402", rows=2, cols=2)
+ gradient_expected = matrix("1.050701 1.0663412
+ 1.7580993 1.050701", rows=2, cols=2)
+
+ out = selu::forward(X)
+ test_util::check_all_close(out, out_expected, 0.00001)
+
+ gradient = selu::backward(dout, X)
+ test_util::check_all_close(gradient, gradient_expected, 0.00001)
+}
+
+selu_test2 = function() {
+ print("Testing SeLU, test 2")
+
+ X = matrix("0.5 -1.5
+ 1. -2.", rows=2, cols=2)
+ dout = matrix("1 1
+ 1 1", rows=2, cols=2)
+
+ # Precomputed reference from PyTorch nn.SELU
+ out_expected = matrix("0.5253505 -1.3658143
+ 1.050701 -1.5201665", rows=2, cols=2)
+ gradient_expected = matrix("1.050701 0.392285
+ 1.050701 0.23793286", rows=2, cols=2)
+
+ out = selu::forward(X)
+ test_util::check_all_close(out, out_expected, 0.00001)
+
+ gradient = selu::backward(dout, X)
+ test_util::check_all_close(gradient, gradient_expected, 0.00001)
+}
+
+selu_test1()
+selu_test2()
