This is an automated email from the ASF dual-hosted git repository.
arnabp20 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 85331dc0a3 [SYSTEMDS-3811] Multi-head Attention Layer
85331dc0a3 is described below
commit 85331dc0a3e1234cd092c0815e142f60e3c26425
Author: Maximilian.S <[email protected]>
AuthorDate: Mon Jan 6 13:11:08 2025 +0100
[SYSTEMDS-3811] Multi-head Attention Layer
This patch introduces multi-head attention layer with forward and backward
passes as a built-in layer.
The multi-head attention layer is the base layer of all most Transformer
models, with many variations for different models. This implementation is
in-line with the basic BERT attention layer. The functionality is currently
kept to a minimum without features like attention masking, head masking,
cross-attention, etc.
This patch is the first in a number of commits in an effort to support the
BERT
model in SystemDS and other transformer models in the future.
Closes #2172
---
scripts/nn/layers/multi_attention.dml | 205 +++++++++++++++++++++
.../nn/transformers/MultiAttentionLayerTest.java | 139 ++++++++++++++
.../input_attention_test4.csv | 2 +
.../input_attention_test5.csv | 8 +
.../input_attention_test6.csv | 1 +
.../multi_attention_layer/input_dcontext_test4.csv | 2 +
.../multi_attention_layer/input_dcontext_test5.csv | 8 +
.../multi_attention_layer/input_dcontext_test6.csv | 1 +
.../multi_attention_layer/input_key_test1.csv | 2 +
.../multi_attention_layer/input_key_test2.csv | 8 +
.../multi_attention_layer/input_key_test3.csv | 1 +
.../multi_attention_layer/input_key_test4.csv | 2 +
.../multi_attention_layer/input_key_test5.csv | 8 +
.../multi_attention_layer/input_key_test6.csv | 1 +
.../multi_attention_layer/input_query_test1.csv | 2 +
.../multi_attention_layer/input_query_test2.csv | 8 +
.../multi_attention_layer/input_query_test3.csv | 1 +
.../multi_attention_layer/input_query_test4.csv | 2 +
.../multi_attention_layer/input_query_test5.csv | 8 +
.../multi_attention_layer/input_query_test6.csv | 1 +
.../multi_attention_layer/input_value_test1.csv | 2 +
.../multi_attention_layer/input_value_test2.csv | 8 +
.../multi_attention_layer/input_value_test3.csv | 1 +
.../multi_attention_layer/input_value_test4.csv | 2 +
.../multi_attention_layer/input_value_test5.csv | 8 +
.../multi_attention_layer/input_value_test6.csv | 1 +
.../output_attention_test1.csv | 2 +
.../output_attention_test2.csv | 8 +
.../output_attention_test3.csv | 1 +
.../multi_attention_layer/output_context_test1.csv | 2 +
.../multi_attention_layer/output_context_test2.csv | 8 +
.../multi_attention_layer/output_context_test3.csv | 1 +
.../multi_attention_layer/output_dkey_test4.csv | 2 +
.../multi_attention_layer/output_dkey_test5.csv | 8 +
.../multi_attention_layer/output_dkey_test6.csv | 1 +
.../multi_attention_layer/output_dquery_test4.csv | 2 +
.../multi_attention_layer/output_dquery_test5.csv | 8 +
.../multi_attention_layer/output_dquery_test6.csv | 1 +
.../multi_attention_layer/output_dvalue_test4.csv | 2 +
.../multi_attention_layer/output_dvalue_test5.csv | 8 +
.../multi_attention_layer/output_dvalue_test6.csv | 1 +
.../nn/component/multi_attention_backward.dml | 66 +++++++
.../nn/component/multi_attention_forward.dml | 53 ++++++
43 files changed, 606 insertions(+)
diff --git a/scripts/nn/layers/multi_attention.dml
b/scripts/nn/layers/multi_attention.dml
new file mode 100644
index 0000000000..7b863f34b5
--- /dev/null
+++ b/scripts/nn/layers/multi_attention.dml
@@ -0,0 +1,205 @@
+
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("nn/layers/softmax.dml") as softmax
+source("nn/layers/dropout.dml") as dropout
+source("scripts/nn/util.dml") as util
+
+
+forward = function(matrix[double] Q, matrix[double] K,
+ matrix[double] V, int H, int T, int D, double dropout_p)
+ return (matrix[double] context, matrix[double] attention, matrix[double]
dropout_mask) {
+ /*
+ * Computes the forward pass for a multi-head attention layer.
+ *
+ * Inputs (B: Batch size, T: Sequence length, D: Embedding length, H: Heads):
+ * - Q: Input querys, of shape (B,T*H*D).
+ * - K: Input keys, of shape (B,T*H*D).
+ * - V: Input values, of shape (B,T*H*D).
+ * - H: Head count.
+ * - T: Sequence length.
+ * - D: Embedding length of single query, value, key,
+ * - dropout_p: Dropout probability.
+ *
+ * Outputs:
+ * - context: Token context embeddings, of shape (B, T*H*D)
+ * - attention: Attention on value(s) for given query(s), of shape (B, H*T*T)
+ * - dropout_mask: Dropout mask used on attention, of shape (B, H*T*T)
+ */
+ B = nrow(Q)
+
+ # Transpose head and token dimension for per-head computation
+ Q = util::transpose_ABCD_to_ACBD(Q, T, H) # Shape (B, H*T*D)
+ K = util::transpose_ABCD_to_ACBD(K, T, H) # Shape (B, H*T*D)
+ V = util::transpose_ABCD_to_ACBD(V, T, H) # Shape (B, H*T*D)
+
+ attention = matrix(0, rows=B, cols=H*T*T)
+ dropout_mask = matrix(0, rows=B, cols=H*T*T)
+ context = matrix(0, rows=B, cols=H*T*D)
+ K_norm = K / sqrt(D)
+
+ # For loops for tensor operations
+ for (batch in 1:B) {
+ attention_probs_b = matrix(0, rows=H, cols=T*T)
+ if (dropout_p > 0.0) {
+ dropout_mask_b = matrix(0, rows=H, cols=T*T)
+ }
+ context_b = matrix(0, rows=H, cols=T*D)
+ Q_b = matrix(Q[batch], rows=H, cols=T*D)
+ K_norm_b = matrix(K_norm[batch], rows=H, cols=T*D)
+ V_b = matrix(V[batch], rows=H, cols=T*D)
+
+ for (head in 1:H) {
+ Q_h = matrix(Q_b[head], rows=T, cols=D)
+ K_norm_h = matrix(K_norm_b[head], rows=T, cols=D)
+ V_h = matrix(V_b[head], rows=T, cols=D)
+
+ attention_scores = Q_h %*% t(K_norm_h) # Shape (T, T)
+
+ # TODO: Add support for attention mask here
+
+ # Column-wise softmax
+ attention_probs_h = softmax::forward(attention_scores)
+
+ if (dropout_p > 0.0) {
+ [attention_probs_h, dropout_mask_h] =
dropout::forward(attention_probs_h, dropout_p, -1)
+ }
+
+ context_h = attention_probs_h %*% V_h # Shape (T, D)
+
+ attention_probs_b[head] = matrix(attention_probs_h, rows=1, cols=T*T)
+ if (dropout_p > 0.0) {
+ dropout_mask_b[head] = matrix(dropout_mask_h, rows=1, cols=T*T)
+ }
+ context_b[head] = matrix(context_h, rows=1, cols=T*D)
+ }
+
+ attention[batch] = matrix(attention_probs_b, rows=1, cols=H*T*T)
+ if (dropout_p > 0.0) {
+ dropout_mask[batch] = matrix(dropout_mask_b, rows=1, cols=H*T*T)
+ }
+ context[batch] = matrix(context_b, rows=1, cols=H*T*D)
+ }
+
+ # Swap head and token dimension for original shape
+ context = util::transpose_ABCD_to_ACBD(context, H, T)
+}
+
+
+backward = function(matrix[double] dcontext,
+ matrix[double] dropout_mask, matrix[double] attention, matrix[double] Q,
+ matrix[double] K, matrix[double] V, int H, int T,
+ int D, double dropout_p)
+ return (matrix[double] dQ, matrix[double] dK, matrix[double] dV) {
+ /*
+ * Computes the backward pass for a multi-head attention layer.
+ *
+ * Inputs (B: Batch size, T: Sequence length, D: Embedding length, H: Heads):
+ * - dcontext: Gradient w.r.t. the context matrix of shape (B, T*H*D)
+ * - dropout_mask: Dropout mask from forward pass of shape (B, H*T*T)
+ * - attention: Attention output from forward pass of shape (B, H*T*T)
+ * - Q: Input querys, of shape (B,T*H*D).
+ * - K: Input keys, of shape (B,T*H*D).
+ * - V: Input values, of shape (B,T*H*D).
+ * - H: Head count.
+ * - T: Sequence length.
+ * - D: Embedding length of single query, value, key,
+ * - dropout_p: Dropout probability.
+ *
+ * Outputs:
+ * - dQ: Gradient w.r.t. input querys, of shape (B,T*H*D).
+ * - dK: Gradient w.r.t. input keys, of shape (B,T*H*D).
+ * - dV: Gradient w.r.t. input values, of shape (B,T*H*D).
+ */
+ B = nrow(Q)
+
+ # Transpose head and token dimension for per-head computation
+ dcontext = util::transpose_ABCD_to_ACBD(dcontext, T, H) # Shape (B, H*T*D)
+ Q = util::transpose_ABCD_to_ACBD(Q, T, H) # Shape (B, H*T*D)
+ K = util::transpose_ABCD_to_ACBD(K, T, H) # Shape (B, H*T*D)
+ V = util::transpose_ABCD_to_ACBD(V, T, H) # Shape (B, H*T*D)
+
+ dQ = matrix(0, rows=B, cols=H*T*D) # Shape (B, H*T*D)
+ dK = matrix(0, rows=B, cols=H*T*D) # Shape (B, H*T*D)
+ dV = matrix(0, rows=B, cols=H*T*D) # Shape (B, H*T*D)
+
+ K_norm = K / sqrt(D)
+
+ # For loops for tensor operations
+ for (batch in 1:B) {
+ dcontext_b = matrix(dcontext[batch], rows=H, cols=T*D)
+ if (dropout_p > 0.0) {
+ dropout_mask_b = matrix(dropout_mask[batch], rows=H, cols=T*T)
+ }
+ attention_b = matrix(attention[batch], rows=H, cols=T*T)
+
+ Q_b = matrix(Q[batch], rows=H, cols=T*D)
+ K_norm_b = matrix(K_norm[batch], rows=H, cols=T*D)
+ V_b = matrix(V[batch], rows=H, cols=T*D)
+
+ dQ_b = matrix(0, rows=H, cols=T*D)
+ dK_b = matrix(0, rows=H, cols=T*D)
+ dV_b = matrix(0, rows=H, cols=T*D)
+
+ for (head in 1:H) {
+ dcontext_h = matrix(dcontext_b[head], rows=T, cols=D)
+ if (dropout_p > 0.0) {
+ dropout_mask_h = matrix(dropout_mask_b[head], rows=T, cols=T)
+ }
+ attention_h = matrix(attention_b[head], rows=T, cols=T)
+
+ # Compute dV early to release attention_h
+ dV_h = t(attention_h) %*% dcontext_h
+
+ Q_h = matrix(Q_b[head], rows=T, cols=D)
+ K_norm_h = matrix(K_norm_b[head], rows=T, cols=D)
+ V_h = matrix(V_b[head], rows=T, cols=D)
+
+ dattention_probs = dcontext_h %*% t(V_h)
+
+ if (dropout_p > 0.0) {
+ # Provide unnecessary required X input matrix via empty matrix
+ dattention_probs = dropout::backward(dattention_probs, matrix(0,
rows=1, cols=1), dropout_p, dropout_mask_h)
+ }
+ attention_scores = Q_h %*% t(K_norm_h) # Shape (T, T)
+ dattention_scores = softmax::backward(dattention_probs, attention_scores)
+
+ dQ_h = dattention_scores %*% K_norm_h
+ dK_h = t(dattention_scores) %*% (Q_h / sqrt(D))
+
+ # Append to batch matrices
+ dK_b[head] = matrix(dK_h, rows=1, cols=T*D)
+ dQ_b[head] = matrix(dQ_h, rows=1, cols=T*D)
+ dV_b[head] = matrix(dV_h, rows=1, cols=T*D)
+ }
+
+ # Append to output matrices
+ dK[batch] = matrix(dK_b, rows=1, cols=H*T*D)
+ dQ[batch] = matrix(dQ_b, rows=1, cols=H*T*D)
+ dV[batch] = matrix(dV_b, rows=1, cols=H*T*D)
+ }
+
+ # Swap head and token dimensions
+ dK = util::transpose_ABCD_to_ACBD(dK, H, T)
+ dQ = util::transpose_ABCD_to_ACBD(dQ, H, T)
+ dV = util::transpose_ABCD_to_ACBD(dV, H, T)
+}
\ No newline at end of file
diff --git
a/src/test/java/org/apache/sysds/test/applications/nn/transformers/MultiAttentionLayerTest.java
b/src/test/java/org/apache/sysds/test/applications/nn/transformers/MultiAttentionLayerTest.java
new file mode 100644
index 0000000000..ba10f50e89
--- /dev/null
+++
b/src/test/java/org/apache/sysds/test/applications/nn/transformers/MultiAttentionLayerTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sysds.test.applications.nn.transformers;
+
+import org.apache.sysds.common.Types;
+import org.apache.sysds.test.AutomatedTestBase;
+import org.apache.sysds.test.TestConfiguration;
+import org.apache.sysds.test.TestUtils;
+import org.junit.Test;
+
+public class MultiAttentionLayerTest extends AutomatedTestBase {
+ private static final String TEST_NAME_FORWARD =
"multi_attention_forward";
+ private static final String TEST_NAME_BACKWARD =
"multi_attention_backward";
+ private static final String TEST_DIR = "applications/nn/component/";
+ private static final String RESOURCE_DIR =
"src/test/resources/component/transformers/multi_attention_layer/";
+
+ @Override
+ public void setUp() {
+ TestUtils.clearAssertionInformation();
+ addTestConfiguration(TEST_NAME_FORWARD, new
TestConfiguration(TEST_DIR, TEST_NAME_FORWARD));
+ addTestConfiguration(TEST_NAME_BACKWARD, new
TestConfiguration(TEST_DIR, TEST_NAME_BACKWARD));
+ }
+
+ @Test
+ public void testMultiAttentionForwardSimple() {
+ runMultiAttentionTest("test1", 2, 3, 4, 5, 0,
TEST_NAME_FORWARD, 1e-5, true);
+ }
+
+ @Test
+ public void testMultiAttentionForwardLarge() {
+ runMultiAttentionTest("test2", 8, 12, 10, 4, 0,
TEST_NAME_FORWARD, 1e-5, true);
+ }
+
+ @Test
+ public void testMultiAttentionForwardSmall() {
+ runMultiAttentionTest("test3", 1, 1, 1, 1, 0,
TEST_NAME_FORWARD, 1e-5, true);
+ }
+
+ @Test
+ public void testMultiAttentionBackwardSimple() {
+ runMultiAttentionTest("test4", 2, 3, 4, 5, 0,
TEST_NAME_BACKWARD, 1e-5, false);
+ }
+
+ @Test
+ public void testMultiAttentionBackwardLarge() {
+ runMultiAttentionTest("test5", 8, 12, 10, 5, 0,
TEST_NAME_BACKWARD, 1e-5, false);
+ }
+
+ @Test
+ public void testMultiAttentionBackwardSmall() {
+ runMultiAttentionTest("test6", 1, 1, 1, 1, 0,
TEST_NAME_BACKWARD, 1e-5, false);
+ }
+
+ private void runMultiAttentionTest(String testSuffix, int batchSize,
int seqLength, int numHeads, int embeddingDim,
+ int debug, String testname, double precision, boolean
isForward) {
+ // Set execution platform
+ Types.ExecMode platformOld =
setExecMode(Types.ExecMode.SINGLE_NODE);
+
+ try {
+ // Load test configuration
+ getAndLoadTestConfiguration(testname);
+ fullDMLScriptName = getScript();
+
+ // Program arguments
+ if (isForward) {
+ programArgs = new String[] {
+ "-stats", "-args",
+ String.valueOf(batchSize),
String.valueOf(seqLength),
+ String.valueOf(numHeads),
String.valueOf(embeddingDim),
+ String.valueOf(debug),
+ RESOURCE_DIR + "input_query_" +
testSuffix + ".csv",
+ RESOURCE_DIR + "input_key_" +
testSuffix + ".csv",
+ RESOURCE_DIR + "input_value_" +
testSuffix + ".csv",
+ RESOURCE_DIR + "output_context_" +
testSuffix + ".csv",
+ RESOURCE_DIR + "output_attention_" +
testSuffix + ".csv",
+ output("context_error"),
+ output("attention_error"),
+ };
+ } else {
+ programArgs = new String[] {
+ "-stats", "-args",
+ String.valueOf(batchSize),
String.valueOf(seqLength),
+ String.valueOf(numHeads),
String.valueOf(embeddingDim),
+ String.valueOf(debug),
+ RESOURCE_DIR + "input_query_" +
testSuffix + ".csv",
+ RESOURCE_DIR + "input_key_" +
testSuffix + ".csv",
+ RESOURCE_DIR + "input_value_" +
testSuffix + ".csv",
+ RESOURCE_DIR + "input_dcontext_" +
testSuffix + ".csv",
+ RESOURCE_DIR + "input_attention_" +
testSuffix + ".csv",
+ RESOURCE_DIR + "output_dquery_" +
testSuffix + ".csv",
+ RESOURCE_DIR + "output_dkey_" +
testSuffix + ".csv",
+ RESOURCE_DIR + "output_dvalue_" +
testSuffix + ".csv",
+ output("dquery_error"),
+ output("dkey_error"),
+ output("dvalue_error"),
+ };
+ }
+
+ // Run the test
+ runTest(true, EXCEPTION_NOT_EXPECTED, null, -1);
+
+ // Compare results
+ if (isForward) {
+ double contextMaxError = (Double)
readDMLScalarFromOutputDir("context_error").values().toArray()[0];
+ assert contextMaxError < precision;
+ double attentionMaxError = (Double)
readDMLScalarFromOutputDir("context_error").values().toArray()[0];
+ assert attentionMaxError < precision;
+ } else {
+ double dqueryMaxError = (Double)
readDMLScalarFromOutputDir("dquery_error").values().toArray()[0];
+ assert dqueryMaxError < precision;
+ double dkeyMaxError = (Double)
readDMLScalarFromOutputDir("dkey_error").values().toArray()[0];
+ assert dkeyMaxError < precision;
+ double dvalueMaxError = (Double)
readDMLScalarFromOutputDir("dvalue_error").values().toArray()[0];
+ assert dvalueMaxError < precision;
+ }
+ } catch (Throwable ex) {
+ ex.printStackTrace(System.out); // Log or debug all
exceptions or errors
+ throw new RuntimeException(ex);
+ } finally {
+ resetExecMode(platformOld);
+ }
+ }
+}
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_attention_test4.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_attention_test4.csv
new file mode 100644
index 0000000000..d05234c46f
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_attention_test4.csv
@@ -0,0 +1,2 @@
+0.328471,0.331907,0.339622,0.336846,0.335182,0.327972,0.335271,0.335710,0.329019,0.329675,0.336258,0.334066,0.342494,0.325649,0.331857,0.342283,0.335882,0.321835,0.331431,0.336706,0.331864,0.313857,0.340129,0.346014,0.346816,0.319861,0.333323,0.353853,0.299062,0.347086,0.330171,0.313161,0.356668,0.339396,0.324497,0.336107
+0.300318,0.358489,0.341193,0.319460,0.342199,0.338342,0.321149,0.352661,0.326190,0.331912,0.333238,0.334850,0.324741,0.354198,0.321062,0.307494,0.367816,0.324689,0.357409,0.339341,0.303250,0.357259,0.333069,0.309672,0.359344,0.327826,0.312830,0.333608,0.334254,0.332138,0.343666,0.329218,0.327117,0.328783,0.338962,0.332255
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_attention_test5.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_attention_test5.csv
new file mode 100644
index 0000000000..44b293bd37
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_attention_test5.csv
@@ -0,0 +1,8 @@
+0.083246,0.093026,0.095161,0.084799,0.082877,0.085630,0.073961,0.080433,0.080703,0.080499,0.080308,0.079355,0.083660,0.091285,0.094544,0.082730,0.083511,0.085979,0.075159,0.081966,0.080310,0.079373,0.080985,0.080498,0.080534,0.086752,0.086826,0.086371,0.084001,0.084610,0.076545,0.083232,0.082672,0.084408,0.081838,0.082210,0.083612,0.089648,0.092429,0.082099,0.082490,0.084556,0.076157,0.082465,0.081804,0.082307,0.081720,0.080714,0.081756,0.088847,0.090762,0.085248,0.083340,0.085068,0.0756
[...]
+0.076886,0.082187,0.085954,0.086862,0.086959,0.072629,0.086114,0.073056,0.085288,0.084685,0.095198,0.084183,0.076079,0.083301,0.084154,0.086652,0.088389,0.076259,0.086006,0.073761,0.085110,0.082340,0.092937,0.085012,0.080762,0.082302,0.083892,0.083839,0.085007,0.080264,0.084026,0.077797,0.084960,0.084762,0.088494,0.083893,0.079634,0.082348,0.086107,0.084794,0.085132,0.077153,0.085190,0.076187,0.083679,0.088475,0.087789,0.083511,0.075992,0.081963,0.082636,0.084902,0.086193,0.075555,0.0844
[...]
+0.088535,0.085611,0.081004,0.080250,0.081928,0.083938,0.079994,0.083855,0.076772,0.085693,0.085883,0.086537,0.091803,0.085370,0.079335,0.083439,0.083697,0.083327,0.079478,0.085658,0.075901,0.081422,0.087296,0.083273,0.088303,0.085367,0.081242,0.083187,0.080815,0.084841,0.079381,0.086297,0.076722,0.081680,0.088161,0.084004,0.092738,0.085599,0.079950,0.082787,0.080675,0.084185,0.079885,0.086370,0.074738,0.082466,0.085929,0.084679,0.089940,0.084776,0.079597,0.083781,0.086776,0.081981,0.0803
[...]
+0.078778,0.086965,0.085276,0.077643,0.082194,0.079389,0.084949,0.084716,0.084647,0.084820,0.086328,0.084295,0.080507,0.082314,0.085596,0.075771,0.087879,0.081024,0.084923,0.082644,0.085461,0.084909,0.081851,0.087121,0.081450,0.081106,0.084928,0.077253,0.087547,0.080101,0.086582,0.082432,0.086150,0.082535,0.082798,0.087119,0.081565,0.083473,0.084241,0.079945,0.084080,0.080651,0.085330,0.083424,0.085038,0.083019,0.084373,0.084862,0.082326,0.083700,0.083447,0.080655,0.082578,0.079621,0.0851
[...]
+0.088361,0.081150,0.082599,0.087709,0.085758,0.079785,0.074900,0.087355,0.083374,0.079491,0.084184,0.085334,0.087179,0.079620,0.082231,0.085124,0.086932,0.081918,0.078434,0.083004,0.083314,0.083366,0.086402,0.082474,0.086558,0.083541,0.076178,0.087914,0.086695,0.081972,0.072286,0.084344,0.081654,0.080238,0.090636,0.087983,0.090664,0.079745,0.079177,0.088350,0.089788,0.080162,0.070727,0.082545,0.081571,0.082029,0.091894,0.083346,0.084412,0.086251,0.075446,0.087006,0.085179,0.083065,0.0741
[...]
+0.085291,0.077350,0.086007,0.082966,0.079595,0.087412,0.081701,0.079746,0.081683,0.080866,0.086046,0.091336,0.084837,0.076279,0.087638,0.084869,0.077214,0.087808,0.079006,0.080330,0.084216,0.079111,0.091136,0.087557,0.084059,0.080116,0.085483,0.084019,0.079870,0.084086,0.081111,0.081996,0.084847,0.082528,0.087893,0.083991,0.085596,0.077631,0.085187,0.085067,0.077570,0.084437,0.079836,0.080787,0.083872,0.082477,0.090265,0.087275,0.085502,0.078400,0.083609,0.086392,0.074884,0.079492,0.0774
[...]
+0.074832,0.082944,0.087172,0.081536,0.079976,0.080736,0.081339,0.082932,0.091700,0.077714,0.090824,0.088295,0.075877,0.084505,0.084685,0.084942,0.082211,0.084157,0.080639,0.082417,0.089254,0.078693,0.085217,0.087404,0.076486,0.089269,0.082465,0.081309,0.076158,0.076071,0.085768,0.088562,0.091641,0.077982,0.090202,0.084089,0.080397,0.090243,0.082019,0.078586,0.073878,0.075507,0.089030,0.088589,0.089821,0.079817,0.089968,0.082145,0.079801,0.087908,0.082293,0.081017,0.078115,0.075649,0.0866
[...]
+0.079750,0.084024,0.092664,0.086539,0.082179,0.081699,0.076877,0.089044,0.073066,0.087040,0.088807,0.078309,0.081166,0.083937,0.095576,0.085439,0.081386,0.080915,0.082825,0.083942,0.074163,0.084390,0.090347,0.075913,0.078402,0.081582,0.087875,0.086755,0.085234,0.084393,0.073026,0.092813,0.072584,0.088716,0.089655,0.078964,0.078727,0.081478,0.089123,0.086612,0.084021,0.083772,0.074176,0.092157,0.073616,0.087994,0.089830,0.078494,0.079082,0.081256,0.090280,0.085674,0.085406,0.083292,0.0753
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_attention_test6.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_attention_test6.csv
new file mode 100644
index 0000000000..961b372472
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_attention_test6.csv
@@ -0,0 +1 @@
+1.000000
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_dcontext_test4.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_dcontext_test4.csv
new file mode 100644
index 0000000000..1f131bef5c
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_dcontext_test4.csv
@@ -0,0 +1,2 @@
+0.994792,0.176474,0.472702,0.309138,0.938715,0.928020,0.486365,0.326684,0.191067,0.857770,0.465568,0.970905,0.785131,0.937652,0.895462,0.175079,0.646351,0.292671,0.750877,0.455045,0.543285,0.154196,0.053137,0.688286,0.616193,0.543431,0.839597,0.291213,0.677490,0.224619,0.603600,0.385851,0.850163,0.937241,0.934262,0.522492,0.794847,0.000431,0.206703,0.589362,0.502788,0.107464,0.278694,0.266473,0.004619,0.547685,0.840847,0.702932,0.262434,0.040867,0.096611,0.509275,0.843957,0.297514,0.2279
[...]
+0.261542,0.409818,0.366224,0.060747,0.411853,0.653058,0.296971,0.478690,0.497608,0.002248,0.863358,0.903886,0.840903,0.335801,0.761282,0.031539,0.376307,0.398308,0.323425,0.710000,0.272171,0.442048,0.355623,0.030459,0.417430,0.076262,0.791552,0.767231,0.968396,0.332099,0.525426,0.597945,0.999109,0.958307,0.241230,0.074549,0.049356,0.259220,0.920393,0.667925,0.982427,0.870610,0.507249,0.190112,0.988250,0.901086,0.716471,0.226868,0.893890,0.545508,0.830617,0.708683,0.765504,0.061262,0.2227
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_dcontext_test5.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_dcontext_test5.csv
new file mode 100644
index 0000000000..65f45eacbb
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_dcontext_test5.csv
@@ -0,0 +1,8 @@
+0.392963,0.228653,0.142936,0.504400,0.105187,0.812799,0.566488,0.394110,0.932217,0.491263,0.559137,0.827782,0.584502,0.502480,0.957342,0.116007,0.747177,0.626792,0.422486,0.663463,0.540956,0.189024,0.162086,0.249021,0.951363,0.890452,0.446537,0.752374,0.802792,0.921301,0.632434,0.881485,0.366075,0.605218,0.195381,0.082435,0.391627,0.097465,0.666065,0.828428,0.600497,0.021426,0.712366,0.946326,0.579454,0.047267,0.397807,0.985339,0.213748,0.114253,0.869542,0.885512,0.505891,0.967943,0.0686
[...]
+0.435245,0.196422,0.266628,0.693277,0.913388,0.219769,0.257218,0.191786,0.534461,0.283960,0.672447,0.312599,0.366574,0.946272,0.248338,0.067917,0.568958,0.122334,0.342545,0.103301,0.142244,0.544804,0.727124,0.371206,0.018862,0.091397,0.467174,0.750291,0.376882,0.713939,0.602377,0.938038,0.721032,0.580360,0.238642,0.453445,0.916327,0.684319,0.168930,0.898290,0.768703,0.223879,0.535293,0.670993,0.706892,0.427597,0.484618,0.799026,0.506837,0.263136,0.871176,0.860276,0.293920,0.047590,0.2589
[...]
+0.005438,0.309060,0.403121,0.040276,0.658173,0.235356,0.356507,0.700954,0.981349,0.739398,0.129110,0.861830,0.449297,0.682596,0.192401,0.967800,0.180784,0.814006,0.919087,0.548716,0.887478,0.798158,0.424584,0.563203,0.510503,0.419108,0.057029,0.782634,0.022119,0.682446,0.318096,0.531715,0.396877,0.214480,0.312315,0.049295,0.201913,0.759497,0.912774,0.025016,0.385920,0.544826,0.221769,0.592712,0.445012,0.582598,0.222047,0.069186,0.447821,0.840668,0.934908,0.896806,0.526453,0.876954,0.2958
[...]
+0.743444,0.912176,0.186040,0.407370,0.034656,0.130231,0.687461,0.445552,0.870194,0.452434,0.271708,0.335063,0.552953,0.820300,0.540956,0.614040,0.324415,0.689961,0.162540,0.810615,0.478464,0.920027,0.245480,0.990159,0.817664,0.096180,0.351607,0.936668,0.669816,0.915858,0.395133,0.286470,0.831608,0.349973,0.953444,0.791975,0.997769,0.877089,0.351529,0.921705,0.112090,0.989255,0.498690,0.272127,0.646004,0.768354,0.523464,0.855374,0.464464,0.008649,0.822449,0.504515,0.780595,0.253042,0.8266
[...]
+0.636876,0.000282,0.849768,0.273276,0.409211,0.385552,0.888260,0.190617,0.005891,0.040134,0.727515,0.323416,0.574310,0.742284,0.509485,0.997824,0.700704,0.021430,0.105187,0.050619,0.159748,0.377453,0.312517,0.027364,0.709439,0.595003,0.433547,0.693085,0.364772,0.541178,0.287436,0.766436,0.894971,0.593390,0.904135,0.666195,0.359229,0.870612,0.599590,0.601885,0.552962,0.906047,0.181358,0.315016,0.193624,0.170993,0.757460,0.277928,0.576442,0.919408,0.759895,0.542823,0.907092,0.324119,0.9047
[...]
+0.311250,0.801719,0.730674,0.725569,0.213635,0.432845,0.661967,0.484496,0.303342,0.419031,0.680208,0.246374,0.999653,0.688037,0.869686,0.577211,0.674353,0.406128,0.313493,0.412849,0.159293,0.036703,0.448583,0.821436,0.698705,0.988855,0.435711,0.576177,0.581650,0.777461,0.940502,0.123074,0.328283,0.448640,0.404594,0.813630,0.008238,0.158328,0.173900,0.713493,0.123817,0.557792,0.436119,0.608168,0.491527,0.520943,0.863473,0.595027,0.733640,0.146786,0.387013,0.247390,0.148911,0.204675,0.3728
[...]
+0.857299,0.597123,0.042459,0.925121,0.577650,0.272020,0.480975,0.196170,0.710386,0.981617,0.853808,0.492631,0.016489,0.175792,0.162700,0.150639,0.928026,0.224792,0.718500,0.354699,0.475272,0.006267,0.673204,0.814622,0.108602,0.396067,0.421265,0.825920,0.641407,0.277450,0.865135,0.589091,0.289577,0.213410,0.754898,0.831026,0.929383,0.726606,0.638451,0.049486,0.547469,0.395432,0.490468,0.652479,0.382443,0.497963,0.888112,0.626110,0.801838,0.826507,0.423886,0.522563,0.112587,0.470462,0.2021
[...]
+0.328516,0.393186,0.128973,0.813804,0.937328,0.937696,0.731795,0.694886,0.084904,0.524996,0.921303,0.611605,0.735237,0.975499,0.558761,0.328898,0.971263,0.041974,0.489240,0.601086,0.557627,0.086337,0.280207,0.944861,0.465849,0.650188,0.689696,0.698381,0.810267,0.671071,0.411337,0.418965,0.204668,0.787957,0.992922,0.956280,0.205757,0.702096,0.210014,0.014669,0.920891,0.736578,0.078922,0.606529,0.959100,0.482750,0.587874,0.726272,0.745925,0.556380,0.579923,0.275611,0.713789,0.079457,0.7153
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_dcontext_test6.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_dcontext_test6.csv
new file mode 100644
index 0000000000..695ab823d8
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_dcontext_test6.csv
@@ -0,0 +1 @@
+0.896445
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_key_test1.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_key_test1.csv
new file mode 100644
index 0000000000..bf6fc9eee3
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_key_test1.csv
@@ -0,0 +1,2 @@
+-0.003111,-0.263699,-0.219225,0.327035,0.045647,0.022484,0.930655,0.440229,-0.016382,0.007087,-0.537432,-0.508113,0.306420,0.371754,-0.432621,0.405895,-0.183697,-0.552655,0.987270,0.066346,0.263197,0.002825,-0.088433,0.117729,-0.012996,0.010778,0.501280,0.425649,0.485244,0.296500,-0.242799,-0.306510,0.079727,0.434672,-0.283247,0.112271,-0.316979,-0.188155,0.570048,0.032841,-0.332163,-0.417790,0.011969,-0.116415,-0.205034,-0.147130,0.586599,0.185895,0.106592,0.270192,-0.133467,-0.248289,0
[...]
+-0.079516,-0.182464,0.034437,0.019141,-0.177563,-0.402828,0.780022,0.390347,0.085229,0.332268,-0.288753,-0.372717,0.146940,0.310685,-0.258524,0.525569,-0.074928,-0.522963,0.685508,0.078992,0.036685,-0.351236,-0.202489,0.148448,0.233867,0.231496,0.687291,0.240910,0.265501,0.034481,-0.181652,-0.423940,0.235418,0.348519,-0.259740,0.193475,-0.153588,-0.074949,0.575193,-0.047308,-0.009239,-0.038608,-0.334147,0.158105,-0.001435,-0.270891,0.690794,0.637097,0.458293,0.127587,-0.433201,-0.138821,
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_key_test2.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_key_test2.csv
new file mode 100644
index 0000000000..b13cd887d1
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_key_test2.csv
@@ -0,0 +1,8 @@
+-0.019784,0.592825,0.485221,-0.042171,0.498676,-0.472326,0.387562,0.096827,-0.450321,-0.675340,0.286012,-0.455466,-0.092317,-0.091090,-0.813935,0.174188,-0.198330,0.112594,0.310628,-0.931859,0.260157,-0.227872,-0.144571,-0.255912,0.534090,0.447508,0.004799,-0.562479,0.065026,0.194116,-0.323405,-0.019486,-0.286153,-1.031396,0.045508,-0.325508,0.049557,-0.642433,-0.044138,0.439802,0.149233,0.058272,0.448246,-0.048490,0.309822,-0.546974,0.245256,0.140077,-0.146564,-0.474474,0.209223,-0.0458
[...]
+0.187476,0.513769,0.328483,-0.066030,0.468745,-0.288224,0.418211,0.251375,0.181374,-0.195928,-0.297098,-0.020210,0.230911,0.085856,-0.839437,0.453963,-0.133098,0.447858,0.195468,-0.763352,0.487551,-0.149894,-0.119674,-0.301526,0.357042,0.500993,0.194950,-0.353119,-0.179499,0.430127,-0.276024,-0.094204,-0.107237,-0.667764,0.075525,-0.029678,-0.075073,-0.347600,-0.182207,0.309093,0.379645,0.663649,0.532994,-0.140945,0.784739,-0.267382,0.451052,0.076977,0.095782,-0.743944,-0.302293,-0.06158
[...]
+0.202491,0.158303,0.538683,-0.003633,0.407307,-0.353154,0.050268,0.032194,-0.612230,-0.528217,0.016142,0.207087,-0.168262,-0.079771,-0.866878,0.223149,0.016679,0.081869,0.257942,-0.670623,0.552379,0.074433,0.095533,-0.313008,0.398069,0.438990,0.271709,-0.384688,0.055190,0.129990,-0.426558,-0.304937,-0.241429,-0.840036,-0.166798,-0.285856,0.196938,-0.537138,-0.075189,0.324852,-0.095569,0.437866,0.522933,0.072473,0.508897,-0.587022,0.177272,0.291227,-0.031979,-0.392354,0.143814,-0.286279,-
[...]
+-0.022438,0.246378,0.770405,0.197988,0.382745,-0.232710,0.266350,0.009489,0.030418,-0.548919,-0.054620,-0.312704,0.106770,0.277785,-0.694199,0.329104,-0.069741,0.140278,0.519268,-0.888393,0.554501,-0.197496,-0.315425,-0.345102,0.299451,0.419219,0.128964,-0.442156,-0.217417,0.291519,-0.746670,-0.299555,-0.498572,-0.725467,0.656797,0.020635,0.219670,-0.687436,-0.030023,-0.044998,0.162585,0.675955,0.798082,0.059453,0.536960,-0.253862,0.164790,-0.019524,-0.295146,-0.313944,-0.405270,-0.20571
[...]
+0.073238,0.246662,0.347549,-0.094479,0.624785,-0.414522,0.327367,-0.014262,0.064987,-0.427631,-0.131182,-0.128853,0.005160,0.070717,-0.656401,0.195310,0.152748,0.286426,0.330651,-0.652577,0.478329,0.001652,-0.143860,-0.270924,0.570155,0.276149,0.077006,-0.006968,0.137234,0.097609,-0.692478,-0.598711,-0.055181,-0.654876,0.176624,-0.375853,-0.113148,-0.353952,0.318690,0.237601,0.558614,0.300292,0.490652,-0.357290,0.538619,-0.387594,0.274766,0.380698,-0.155359,-0.526244,-0.138594,-0.203792,
[...]
+-0.098460,0.358316,0.268473,-0.404690,0.486533,-0.476736,0.090795,0.180635,-0.089890,-0.643984,-0.019487,-0.119911,0.169516,0.196475,-0.885103,0.638870,-0.046920,0.467567,0.445331,-0.897811,0.396558,-0.108231,-0.228384,-0.432271,0.355327,0.141782,0.022153,-0.235842,-0.080544,0.426172,-0.656605,-0.440478,-0.432600,-0.771102,0.179635,-0.221216,-0.373891,-0.699700,-0.157620,0.236476,0.333914,0.553679,0.559188,-0.113966,0.609412,-0.611604,0.338888,0.237825,0.316756,-0.424884,0.056294,-0.0135
[...]
+0.160168,0.601306,0.770819,-0.021678,0.409857,-0.338438,0.357244,0.290470,0.003148,-0.237341,0.225779,-0.166187,0.036913,-0.077664,-0.670933,0.411168,0.013884,0.176041,0.293927,-0.525540,0.416292,0.064136,0.057885,-0.183260,0.411534,0.589197,0.149763,-0.480832,-0.269229,-0.118198,-0.370355,-0.220153,-0.329471,-0.480822,0.081322,-0.231301,-0.195929,-0.220752,0.210950,0.130387,-0.022762,0.564233,0.290253,0.204911,0.648510,-0.101435,0.358787,0.013281,0.170876,-0.350731,-0.210265,-0.217442,-
[...]
+0.104437,0.621212,0.708235,0.112525,0.370063,-0.257888,0.383515,0.049643,-0.101817,-0.567441,0.108223,-0.265986,-0.181267,0.049764,-0.613782,0.323250,-0.035831,0.149240,0.633008,-0.747141,0.336768,-0.128397,0.039464,-0.257612,0.178236,0.318673,0.087014,-0.434625,-0.064724,0.156126,-0.434349,-0.065540,-0.432033,-0.631909,-0.133848,-0.262326,0.050379,-0.543199,-0.143827,0.045684,0.182325,0.513371,0.007456,-0.178798,0.567030,-0.510287,0.321131,-0.038958,-0.125876,-0.630909,-0.230341,0.09664
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_key_test3.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_key_test3.csv
new file mode 100644
index 0000000000..9cdcb8662b
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_key_test3.csv
@@ -0,0 +1 @@
+0.174517
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_key_test4.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_key_test4.csv
new file mode 100644
index 0000000000..23d35d8777
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_key_test4.csv
@@ -0,0 +1,2 @@
+-0.049813,-0.127531,0.080303,-0.119295,-0.311173,0.080868,0.093306,-0.288081,0.329784,-0.301730,-0.126224,-0.042292,-0.735796,0.029484,0.074487,0.296771,0.470246,-0.180602,0.095718,-0.127066,-0.265502,-0.116926,0.041264,0.009459,-0.239862,-0.073799,-0.237116,-0.184763,0.190987,-0.199077,0.011812,-0.159021,-0.453350,0.163381,0.056998,0.389502,0.425955,0.300344,0.510214,-0.286531,-0.307838,-0.096709,-0.230734,-0.131694,0.168220,-0.136452,-0.027481,-0.000078,-0.004126,-0.096738,-0.061534,0.
[...]
+0.101039,-0.087095,0.276825,0.138073,-0.373106,-0.557663,0.255139,0.180785,0.044436,-0.208070,0.030210,0.346775,-0.535957,-0.097876,0.018476,0.319400,0.276720,0.277750,0.636916,0.528857,-0.195796,-0.257324,-0.438190,-0.228511,-0.004141,0.125456,-0.142045,0.001907,-0.408400,-0.261207,0.314263,-0.125428,-0.174890,-0.035556,0.174241,0.236069,0.381385,-0.167159,0.404523,-0.358203,-0.162557,-0.155455,0.166854,-0.225371,-0.230498,-0.322921,-0.042122,0.154774,0.148843,-0.227094,-0.137203,0.0145
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_key_test5.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_key_test5.csv
new file mode 100644
index 0000000000..6c25046037
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_key_test5.csv
@@ -0,0 +1,8 @@
+-0.254842,0.137988,0.519993,-0.115428,-0.145594,-0.384752,-0.202581,-0.460238,0.009269,0.488765,0.495987,0.288226,0.194663,-0.403606,-0.345200,-0.183141,0.014639,-0.009164,-0.036285,0.782263,0.449976,-0.253597,0.059719,0.781914,0.415032,0.678307,-0.274430,0.210903,-0.358295,0.126779,0.103797,-0.185029,-0.043665,0.457888,0.288905,-0.433694,-0.302164,-0.192537,0.458809,-0.093987,-0.021525,0.390938,0.040035,0.045252,-0.084313,-0.366238,-0.299928,-0.392524,-0.518639,-0.158428,-0.279040,0.550
[...]
+-0.202148,-0.148228,0.496473,0.179618,-0.076655,-0.527163,-0.282120,-0.422674,-0.584668,0.435227,0.140159,-0.007978,0.259804,-0.084685,-0.401694,-0.133829,0.138107,-0.176545,-0.253066,0.687066,0.374219,-0.310815,-0.067761,0.694951,0.284859,0.598754,0.063498,0.210928,-0.015257,-0.074390,0.371958,0.024919,-0.057157,0.489172,0.354568,-0.301794,-0.598355,-0.092505,0.037684,-0.060077,0.374045,0.082963,-0.414386,0.047168,0.451367,0.222762,-0.221124,0.130327,-0.423148,-0.304855,-0.159069,0.3929
[...]
+-0.527933,0.186188,0.045927,-0.082526,0.059241,-0.568605,-0.227692,-0.752986,-0.049374,0.273739,0.043243,0.151009,0.344937,-0.450116,0.020800,-0.572759,0.266173,-0.232539,-0.187534,0.578371,0.204456,-0.359203,0.094884,0.266496,0.306734,0.588349,-0.189423,0.213358,-0.366179,-0.341033,0.588369,-0.002569,-0.062497,0.378920,0.061658,-0.026779,-0.185307,-0.093458,-0.010545,0.096270,-0.455932,0.334689,-0.327516,0.361939,0.070677,0.153257,-0.157590,0.045821,-0.330047,-0.451437,-0.227220,0.16615
[...]
+-0.185583,-0.175646,0.418037,-0.004417,-0.081861,-0.241825,-0.486045,-0.623079,-0.014743,0.912440,0.427397,0.097249,0.260574,-0.385720,-0.038476,-0.200817,0.555642,-0.232288,-0.104016,0.650821,0.296576,-0.139373,-0.133430,0.800513,0.198852,0.525020,0.472940,0.118230,-0.497866,0.218245,0.066129,0.298738,0.105181,0.409390,0.312406,-0.358156,-0.187281,0.118733,0.148834,0.134941,0.169479,0.142042,-0.036655,-0.021930,0.224727,-0.157274,-0.130722,-0.253902,-0.400472,-0.504891,-0.201185,0.15541
[...]
+-0.236113,0.029437,0.045546,-0.110824,0.067278,-0.376718,0.140420,-0.762952,-0.107796,0.635872,-0.062553,0.114364,0.381783,0.056653,0.241552,-0.362866,0.141094,-0.463668,-0.541904,0.393130,0.269162,-0.346660,-0.033851,0.618625,0.286017,0.512493,-0.027340,0.554181,0.075968,-0.063172,0.502909,-0.230292,-0.243830,0.183357,0.360273,0.054498,-0.443002,-0.012773,-0.099563,0.127705,-0.031787,-0.085127,-0.569803,0.327496,0.160675,0.266071,-0.173891,0.175796,-0.457203,-0.442401,-0.372682,-0.03102
[...]
+-0.110813,0.149419,0.241846,-0.036050,0.018424,-0.708553,-0.127234,-0.425704,0.007837,0.441930,0.136355,0.032361,0.680482,-0.262556,-0.240611,-0.418376,0.246063,-0.145057,-0.042304,0.787075,0.596549,-0.712688,0.051808,0.767216,0.465254,0.558376,0.210405,0.350169,-0.185506,0.098465,0.275085,-0.183359,-0.071836,0.546709,0.283419,-0.413656,-0.547762,-0.203571,0.400098,0.128143,0.040452,0.283709,-0.135247,0.044949,-0.040833,0.001920,-0.076147,0.005220,-0.377127,0.073558,-0.256108,0.157701,0.
[...]
+-0.374054,0.140026,0.308526,-0.086547,0.464332,-0.478697,-0.371746,-0.374215,-0.238947,0.252964,0.141019,-0.160321,0.438503,-0.414194,-0.450852,-0.293784,0.193494,-0.316615,-0.085905,0.304297,0.779695,-0.496837,-0.004622,0.953814,0.202126,0.634176,0.106964,0.323004,-0.205374,0.266981,0.081647,-0.018919,0.016867,0.518862,0.175875,-0.047966,-0.160975,-0.005465,0.083820,0.016748,0.456405,0.121488,-0.001608,0.081307,0.233356,-0.073400,-0.576489,0.075253,-0.275227,-0.467870,-0.064230,-0.10385
[...]
+-0.329376,0.146996,0.418863,-0.086718,0.175295,-0.262880,-0.420101,-0.520474,-0.383489,0.686103,0.433731,-0.013110,0.118720,-0.043925,-0.143080,-0.336373,0.210505,-0.378264,-0.285056,0.403662,0.522883,-0.567226,-0.086145,0.718694,0.657392,0.667505,0.183513,0.304115,-0.153646,0.227959,0.330990,-0.256527,-0.010701,0.377956,-0.061585,-0.121764,0.027732,-0.337139,0.370638,-0.376168,0.286441,0.163525,-0.064903,0.141215,0.213006,-0.127716,-0.455307,0.132202,-0.585739,-0.554990,-0.184189,0.0523
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_key_test6.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_key_test6.csv
new file mode 100644
index 0000000000..fc1be22e66
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_key_test6.csv
@@ -0,0 +1 @@
+-0.750369
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_query_test1.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_query_test1.csv
new file mode 100644
index 0000000000..ca049afd92
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_query_test1.csv
@@ -0,0 +1,2 @@
+-0.368528,-0.022005,0.036665,0.628141,0.076853,-0.786394,0.450301,0.706332,0.637412,-0.160975,-0.504275,0.355853,-0.186994,0.345306,0.425570,0.037660,0.094517,0.432064,-0.681117,0.351171,-0.081002,0.371774,-0.350291,0.386241,0.071336,-0.428833,0.189132,0.263145,0.117348,-0.357378,-0.493600,0.146321,-0.196000,0.271219,0.209588,0.357517,0.124855,0.664095,-0.485595,-0.044402,-0.169968,0.330667,-0.149482,0.451134,-0.064948,-0.632721,0.596234,0.353431,0.156939,-0.116098,-0.187777,0.151780,0.0
[...]
+-0.384208,0.303074,-0.077292,0.439393,0.204769,-0.693747,0.565648,0.471843,0.239608,-0.007381,-0.238633,0.234540,-0.157128,0.441106,0.468595,0.128743,0.130886,0.375342,-0.431579,0.036377,-0.092307,-0.108694,-0.129631,0.537973,0.048605,-0.812956,0.122121,0.524155,0.336926,-0.192440,-0.473328,0.252406,-0.107975,0.417049,0.600812,0.428432,0.043938,0.494915,-0.435159,0.342768,0.007962,0.308129,-0.097410,0.608545,-0.164988,-0.511101,0.222764,0.287548,0.015718,-0.064185,-0.201857,0.341949,-0.2
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_query_test2.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_query_test2.csv
new file mode 100644
index 0000000000..379d567d99
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_query_test2.csv
@@ -0,0 +1,8 @@
+0.082577,0.461491,0.182286,0.028984,-0.124697,-0.077265,0.231703,0.369116,-0.136432,-0.214301,-0.341742,-0.185099,0.536347,-0.232304,0.221388,-0.060655,0.036448,0.389234,-0.162352,-0.060161,0.054276,0.481544,-0.315485,-0.399061,0.753004,0.588980,-0.351970,-0.598501,0.317567,-0.356739,0.224246,-0.665409,-0.066840,0.216675,0.024749,-0.546386,0.066238,0.144950,0.361534,-0.238218,-0.181723,0.201938,0.165457,-0.100015,-0.190691,0.024278,0.127285,0.144529,-0.075329,-0.428301,-0.136466,-0.21681
[...]
+0.187640,0.506308,-0.029212,0.098479,0.114765,-0.253903,0.272247,0.269549,-0.447221,-0.328949,-0.322670,-0.482135,-0.070748,-0.208727,0.187031,-0.068277,-0.126051,0.232768,0.042426,-0.141270,0.145770,0.258670,-0.480900,-0.780113,0.468032,0.612973,-0.552493,-0.664392,-0.041033,-0.140708,-0.024951,-0.734026,0.033647,0.141096,0.253803,-0.358029,0.228832,0.133430,0.287919,-0.310769,0.276484,0.337164,0.088879,0.050668,0.273646,-0.200992,0.388124,0.013770,-0.228547,-0.408805,-0.490636,-0.80047
[...]
+0.027160,0.372299,0.279773,0.158604,-0.184461,-0.278054,-0.082436,-0.132053,0.112392,-0.008747,-0.354664,-0.290750,0.317802,-0.052394,0.204379,-0.589926,-0.129836,0.482186,0.086751,-0.320458,0.300445,0.245541,-0.256464,-0.141359,0.777902,0.769282,-0.407655,-0.206245,0.382142,-0.164657,0.341288,-0.734469,0.096562,0.253406,-0.093069,-0.409177,0.110402,0.019281,0.078204,-0.138239,0.176822,0.683632,0.088654,-0.084887,-0.223332,-0.127782,0.205710,0.489501,0.003707,0.024349,-0.168822,-0.249337
[...]
+0.021467,0.355419,0.244420,0.118818,0.130198,0.238099,0.425191,0.057929,0.098808,-0.053353,-0.249774,-0.714914,0.080217,-0.355625,-0.058601,-0.100683,0.100955,0.239735,0.256481,-0.392873,-0.003741,0.237399,-0.497811,-0.537782,0.809082,0.420700,-0.699646,-0.408071,0.112222,-0.118941,0.283635,-0.624826,0.174224,0.259601,0.022536,-0.313912,-0.156698,0.461352,0.050380,-0.171635,0.103747,0.511266,0.137879,-0.020726,-0.081002,-0.270759,0.293873,0.198951,-0.033827,-0.033206,-0.324214,-0.625329,
[...]
+-0.165395,0.379471,0.304470,-0.332616,-0.079217,-0.028043,0.107828,0.146295,-0.201026,-0.256508,-0.355475,-0.092217,0.445763,-0.490068,-0.054282,-0.220821,-0.132360,0.334422,0.351419,-0.121957,0.135538,0.353194,-0.410915,-0.155930,0.660828,0.285569,-0.309597,-0.278053,0.266910,0.098828,-0.049727,-1.065791,0.118705,0.270047,-0.026711,-0.484730,-0.193554,0.170535,0.144092,-0.066409,0.394333,0.391122,0.238882,0.104009,-0.007315,-0.192323,0.340348,0.242588,-0.203157,-0.540404,-0.351513,-0.26
[...]
+0.133723,0.595980,0.132551,0.108579,-0.316708,-0.107228,0.344459,0.173678,-0.120832,-0.509820,-0.280175,-0.382125,0.587011,-0.551648,-0.100597,-0.024411,-0.205063,0.314483,0.076713,-0.402044,0.172324,0.110621,-0.493795,-0.609905,0.845116,0.696241,-0.387526,-0.587310,0.097785,-0.027349,-0.132349,-0.660667,0.072488,0.341878,0.160503,-0.323470,-0.046062,0.264607,0.151326,-0.055638,0.166325,0.253104,0.121721,-0.148745,-0.333983,0.186914,-0.271221,0.295006,-0.132622,-0.334782,-0.283189,-0.085
[...]
+0.011255,0.275860,0.068814,-0.262031,-0.055698,0.187403,0.140826,0.221797,0.008379,-0.225725,-0.229300,-0.148969,0.479475,-0.271663,0.391326,-0.162479,-0.049709,0.185512,0.232083,-0.063632,-0.077021,0.411851,-0.130684,-0.474202,0.313686,0.551695,-0.425808,-0.458760,0.423192,-0.004063,0.147665,-0.634370,0.290074,0.035611,0.474613,-0.438711,0.333034,0.260571,0.380325,-0.405890,0.033673,0.626554,0.229572,0.136523,0.023046,0.142549,0.115602,-0.106118,0.068938,-0.106399,-0.494274,-0.344458,0.
[...]
+0.057030,0.252734,0.235064,-0.047890,-0.037180,0.183786,-0.155754,0.141269,0.015503,-0.324622,-0.180887,-0.083984,0.677361,0.063952,-0.002800,-0.010200,0.176959,0.111628,-0.024678,-0.496320,0.165364,0.303463,-0.136559,-0.249473,0.565432,0.608163,-0.363315,-0.564718,0.140066,0.334405,-0.087456,-0.347387,-0.091418,0.138427,0.241973,-0.501234,-0.125040,0.521978,-0.098088,-0.077887,0.292481,0.499737,0.344788,0.291764,-0.184987,-0.285133,-0.103608,-0.169096,-0.017949,-0.345298,-0.404560,-0.34
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_query_test3.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_query_test3.csv
new file mode 100644
index 0000000000..97ad812c58
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_query_test3.csv
@@ -0,0 +1 @@
+-0.032375
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_query_test4.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_query_test4.csv
new file mode 100644
index 0000000000..f06633bb91
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_query_test4.csv
@@ -0,0 +1,2 @@
+-0.231513,-0.319015,-0.160277,-0.197611,-0.057491,-0.099871,-0.097508,-0.045165,-0.117926,-0.147653,0.750150,0.531435,-0.209658,0.395810,-0.000917,-0.584972,-0.133247,-0.493617,-0.128481,0.233111,0.032181,-0.230260,-0.485919,0.064414,-0.405997,0.017191,0.286678,-0.085608,-0.097799,-0.196052,0.822677,0.628560,0.354855,0.335803,0.321795,-0.296523,-0.285431,-0.145652,0.098115,0.464289,-0.174562,-0.059588,-0.087110,-0.159043,-0.238558,0.209597,-0.101130,-0.292678,-0.102115,-0.264466,0.294311
[...]
+-0.295776,-0.213294,-0.160206,-0.515858,-0.086320,0.000311,-0.057696,-0.028074,0.032395,0.085910,0.900377,0.419567,-0.623060,0.234508,0.235769,-0.162099,-0.295638,-0.789940,0.026089,0.364832,-0.443535,-0.240440,-0.102947,-0.087776,-0.337754,0.215590,0.184995,0.044527,-0.247155,-0.308647,0.736417,0.424900,-0.396127,-0.016338,-0.136535,-0.236500,0.146230,-0.046430,-0.387025,0.272419,0.165151,-0.006284,-0.363064,-0.175462,-0.180664,0.439337,-0.067702,-0.101911,-0.090186,-0.272128,0.577914,0
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_query_test5.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_query_test5.csv
new file mode 100644
index 0000000000..aecb46f357
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_query_test5.csv
@@ -0,0 +1,8 @@
+0.100289,0.157209,-0.634036,-0.409333,-0.498683,0.604410,0.143299,0.326166,-0.027580,-0.201876,0.226909,0.477064,-0.087034,0.138371,0.386537,0.392263,-0.171256,0.216197,-0.199727,-0.285217,0.445005,-0.198842,0.269268,-0.091627,0.215328,0.329377,-0.103686,-0.572041,-0.046001,0.086353,0.024692,0.427636,-0.166126,-0.389675,0.226858,-0.117578,0.124979,0.710012,-0.021382,0.103024,0.182377,-0.130566,-0.244070,0.709170,0.505710,-0.458052,0.014450,0.220508,-0.124254,-0.058869,0.162317,0.187010,-
[...]
+0.323568,0.256695,-0.577812,-0.147822,-0.546486,0.547864,0.080499,0.297201,-0.057487,-0.011173,-0.057384,0.378134,0.299202,-0.488600,-0.352016,0.580862,0.104533,0.144678,-0.218438,-0.268178,0.026484,-0.236303,-0.047902,-0.174082,0.184367,0.398705,-0.120185,-0.598090,-0.365643,0.250930,-0.089706,0.100883,0.101444,-0.151703,-0.238026,-0.672762,-0.108662,0.420527,-0.079838,0.269839,-0.020156,-0.126265,-0.538640,0.671084,0.212637,-0.408258,0.023278,0.051374,0.137253,0.013979,0.379628,0.24097
[...]
+-0.056568,0.146638,-0.456436,0.098395,-0.811011,0.495644,0.183795,0.230660,-0.194578,-0.061613,0.075415,0.360673,-0.204444,-0.164152,0.168364,0.539164,-0.384210,0.003881,-0.150976,-0.294538,0.125222,-0.536215,0.703297,0.252075,0.202324,0.482725,-0.361196,-0.840674,-0.321735,0.531849,0.356567,0.124709,-0.189984,-0.350024,-0.086076,-0.455081,0.075089,0.322418,-0.031815,0.034313,0.305768,0.070051,-0.008144,0.690341,0.479253,-0.136250,-0.197907,0.194114,0.155596,0.113514,0.125566,0.044274,-0
[...]
+0.141361,0.153819,-0.742168,-0.247818,-0.181050,0.376193,0.437072,0.292586,-0.228562,-0.228831,0.138952,0.476866,0.197689,-0.103904,0.041722,0.294463,-0.568225,0.137623,-0.119222,-0.159914,0.438089,0.036604,0.079649,-0.159947,0.334795,0.193879,-0.168633,-0.651697,-0.465226,0.518645,0.339801,0.183258,-0.404342,-0.374996,0.174889,-0.324887,0.022824,0.589189,-0.191233,0.307729,0.032350,-0.144538,-0.502723,0.586498,0.287868,-0.530440,-0.043154,0.394403,0.051082,0.263261,-0.309101,0.277042,-0
[...]
+-0.218197,0.188286,-0.434609,0.071313,-0.336148,0.078626,0.152097,0.268961,-0.256940,0.088283,0.016648,0.322682,0.268265,-0.123122,-0.039758,0.619544,-0.256389,0.469085,0.034972,0.036794,0.009958,-0.113469,0.185843,-0.051712,0.617038,0.140052,-0.167970,-0.705518,0.131559,0.242948,0.322949,0.023070,-0.127773,-0.092273,0.001035,-0.708963,0.412060,0.627792,-0.097335,0.216641,0.058617,-0.198305,-0.305766,1.072948,0.274709,-0.257714,-0.204414,0.199743,0.414865,-0.047754,0.082272,0.109740,-0.3
[...]
+0.112032,0.169193,-0.448280,-0.247962,-0.645115,0.586475,-0.036869,0.200395,0.012053,-0.039406,-0.223170,0.381503,0.262688,-0.198789,-0.344995,0.415228,-0.194531,0.195243,-0.073722,-0.030443,0.124897,-0.113547,-0.015239,0.133345,0.348098,0.300771,-0.053779,-0.878642,-0.242136,0.422017,-0.022946,0.197213,-0.152542,-0.089573,-0.347953,-0.091961,-0.045051,0.726564,0.083673,0.205618,-0.047137,0.033502,-0.334702,0.586090,0.397906,-0.485433,-0.056993,-0.156714,0.152364,0.057812,-0.144893,0.167
[...]
+-0.189369,0.266800,-0.604496,-0.329263,-0.592743,0.582927,0.356620,0.419440,0.269493,-0.029339,0.027025,0.234043,0.105808,-0.210952,-0.306262,0.687045,-0.198246,0.188302,-0.013503,-0.061036,0.268834,-0.170159,-0.044981,0.168009,0.222416,0.460862,-0.035309,-0.722267,0.013829,0.360681,-0.072319,0.144398,-0.083390,0.232737,0.033197,-0.208592,-0.040922,0.565264,0.110527,0.312133,-0.183011,-0.137255,-0.610982,0.611247,0.553975,-0.496809,-0.042657,-0.283822,0.151838,0.354914,0.085415,0.170378,
[...]
+0.206373,0.060220,-0.628899,-0.297584,-0.398253,0.488231,0.128032,0.088651,-0.022924,-0.216965,0.004021,0.421580,0.023492,-0.036508,-0.144941,0.732754,-0.110158,-0.080537,-0.119094,-0.341598,0.463299,-0.225139,0.225857,0.130873,0.373233,0.290363,-0.504426,-0.557271,0.023233,0.470722,0.502612,-0.001731,0.143594,0.010739,0.276409,-0.675856,0.196045,0.721700,0.233661,0.278742,-0.154628,-0.351704,-0.439502,0.627646,0.515803,-0.282315,-0.052633,0.111046,0.201188,0.334983,-0.144673,-0.008549,-
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_query_test6.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_query_test6.csv
new file mode 100644
index 0000000000..97059c0374
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_query_test6.csv
@@ -0,0 +1 @@
+-0.556831
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_value_test1.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_value_test1.csv
new file mode 100644
index 0000000000..676121c052
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_value_test1.csv
@@ -0,0 +1,2 @@
+0.011276,-0.552748,-0.272120,-0.659707,-0.099411,-0.136320,0.316806,-0.127630,0.160184,-0.385037,-0.058796,0.091652,0.156500,0.491553,0.230843,-0.098786,-0.164709,-0.125919,0.497394,-0.029966,-0.288149,-0.510635,0.029103,-0.573070,-0.028137,0.072260,0.184820,-0.319495,-0.104118,-0.130985,0.042002,-0.093236,0.367752,0.153147,-0.116999,0.248736,-0.186403,-0.031435,0.364887,-0.014942,-0.100243,-0.379393,-0.186525,-0.554986,0.216246,-0.292252,0.137599,0.133561,0.324600,-0.368008,0.480979,0.2
[...]
+-0.046973,-0.588374,-0.062508,-0.577904,-0.110970,-0.297613,0.028051,-0.053563,0.531156,-0.307522,0.038859,0.026366,0.142764,0.194628,0.230310,0.152338,-0.009142,-0.257903,0.306347,0.319039,-0.181963,-0.294963,-0.271088,-0.592370,-0.025137,0.034835,0.333417,0.127609,0.122721,-0.249855,0.437969,0.179052,0.091351,0.411968,-0.048785,-0.320129,-0.086639,0.068015,0.333704,0.177210,-0.575791,-0.477875,0.076523,-0.462514,0.060892,-0.197661,-0.141245,-0.115033,0.180807,-0.380116,0.461637,-0.3413
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_value_test2.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_value_test2.csv
new file mode 100644
index 0000000000..9ed8519a3a
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_value_test2.csv
@@ -0,0 +1,8 @@
+-0.135634,0.395091,0.097758,0.526070,-0.421527,-0.251836,0.186455,0.114455,-0.561665,0.737467,0.179376,-0.322035,0.143510,-0.068463,0.044905,-0.076750,0.029161,0.476064,-0.575137,-0.459609,0.619805,-0.308895,-0.303575,0.391219,-0.569634,-1.276704,-0.469837,-0.267725,-0.291944,-0.452748,0.240087,0.962458,0.191166,0.787170,0.013735,-0.455640,-0.135101,0.894998,0.218237,-0.368686,-0.357588,0.312785,-0.109378,0.891291,-0.117635,-0.183233,-0.012959,0.243157,-0.631415,0.414164,-0.129570,-0.316
[...]
+0.078108,0.162958,-0.251122,0.711622,-0.372453,-0.610192,-0.007698,0.018881,-0.759478,0.664228,0.022072,-0.023113,0.174427,-0.025556,0.070669,0.022558,0.080620,-0.045893,0.184032,-0.243000,0.653384,-0.014302,0.105771,0.304934,-0.364138,-1.130032,-0.500733,-0.445465,-0.174237,-0.208143,0.048790,0.489943,-0.019269,0.525283,-0.255809,-0.299515,0.036257,0.422030,-0.264013,-0.165797,0.095216,0.436818,-0.094952,0.376392,-0.401584,-0.380560,0.167055,0.361971,-0.836134,0.561108,0.343090,-0.11994
[...]
+-0.025370,0.319371,-0.115431,0.326305,-0.274020,-0.093748,0.149778,0.041573,-0.656989,0.347706,0.003869,-0.305735,0.118599,0.004471,-0.274985,0.054612,0.138950,0.149898,-0.269478,-0.375038,0.693099,-0.087722,0.022209,0.334126,-0.599416,-1.101336,-0.545966,-0.254866,-0.584182,-0.274468,0.101796,0.354398,-0.050078,0.465497,0.192123,-0.280255,-0.108706,0.998429,-0.085857,-0.252148,0.188999,0.341633,0.086733,0.894500,-0.200655,-0.448612,0.023567,0.019186,-0.245538,0.837295,-0.045620,0.094944
[...]
+-0.080297,0.390339,-0.123248,0.741724,-0.117765,-0.299310,-0.177795,0.253321,-0.737934,0.757596,-0.066708,-0.228726,0.066078,-0.202493,-0.094323,0.156148,0.366988,0.181063,0.042920,-0.448596,0.942105,0.158615,0.069077,0.305982,-0.266542,-0.804868,-0.425277,-0.281730,-0.233752,-0.653566,-0.080710,0.526888,0.173545,0.661713,-0.152202,-0.146090,0.053306,0.707427,-0.133131,-0.220142,0.205546,0.430469,0.106566,0.626826,-0.494951,-0.199954,-0.082341,0.165784,-0.461486,0.825365,0.119251,0.01627
[...]
+0.010772,0.337803,-0.114442,0.663496,0.041111,-0.350102,-0.174868,0.172210,-0.658491,0.229200,0.074391,0.021116,0.060559,-0.117566,-0.163536,-0.214915,0.066695,0.161989,-0.372148,-0.385813,0.733591,-0.094886,-0.117303,0.187929,-0.393576,-0.724917,-0.204588,-0.598760,-0.382334,-0.450012,-0.238686,0.870645,0.110469,0.494938,-0.156400,-0.350214,-0.052662,0.702509,-0.350709,-0.026910,0.123830,0.173322,-0.301884,0.775039,-0.475002,-0.393831,0.335230,0.073859,-0.428414,0.603097,0.272492,-0.262
[...]
+-0.549863,0.324430,-0.135728,1.036251,-0.075791,-0.589072,-0.199286,0.247405,-0.724970,0.338085,-0.109990,-0.017104,0.123346,0.029339,-0.238248,0.037839,0.157283,0.210315,-0.243766,-0.253850,0.813383,-0.531541,-0.097301,0.239048,-0.556164,-0.952322,-0.352101,-0.575240,-0.159435,-0.604663,0.043599,0.613543,0.073555,0.678998,-0.355514,-0.356214,-0.100680,0.491271,-0.141305,-0.348182,0.088086,0.301198,-0.100547,0.847432,-0.164916,-0.594568,0.024999,0.118720,-0.493290,0.871598,0.136800,0.178
[...]
+-0.000307,0.192405,0.009016,0.632378,-0.346205,-0.430933,-0.068020,-0.209620,-0.389118,0.798441,0.209632,-0.084138,0.193417,0.094311,-0.295927,0.109428,0.169248,0.066694,-0.435969,-0.222525,0.693382,-0.240283,-0.104285,0.608771,-0.255588,-0.876679,-0.356255,-0.171773,-0.291267,-0.148319,0.174362,0.640677,-0.327771,0.870963,-0.187884,-0.074322,-0.191532,0.795270,-0.203928,-0.132377,-0.237735,0.092001,0.022784,0.448034,-0.051981,-0.357628,-0.262221,-0.004712,-0.373413,0.643233,-0.042012,-0
[...]
+-0.129056,0.559424,0.096115,0.336901,-0.137857,-0.087742,0.358165,0.058574,-0.347116,0.730915,-0.119900,-0.395211,0.368959,-0.285447,-0.033938,-0.050849,0.399164,0.183781,-0.617081,-0.170652,0.465145,-0.205737,-0.380156,-0.054664,-0.392929,-0.512695,-0.422145,0.066632,-0.097875,-0.463818,0.201922,0.560717,-0.128508,0.718153,-0.191340,0.006540,-0.087180,0.880595,-0.192883,-0.285612,-0.377457,0.132552,-0.163005,0.535880,-0.235450,-0.341994,-0.148456,0.135440,-0.438536,0.442378,0.076118,-0.
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_value_test3.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_value_test3.csv
new file mode 100644
index 0000000000..3e7c2dd3e3
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_value_test3.csv
@@ -0,0 +1 @@
+0.230546
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_value_test4.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_value_test4.csv
new file mode 100644
index 0000000000..c2c9069e5d
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_value_test4.csv
@@ -0,0 +1,2 @@
+0.384034,-0.056709,-0.331526,-0.054465,0.458240,-0.154681,-0.079055,-0.745441,-0.820306,-0.061527,-0.458524,0.032031,-0.246142,0.281802,-0.119728,0.753474,-0.166423,-0.189239,-0.309784,0.154991,0.355368,-0.221648,-0.605073,-0.143109,0.388979,0.419089,0.096288,-0.705687,-0.658933,0.067154,-0.361009,-0.131988,-0.516667,0.247480,-0.199363,0.552584,-0.126541,0.201819,-0.371045,0.649130,0.584802,0.322901,-0.284859,0.066758,0.262611,0.085567,0.185549,-0.854169,-0.646970,-0.213714,-0.552828,0.5
[...]
+0.172056,0.059040,-0.542174,-0.075194,0.432492,0.190219,0.035747,-0.794741,-0.693341,-0.293452,-0.703951,0.075939,-0.919563,0.104236,-0.051853,0.372015,-0.393849,0.061932,-0.391686,0.792622,0.905552,0.017751,-0.235806,-0.333676,0.579890,0.100798,0.079576,-0.506601,-0.622649,-0.046112,-0.538571,0.522942,-0.129633,0.228910,0.290787,0.672020,0.077324,-0.125502,-0.131848,0.470734,0.341702,0.045942,-0.466897,-0.044115,0.171381,0.363961,0.075483,-0.569737,-0.383889,0.068497,-0.758123,0.227424,
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_value_test5.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_value_test5.csv
new file mode 100644
index 0000000000..47b11b4719
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_value_test5.csv
@@ -0,0 +1,8 @@
+-0.088618,-0.022915,-0.216855,-0.490821,-0.579018,0.340748,-0.598873,-0.500665,0.248918,-0.081467,-0.429255,0.241143,-0.273883,0.100782,0.128625,0.412199,0.548947,0.168003,-0.388547,-0.337886,0.043387,-0.386006,0.605764,0.913057,-0.097860,-0.324240,0.390423,0.401062,0.291816,0.304456,-0.289990,0.129255,-0.094417,0.275203,-0.675631,0.101792,0.475281,0.335321,0.117715,-0.462700,0.396528,0.433522,0.231560,0.107998,0.123914,0.046977,0.619617,0.093064,0.161105,0.206365,-0.356191,-0.092472,-0.
[...]
+-0.014716,0.121567,0.000622,-0.567369,-0.494609,0.494328,-0.663452,-0.551078,0.165779,-0.088615,-0.213571,0.136590,-0.012980,0.171990,-0.041373,0.517517,0.041417,-0.218444,-0.544386,0.326742,0.198479,0.004079,0.753438,0.857703,0.233363,-0.462650,0.397963,0.135466,0.207858,0.305657,-0.534061,-0.024569,0.202648,0.176015,-0.222765,-0.127068,0.524882,0.303806,0.316589,-0.570421,0.930915,0.225293,0.225618,0.118722,0.325689,0.437658,0.568868,0.292854,0.029267,0.298885,0.047729,-0.053590,-0.125
[...]
+-0.103796,-0.143883,-0.324740,-0.561045,-0.303866,0.303233,-0.701367,-0.437451,0.269724,-0.510573,-0.386954,0.440675,-0.248098,-0.084068,0.240447,0.099918,0.116392,0.141374,-0.484924,-0.270041,-0.128603,-0.370684,0.325419,1.048593,-0.149842,-0.566767,0.666990,0.108181,0.325534,0.311576,-0.271763,0.109930,-0.200035,0.369148,-0.365245,0.260469,0.319300,0.430350,0.089621,-0.432142,-0.156550,0.260484,-0.063642,0.060000,0.008476,-0.180151,0.833472,0.091514,0.505233,0.553627,-0.045237,-0.24145
[...]
+-0.200333,-0.354961,0.018583,-0.370726,-0.508255,0.433255,-0.535185,-0.382434,0.547405,-0.028783,-0.358815,0.298057,-0.315634,0.079815,-0.052657,0.504144,0.409413,0.067261,-0.601677,-0.158278,0.299734,-0.209871,0.415208,0.924856,0.164519,-0.416399,0.366263,0.084420,-0.079826,0.457049,-0.152549,0.358454,0.091155,0.335514,-0.291334,-0.058481,0.451879,0.309951,0.152147,-0.568359,0.749941,0.312131,0.207021,0.145143,0.104716,0.248173,0.701566,0.169483,0.223724,0.399754,-0.053367,0.266221,-0.0
[...]
+-0.058856,-0.174626,-0.252491,-0.395074,-0.591166,0.270451,-0.650871,-0.258918,0.442786,0.236918,-0.241567,0.844200,-0.399258,-0.300788,0.164386,0.536262,-0.279908,-0.067669,-0.558193,-0.004611,0.516622,0.169438,0.508988,0.811160,0.237769,-0.420372,0.518568,-0.286395,0.193257,0.269890,-0.360318,0.026795,-0.109402,0.074632,-0.179709,0.086965,1.008203,0.303905,0.168207,-0.541278,0.381255,0.248995,0.504799,-0.033077,0.404895,0.244843,0.749931,0.160679,0.159191,0.380788,-0.278933,-0.102407,0
[...]
+-0.197055,0.266835,-0.175943,-0.728479,-0.686868,0.361784,-0.968372,-0.341541,0.244648,-0.206282,-0.346421,0.262157,-0.101743,0.137205,0.192005,0.469633,0.148433,-0.195591,-0.397331,-0.250713,0.008861,-0.152797,0.486452,1.109617,0.248555,-0.530606,0.351657,0.145666,0.232467,0.258258,-0.532773,-0.257704,-0.193843,0.187249,-0.267056,0.057735,0.457290,0.421950,0.467891,-0.262763,0.254099,0.269831,-0.008575,0.031559,0.470209,-0.017451,0.478213,0.118906,0.162772,0.600625,-0.245088,0.106724,-0
[...]
+-0.085404,0.076665,-0.181502,-0.351590,-0.592550,0.379920,-0.529987,-0.604261,0.347072,-0.114188,-0.136212,0.103060,-0.330623,0.066298,-0.150162,0.227120,-0.086564,-0.205914,-0.418131,0.226653,-0.199312,0.055846,0.341430,0.989221,-0.193608,-0.337517,0.168879,0.151483,0.117915,0.298820,-0.554979,-0.366394,-0.104127,0.043303,-0.220064,-0.016449,0.505868,0.624081,-0.050807,-0.351606,0.386973,-0.104330,0.502353,-0.021639,0.468449,0.148428,0.710148,0.203542,0.084209,0.275661,-0.074653,0.11241
[...]
+-0.050682,0.071006,0.046257,-0.454838,-0.647404,0.479335,-0.618808,-0.416270,0.405504,-0.033245,-0.412591,0.217811,-0.293238,-0.186886,-0.001283,0.470888,0.222367,-0.315006,-0.426865,0.091794,0.036866,-0.099196,0.272333,1.289050,0.010082,-0.351729,0.478892,0.081570,0.255434,0.315800,-0.500878,-0.211055,-0.279083,0.383679,-0.321268,-0.005065,0.177351,0.659531,0.191144,-0.322980,0.379451,0.181151,0.388601,0.037714,0.463963,0.161004,0.589930,0.148004,0.055788,0.256598,0.231272,0.177599,0.00
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/input_value_test6.csv
b/src/test/resources/component/transformers/multi_attention_layer/input_value_test6.csv
new file mode 100644
index 0000000000..696dab6cef
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/input_value_test6.csv
@@ -0,0 +1 @@
+0.113262
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/output_attention_test1.csv
b/src/test/resources/component/transformers/multi_attention_layer/output_attention_test1.csv
new file mode 100644
index 0000000000..51df989adf
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/output_attention_test1.csv
@@ -0,0 +1,2 @@
+0.353462,0.318173,0.328366,0.350806,0.342539,0.306654,0.347301,0.337004,0.315695,0.337310,0.349396,0.313294,0.346549,0.327768,0.325683,0.353884,0.322359,0.323757,0.333561,0.341234,0.325205,0.340909,0.337401,0.321691,0.333446,0.343618,0.322936,0.278994,0.334577,0.386429,0.287115,0.331962,0.380923,0.321241,0.346239,0.332520
+0.323153,0.332474,0.344373,0.321667,0.340601,0.337732,0.327312,0.324805,0.347883,0.354234,0.281031,0.364735,0.346476,0.278632,0.374893,0.350848,0.297890,0.351263,0.324791,0.319696,0.355514,0.322111,0.313846,0.364043,0.324223,0.317882,0.357895,0.320102,0.343558,0.336340,0.322254,0.334098,0.343648,0.319757,0.344204,0.336039
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/output_attention_test2.csv
b/src/test/resources/component/transformers/multi_attention_layer/output_attention_test2.csv
new file mode 100644
index 0000000000..7bff61bfe8
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/output_attention_test2.csv
@@ -0,0 +1,8 @@
+0.083841,0.074373,0.080920,0.079670,0.090879,0.076543,0.082114,0.090295,0.083985,0.087113,0.083567,0.086700,0.085299,0.079368,0.081937,0.084231,0.082337,0.084137,0.082039,0.083190,0.084278,0.084048,0.085181,0.083955,0.082384,0.077390,0.082749,0.081534,0.090238,0.077156,0.079749,0.090434,0.083704,0.085035,0.083701,0.085926,0.083826,0.081081,0.082493,0.083387,0.083547,0.080653,0.083300,0.084649,0.079325,0.085730,0.083921,0.088087,0.081333,0.076022,0.081814,0.076761,0.095851,0.073503,0.0835
[...]
+0.085130,0.089431,0.086248,0.077305,0.083493,0.078305,0.080515,0.083471,0.086863,0.082051,0.079671,0.087517,0.083883,0.088983,0.085838,0.079338,0.085894,0.081281,0.078719,0.084731,0.084290,0.081615,0.080285,0.085144,0.083983,0.087440,0.085162,0.079404,0.085559,0.082266,0.080193,0.083451,0.085102,0.081740,0.081282,0.084419,0.081848,0.085490,0.082858,0.080672,0.082493,0.084799,0.082792,0.087415,0.082338,0.084722,0.080238,0.084334,0.082344,0.086906,0.084442,0.083849,0.082723,0.080873,0.0801
[...]
+0.078977,0.083177,0.078889,0.088275,0.081026,0.081921,0.085741,0.090992,0.083295,0.085511,0.080120,0.082076,0.075769,0.080881,0.077963,0.093165,0.084345,0.081275,0.085601,0.092472,0.085313,0.079371,0.083432,0.080414,0.083019,0.082735,0.078583,0.088134,0.080590,0.081198,0.085737,0.088326,0.080793,0.088854,0.079850,0.082181,0.078646,0.080738,0.078396,0.092451,0.084822,0.080982,0.085405,0.089552,0.083717,0.080764,0.083688,0.080840,0.078986,0.082077,0.081402,0.087211,0.082785,0.083140,0.0837
[...]
+0.083158,0.089498,0.083142,0.075967,0.087108,0.081898,0.083071,0.079302,0.080557,0.083814,0.084232,0.088252,0.078913,0.089219,0.083405,0.076416,0.091002,0.080797,0.081417,0.080305,0.081906,0.085701,0.081542,0.089378,0.081605,0.089221,0.082313,0.077277,0.089741,0.083428,0.083111,0.077866,0.082544,0.083486,0.081244,0.088163,0.079608,0.087714,0.081244,0.077135,0.084727,0.080941,0.085093,0.082110,0.083116,0.086973,0.083421,0.087918,0.075858,0.088172,0.087574,0.076565,0.095384,0.076887,0.0760
[...]
+0.079458,0.082332,0.080184,0.082172,0.081048,0.087107,0.085128,0.081258,0.084013,0.087616,0.085330,0.084356,0.076647,0.085526,0.073704,0.084993,0.081725,0.085588,0.084885,0.085636,0.085700,0.088685,0.077290,0.089621,0.079071,0.083405,0.077115,0.084149,0.082140,0.085856,0.085269,0.084568,0.084972,0.086573,0.080337,0.086545,0.076238,0.085352,0.073437,0.084811,0.081436,0.086063,0.085112,0.085310,0.085758,0.089201,0.077607,0.089676,0.077700,0.081744,0.074938,0.085376,0.081453,0.087761,0.0898
[...]
+0.076504,0.086446,0.084030,0.083058,0.082306,0.084128,0.090141,0.080912,0.080880,0.081112,0.087283,0.083201,0.080968,0.085696,0.081541,0.083700,0.083937,0.081414,0.086599,0.083225,0.082766,0.082205,0.086315,0.081635,0.084364,0.083010,0.083597,0.082813,0.082840,0.083137,0.083921,0.083650,0.083246,0.082195,0.083871,0.083354,0.074856,0.085190,0.085676,0.084366,0.082692,0.086710,0.091117,0.078892,0.083029,0.082841,0.083610,0.081021,0.081116,0.084572,0.082512,0.084193,0.084008,0.082657,0.0867
[...]
+0.084102,0.079811,0.082757,0.080099,0.084518,0.086548,0.086055,0.082666,0.082639,0.085251,0.083691,0.081863,0.087486,0.082855,0.083006,0.080533,0.082478,0.083851,0.085205,0.085238,0.081095,0.085028,0.076185,0.087039,0.086114,0.080020,0.083595,0.084169,0.082222,0.077104,0.083333,0.087846,0.084344,0.079918,0.081258,0.090077,0.086158,0.085378,0.083493,0.080332,0.081871,0.085965,0.084961,0.083144,0.081075,0.086075,0.075832,0.085717,0.084987,0.077446,0.083625,0.081010,0.083184,0.081357,0.0860
[...]
+0.086554,0.079359,0.084680,0.083244,0.082488,0.083686,0.078468,0.084522,0.085980,0.084714,0.079623,0.086684,0.090152,0.075387,0.086759,0.083930,0.080337,0.084653,0.070845,0.086358,0.090515,0.086472,0.073479,0.091114,0.085204,0.080868,0.086596,0.084767,0.082487,0.083400,0.075964,0.085853,0.084616,0.085677,0.078767,0.085801,0.086393,0.082768,0.084494,0.083023,0.085759,0.083360,0.078567,0.086114,0.084334,0.083721,0.076705,0.084762,0.089186,0.078965,0.087891,0.084667,0.083148,0.084116,0.0694
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/output_attention_test3.csv
b/src/test/resources/component/transformers/multi_attention_layer/output_attention_test3.csv
new file mode 100644
index 0000000000..961b372472
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/output_attention_test3.csv
@@ -0,0 +1 @@
+1.000000
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/output_context_test1.csv
b/src/test/resources/component/transformers/multi_attention_layer/output_context_test1.csv
new file mode 100644
index 0000000000..609874922a
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/output_context_test1.csv
@@ -0,0 +1,2 @@
+-0.120612,-0.482425,-0.148172,-0.597755,0.026917,-0.112296,0.214546,-0.112837,0.119348,-0.290938,0.151137,0.070833,0.219496,0.288513,0.070071,0.104854,-0.162399,-0.141325,0.398671,0.128460,-0.125487,-0.485163,-0.142691,-0.597917,0.021800,-0.118739,0.215180,-0.105452,0.127102,-0.296221,0.148854,0.071085,0.218784,0.290757,0.071859,0.102700,-0.162478,-0.140903,0.399793,0.126235,-0.124837,-0.483829,-0.143585,-0.597450,0.024260,-0.119567,0.216239,-0.104917,0.128214,-0.297628,0.150153,0.070098
[...]
+-0.273965,-0.452769,-0.083977,-0.542977,-0.023248,-0.167729,0.052120,-0.025068,0.288588,-0.317793,0.316756,-0.055552,0.163079,0.184409,0.016838,0.080253,-0.169572,0.028544,0.385273,0.240573,-0.271550,-0.451119,-0.086596,-0.543860,-0.023692,-0.167511,0.049668,-0.026127,0.286010,-0.318669,0.318027,-0.059582,0.164261,0.181225,0.015490,0.086683,-0.171746,0.029252,0.386524,0.241269,-0.274786,-0.454632,-0.081890,-0.542461,-0.023303,-0.163471,0.059549,-0.021186,0.286423,-0.315843,0.317039,-0.05
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/output_context_test2.csv
b/src/test/resources/component/transformers/multi_attention_layer/output_context_test2.csv
new file mode 100644
index 0000000000..6011bb0cb5
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/output_context_test2.csv
@@ -0,0 +1,8 @@
+-0.117522,0.346017,-0.053089,0.761250,-0.206048,-0.284776,0.097453,0.093608,-0.521281,0.631762,0.056340,-0.200643,0.252619,-0.023594,-0.053775,-0.026848,0.231548,0.277189,-0.307811,-0.290577,0.647949,-0.240369,-0.227517,0.341885,-0.383581,-0.906379,-0.380905,-0.231047,-0.247861,-0.371709,0.129811,0.602819,0.092210,0.714391,-0.173514,-0.163728,-0.085955,0.685285,-0.103525,-0.237930,-0.120528,0.346917,-0.049508,0.764789,-0.206029,-0.284187,0.097638,0.093441,-0.521630,0.631315,0.058548,-0.2
[...]
+-0.035569,0.326823,-0.038932,0.676020,-0.223588,-0.452070,0.041762,0.071362,-0.609897,0.548520,0.120448,-0.143065,0.226641,-0.001276,-0.066680,0.036676,0.200332,0.121154,-0.162313,-0.281330,0.720779,-0.165327,-0.079952,0.327600,-0.451796,-0.931752,-0.287571,-0.436174,-0.228873,-0.415194,0.100709,0.605773,0.096248,0.704005,-0.177976,-0.131317,-0.114539,0.635780,-0.251185,-0.249996,-0.032713,0.328270,-0.038006,0.677002,-0.225219,-0.449986,0.041847,0.073082,-0.610213,0.550519,0.122624,-0.14
[...]
+-0.006288,0.380614,-0.026938,0.635265,-0.212711,-0.331305,0.039212,0.145355,-0.546321,0.618505,0.080980,-0.200142,0.194609,0.034915,-0.108994,-0.021783,0.163647,0.135571,-0.300602,-0.267585,0.735781,-0.110017,-0.097210,0.307721,-0.350206,-0.867862,-0.296070,-0.248226,-0.335913,-0.367380,0.094519,0.605687,0.074367,0.593416,-0.112793,-0.072601,-0.018687,0.701815,-0.145334,-0.218955,-0.008061,0.380253,-0.025457,0.634619,-0.212584,-0.335690,0.038575,0.142550,-0.547131,0.616575,0.082240,-0.20
[...]
+-0.002710,0.364326,0.002412,0.629897,-0.391738,-0.262783,0.102495,0.218040,-0.607559,0.657610,0.064115,-0.124938,0.222093,-0.012397,-0.055040,-0.040374,0.083058,0.199003,-0.231693,-0.271518,0.751965,-0.121225,-0.141271,0.326794,-0.390842,-0.948052,-0.308465,-0.266784,-0.280107,-0.374203,0.091308,0.566526,0.018774,0.721443,-0.215387,-0.176024,-0.078039,0.727437,-0.184399,-0.267821,-0.002896,0.364105,0.000775,0.630106,-0.390812,-0.266293,0.103452,0.212035,-0.607041,0.656150,0.064191,-0.124
[...]
+-0.037909,0.354449,-0.201810,0.637210,-0.188037,-0.371674,0.088127,-0.001254,-0.600008,0.604595,0.091000,-0.094501,0.154672,-0.034591,-0.049681,0.002476,0.146760,0.091180,-0.238992,-0.376675,0.691954,-0.166721,-0.149841,0.181483,-0.467594,-0.874755,-0.295117,-0.356832,-0.299043,-0.314829,0.067757,0.502483,0.027922,0.596780,-0.133525,-0.094843,-0.057475,0.692787,-0.224112,-0.194505,-0.039155,0.352347,-0.206806,0.638253,-0.186814,-0.372500,0.086910,-0.002181,-0.599988,0.603909,0.092506,-0.
[...]
+-0.083416,0.337176,-0.057880,0.698608,-0.192535,-0.398294,0.048559,0.129080,-0.622012,0.654912,0.050051,-0.125803,0.255794,-0.099282,-0.073189,-0.015607,0.181704,0.066302,-0.224570,-0.331452,0.692483,-0.122864,-0.156132,0.266749,-0.394466,-0.910722,-0.314547,-0.334330,-0.315752,-0.394909,0.062578,0.593709,0.071017,0.661427,-0.197191,-0.124251,-0.046466,0.680112,-0.207248,-0.266378,-0.086126,0.336373,-0.060861,0.701026,-0.191042,-0.393963,0.053677,0.134100,-0.622058,0.653063,0.050271,-0.1
[...]
+-0.025028,0.327439,-0.070065,0.674529,-0.185441,-0.430599,-0.038825,0.006075,-0.531446,0.648116,0.160118,-0.129134,0.254268,-0.003386,-0.083576,0.042160,0.206057,0.026941,-0.231182,-0.207756,0.698347,-0.145440,-0.036587,0.347156,-0.324124,-0.912421,-0.286870,-0.310662,-0.265338,-0.287285,0.128922,0.631112,0.010615,0.688741,-0.132570,-0.062363,-0.038493,0.685972,-0.181124,-0.114414,-0.024076,0.329008,-0.069314,0.673601,-0.183589,-0.429853,-0.039391,0.006465,-0.531163,0.647133,0.158826,-0.
[...]
+-0.114102,0.368207,-0.094774,0.605423,-0.187877,-0.329160,0.065250,0.122708,-0.562487,0.637158,0.070056,-0.172297,0.225996,-0.086428,0.024146,0.029847,0.183165,0.085856,-0.317712,-0.317512,0.643462,-0.109325,-0.188870,0.219334,-0.409110,-0.901520,-0.312213,-0.271973,-0.315158,-0.361707,0.105536,0.659183,0.087846,0.688896,-0.189298,-0.173866,-0.043636,0.746005,-0.066002,-0.234959,-0.107707,0.373512,-0.096007,0.607465,-0.186081,-0.331838,0.062410,0.120504,-0.562790,0.638671,0.071028,-0.174
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/output_context_test3.csv
b/src/test/resources/component/transformers/multi_attention_layer/output_context_test3.csv
new file mode 100644
index 0000000000..3e7c2dd3e3
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/output_context_test3.csv
@@ -0,0 +1 @@
+0.230546
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/output_dkey_test4.csv
b/src/test/resources/component/transformers/multi_attention_layer/output_dkey_test4.csv
new file mode 100644
index 0000000000..7d9c46a4c2
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/output_dkey_test4.csv
@@ -0,0 +1,2 @@
+-0.001018,-0.002263,-0.001537,-0.000790,-0.000369,-0.005039,-0.005443,0.019228,0.015178,0.028658,-0.007141,-0.001750,-0.002432,-0.004250,-0.000032,0.032353,0.007619,0.019751,0.001080,-0.023244,0.011265,0.017624,0.018430,0.009013,0.016444,0.002116,0.001227,-0.017631,-0.015635,-0.027768,-0.112671,-0.099088,0.019606,-0.051302,-0.018480,-0.020174,-0.003826,-0.011986,-0.000758,0.014061,-0.010247,-0.015361,-0.016893,-0.008223,-0.016075,0.002923,0.004216,-0.001596,0.000457,-0.000890,0.119812,0.
[...]
+0.003402,0.006878,0.020028,0.016469,0.014312,-0.029537,-0.004940,0.002817,0.015430,0.024891,-0.226789,-0.132868,0.141100,-0.024155,-0.008391,0.009181,0.006551,0.002272,0.003326,0.000049,-0.010743,-0.017197,-0.045410,-0.039257,-0.033365,-0.005049,0.003006,0.002156,-0.000896,0.001076,0.239040,0.141009,-0.150246,0.027488,0.012609,-0.019229,-0.015992,-0.018808,-0.006434,0.007734,0.007341,0.010319,0.025383,0.022788,0.019053,0.034586,0.001934,-0.004974,-0.014535,-0.025967,-0.012251,-0.008141,0
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/output_dkey_test5.csv
b/src/test/resources/component/transformers/multi_attention_layer/output_dkey_test5.csv
new file mode 100644
index 0000000000..7a106d9d65
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/output_dkey_test5.csv
@@ -0,0 +1,8 @@
+0.000589,0.001687,-0.005652,-0.001729,-0.004408,-0.034226,-0.015826,-0.021034,-0.001984,0.005084,-0.000310,0.006020,-0.000209,0.000152,0.000892,0.077882,-0.042954,0.039442,-0.012927,-0.004839,-0.034768,0.018395,-0.027467,0.000691,-0.044428,0.040996,-0.022466,-0.084329,-0.018285,0.041054,-0.012361,-0.004840,0.002285,0.011921,-0.007189,-0.007375,0.001036,0.009486,0.001024,0.001584,-0.000403,-0.000178,-0.000883,0.001018,-0.001244,0.007838,0.000966,-0.000804,-0.003617,-0.003961,-0.003672,-0.
[...]
+0.002286,0.007834,-0.028894,-0.013625,-0.025057,-0.026740,-0.013304,-0.013182,0.001852,0.000590,-0.003442,0.005736,0.004394,-0.010073,-0.002805,0.028406,-0.022964,0.011743,-0.003964,-0.003264,0.018895,-0.011302,0.015797,0.011577,0.022576,0.011607,-0.009991,-0.030205,-0.010116,0.013208,0.014460,0.005283,-0.004424,-0.006959,0.002009,0.007963,-0.002980,-0.011271,-0.003136,-0.006328,-0.003099,-0.022737,-0.054721,0.153786,0.080266,-0.049546,0.001143,0.006737,0.024459,0.019349,0.000018,0.00208
[...]
+-0.000879,0.002107,-0.005685,-0.003832,-0.006781,-0.075093,-0.036755,-0.037164,0.004171,0.006491,-0.000964,0.025813,0.006221,-0.007538,-0.000002,-0.027399,0.026389,-0.005194,0.007385,0.009953,-0.046333,0.049342,-0.063492,-0.012514,-0.056040,0.005707,-0.006851,-0.017075,-0.006861,0.009715,0.014506,0.008715,-0.011038,-0.010947,0.006022,-0.013033,0.005751,0.014944,0.003112,0.005187,0.000712,0.047224,0.052977,-0.145435,-0.079693,-0.044325,-0.005942,0.013303,0.023252,0.013967,0.002056,-0.0021
[...]
+-0.004202,-0.010976,0.024975,0.014261,0.017502,0.064084,0.035288,0.038085,0.000679,-0.005011,0.000301,-0.011977,-0.004401,0.000811,-0.000347,0.034582,-0.038666,0.012282,-0.009403,-0.016976,0.000683,-0.000048,-0.000007,0.000001,0.000990,-0.014976,0.001114,0.035352,0.015305,-0.022306,0.058852,0.027636,-0.016165,-0.045176,0.015660,0.030261,-0.006432,-0.033515,-0.002580,-0.013772,0.003452,-0.012572,-0.022761,0.045900,0.023330,-0.051820,-0.005161,0.013692,0.008380,0.007820,0.005571,0.025233,-
[...]
+-0.000447,-0.004293,0.012895,0.008041,0.010538,0.033837,0.015968,0.017024,-0.002825,-0.002835,-0.010314,0.040885,0.012445,-0.007502,-0.000720,-0.037305,0.021775,-0.016632,0.009074,0.009066,0.036259,-0.028948,0.031388,0.007285,0.046013,-0.011442,0.007467,0.026167,0.006208,-0.010123,0.008879,0.000039,0.001017,-0.001024,0.001617,-0.034274,0.011791,0.075400,0.014904,0.024475,-0.000308,-0.008301,-0.013301,0.032081,0.016215,-0.046691,-0.005195,0.009484,0.013904,0.011338,-0.001574,0.003798,-0.0
[...]
+0.007205,0.000863,0.043782,0.016088,0.035966,-0.050133,-0.014750,-0.025593,0.001591,-0.002659,-0.007735,0.036818,0.017196,-0.023674,-0.014671,-0.016753,0.005529,-0.007145,0.005464,0.006713,0.011387,-0.006210,0.008207,0.001829,0.018058,-0.001916,-0.000527,0.003773,0.001609,0.000662,-0.019538,-0.023361,0.007463,0.016245,-0.007084,-0.040125,0.012901,0.054147,0.017692,0.022269,-0.002092,0.005270,-0.003695,0.000439,0.004552,-0.010519,-0.000421,0.001655,0.004888,0.000742,0.010867,0.003440,0.05
[...]
+0.000359,0.001777,-0.012955,-0.003861,-0.009397,-0.029234,-0.007550,-0.013022,-0.002107,0.000929,-0.000671,-0.006591,0.001225,0.002396,0.000557,-0.010323,0.004577,-0.007544,0.004439,-0.002043,-0.029150,0.006361,-0.010413,-0.008908,-0.042722,-0.013926,0.006007,0.033245,0.007525,-0.016568,-0.032162,-0.027090,0.015286,0.012643,-0.017305,-0.019633,0.004545,0.023703,-0.000056,0.014845,-0.003416,-0.008816,-0.016339,0.020415,0.017084,-0.013275,0.000837,0.001466,0.001847,0.002603,0.004021,0.0038
[...]
+-0.002065,0.004625,-0.013207,-0.002641,-0.012520,0.036295,0.010569,0.014850,0.001747,-0.000274,0.003211,-0.028782,-0.008607,0.021078,0.016190,0.011249,-0.003140,0.001945,-0.001359,-0.005754,0.004788,0.000626,0.003769,0.000926,0.004686,0.011571,-0.013002,-0.035563,-0.004764,0.019310,-0.003804,-0.007054,0.009471,0.001345,-0.003613,-0.004339,0.000135,0.001934,0.001000,0.002387,-0.002701,-0.023459,-0.043888,0.089308,0.055799,0.005689,0.002064,-0.002540,-0.002758,-0.002412,-0.000620,0.022795,
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/output_dkey_test6.csv
b/src/test/resources/component/transformers/multi_attention_layer/output_dkey_test6.csv
new file mode 100644
index 0000000000..945da8ffd3
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/output_dkey_test6.csv
@@ -0,0 +1 @@
+0.000000
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/output_dquery_test4.csv
b/src/test/resources/component/transformers/multi_attention_layer/output_dquery_test4.csv
new file mode 100644
index 0000000000..378a26df56
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/output_dquery_test4.csv
@@ -0,0 +1,2 @@
+0.000201,0.000505,-0.007467,-0.004844,0.011120,-0.006519,-0.019773,0.001923,-0.003486,0.003292,-0.004859,0.018129,0.004885,-0.018692,0.008021,0.002604,-0.005234,0.008213,0.009642,-0.001014,-0.000569,0.000408,-0.005637,-0.003144,0.008439,-0.008544,-0.014589,0.007231,-0.009148,0.006319,-0.003820,0.011857,0.001307,-0.012307,0.005041,0.001975,-0.004566,0.005439,0.007015,-0.000251,-0.001599,0.000530,-0.006875,-0.003192,0.010347,-0.006998,-0.013245,0.005383,-0.006969,0.004946,-0.003447,0.00802
[...]
+-0.004949,-0.004668,-0.023522,-0.005275,0.010241,0.003053,-0.006956,0.000343,0.006355,-0.000262,0.037593,-0.054268,0.043138,0.007745,0.019099,0.000786,-0.000452,0.001118,-0.001414,0.002899,-0.005133,-0.004758,-0.023865,-0.005509,0.010434,0.015595,-0.015787,-0.002601,0.000531,-0.001245,0.025988,-0.050170,0.036971,0.006128,0.015535,0.000026,0.000110,-0.000681,-0.000887,-0.001179,-0.015058,-0.013103,-0.064596,-0.016552,0.028701,0.002975,-0.011085,0.001284,0.013158,-0.000276,0.029356,-0.0397
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/output_dquery_test5.csv
b/src/test/resources/component/transformers/multi_attention_layer/output_dquery_test5.csv
new file mode 100644
index 0000000000..44a55dd0f7
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/output_dquery_test5.csv
@@ -0,0 +1,8 @@
+-0.004042,0.000068,0.001548,0.002906,0.004703,0.009005,0.006394,-0.009592,-0.007054,0.009685,0.002861,-0.006411,-0.000592,0.003626,0.004077,0.003052,0.001049,0.010313,0.008772,0.007011,-0.003797,-0.004048,0.004087,0.004385,0.002522,-0.000808,-0.006090,-0.007088,-0.003014,0.003564,-0.006334,0.006244,-0.006902,0.004890,0.006270,0.000287,-0.011389,-0.004145,0.002667,-0.003084,-0.000029,-0.001147,-0.001391,0.007483,-0.000355,0.006680,-0.004530,-0.004310,-0.000867,0.000734,-0.008504,0.000056,
[...]
+-0.000444,0.007043,-0.005302,-0.001599,-0.001925,-0.002669,-0.000914,-0.001020,0.001072,0.004523,-0.007290,0.001671,-0.001050,0.002223,-0.001045,0.004996,-0.002157,0.003688,0.005818,0.004233,-0.005379,0.003647,-0.000255,0.000645,-0.005527,0.006176,-0.004943,-0.008021,0.010903,-0.005946,-0.002136,-0.000856,-0.000007,-0.000305,0.000162,-0.004094,0.007838,0.001398,0.000625,0.007707,0.005226,0.000848,0.000664,-0.007914,0.006686,-0.001870,-0.003334,0.005112,-0.001594,-0.004763,-0.003694,0.004
[...]
+0.000579,-0.004307,-0.001190,0.000158,-0.001276,0.008193,0.002331,0.006184,-0.005201,0.012584,-0.003489,0.002541,-0.000565,-0.002926,0.004887,-0.005546,-0.001275,0.006905,-0.003429,-0.005481,0.007811,-0.008118,-0.006240,0.007392,-0.004171,0.005409,-0.000012,0.000179,0.002007,0.005547,0.000573,0.000248,-0.003714,0.000912,0.003876,0.000657,-0.002056,0.001565,-0.000554,-0.004059,-0.000626,-0.005311,-0.001197,-0.003737,0.006209,0.007797,-0.007737,-0.001404,-0.003554,0.007112,0.001026,0.00120
[...]
+0.000366,0.006575,-0.004120,-0.001407,0.002920,0.004283,-0.002577,-0.003511,-0.006305,0.007249,-0.002803,0.002172,0.002697,0.000367,0.001925,0.000953,0.003401,0.003719,0.001896,0.003266,-0.006542,0.001068,-0.002800,0.009653,0.003407,0.003074,-0.014065,-0.003707,0.006159,0.011610,-0.003205,0.002216,-0.001630,-0.001387,-0.000649,-0.006115,0.003422,0.009036,-0.003667,-0.003743,-0.001783,0.003140,0.004518,-0.000248,-0.005011,-0.003936,0.000176,0.000956,0.002354,-0.010348,-0.000606,-0.001749,
[...]
+-0.000511,0.000667,0.000075,0.002532,0.003592,-0.002722,-0.000403,0.000636,0.001185,0.001462,-0.011467,-0.006976,-0.000137,0.003098,-0.008308,0.008114,-0.004456,0.003305,0.002001,-0.000126,-0.002098,-0.002946,-0.001911,0.000035,-0.000677,0.010294,-0.003372,0.006549,0.002494,-0.005681,-0.006093,0.009804,-0.015929,0.000718,-0.001009,0.003327,-0.000216,0.004264,-0.003875,0.006203,-0.004321,-0.000360,-0.000663,0.001856,0.001817,0.004882,-0.004702,0.005076,-0.003025,-0.007277,0.004374,-0.0055
[...]
+0.003496,0.003016,-0.005924,-0.001595,0.000176,0.004721,-0.003267,0.001443,-0.000840,-0.002002,0.002627,-0.000658,0.006248,-0.006617,-0.010774,0.001529,0.001275,0.001667,-0.000487,-0.000365,0.000012,-0.006159,-0.000877,0.001072,-0.003886,0.005262,-0.002441,-0.001338,0.001125,-0.000327,-0.002756,-0.001402,0.002157,-0.005870,-0.001266,-0.009948,-0.000072,0.009834,0.003075,0.002894,0.004261,0.005817,0.003299,-0.000182,0.004141,-0.001241,-0.002059,0.008380,0.000015,-0.005037,0.001733,0.00024
[...]
+-0.001923,0.001538,0.000227,-0.000828,-0.001155,0.003156,0.001034,-0.004837,-0.007219,0.004040,-0.000001,-0.005297,-0.000045,-0.002641,-0.000446,0.002504,-0.001080,0.006097,-0.003325,-0.002276,0.001632,-0.008880,-0.003959,0.000739,-0.002442,-0.004110,-0.004646,0.004497,-0.000211,-0.005072,-0.001030,-0.000052,-0.010913,-0.003652,-0.001410,-0.004967,0.002475,-0.002579,0.000019,-0.004622,0.001581,0.000980,-0.001799,-0.002225,0.003518,0.006824,-0.004553,0.005873,0.005783,-0.006091,0.000184,0
[...]
+-0.006890,0.001119,-0.000045,-0.005180,0.001289,0.006136,0.001648,-0.000273,0.002924,0.003307,-0.005309,-0.000108,0.009845,-0.008241,0.001765,0.002034,0.000377,0.001763,-0.004029,-0.001169,-0.001611,-0.004388,0.004474,0.003022,0.002089,0.003880,-0.000073,0.000513,-0.005847,0.000646,-0.006418,0.007321,-0.000583,0.000352,-0.004668,-0.002401,0.003734,-0.002282,0.006459,-0.008666,0.009354,-0.007023,0.002204,0.000429,0.007877,0.001912,-0.001522,-0.001647,-0.003137,0.000728,-0.003277,-0.001882
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/output_dquery_test6.csv
b/src/test/resources/component/transformers/multi_attention_layer/output_dquery_test6.csv
new file mode 100644
index 0000000000..945da8ffd3
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/output_dquery_test6.csv
@@ -0,0 +1 @@
+0.000000
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/output_dvalue_test4.csv
b/src/test/resources/component/transformers/multi_attention_layer/output_dvalue_test4.csv
new file mode 100644
index 0000000000..82f73343a2
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/output_dvalue_test4.csv
@@ -0,0 +1,2 @@
+0.678334,0.145936,0.266606,0.422730,0.517451,0.679531,0.735708,0.448040,0.384853,0.373705,0.377254,0.619514,0.819744,0.708109,0.669057,0.313635,0.646156,0.112248,0.353896,0.692356,0.681069,0.146333,0.268264,0.422764,0.519653,0.672980,0.719383,0.440785,0.373018,0.375306,0.392964,0.621045,0.823472,0.729659,0.692182,0.291679,0.590416,0.095830,0.308363,0.642616,0.681462,0.145864,0.269663,0.418404,0.522422,0.666626,0.711718,0.432003,0.373119,0.374246,0.395562,0.625471,0.836035,0.734640,0.6964
[...]
+0.480998,0.543887,0.386493,0.089028,0.574414,0.518602,0.575928,0.477795,0.754506,0.276333,0.794762,0.791339,0.932566,0.484396,0.438318,0.299130,0.272643,0.243338,0.738093,0.627241,0.533360,0.605214,0.431868,0.099245,0.639007,0.576070,0.642858,0.514715,0.837613,0.319024,0.740274,0.738208,0.869078,0.453217,0.411707,0.306216,0.276203,0.240512,0.734723,0.623029,0.501781,0.573375,0.410735,0.093045,0.604113,0.535734,0.586208,0.480280,0.767775,0.284497,0.684365,0.680968,0.803873,0.417757,0.3752
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/output_dvalue_test5.csv
b/src/test/resources/component/transformers/multi_attention_layer/output_dvalue_test5.csv
new file mode 100644
index 0000000000..e8f73e7644
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/output_dvalue_test5.csv
@@ -0,0 +1,8 @@
+0.489546,0.602237,0.600384,0.542103,0.384455,0.564575,0.405709,0.498108,0.525837,0.439241,0.594427,0.486798,0.598099,0.485575,0.488636,0.680661,0.595068,0.541757,0.498490,0.511882,0.521717,0.505539,0.636429,0.606702,0.519989,0.684245,0.660933,0.540435,0.619269,0.487860,0.463788,0.441880,0.427054,0.521775,0.626341,0.613173,0.412579,0.390034,0.444907,0.535385,0.470380,0.358163,0.500840,0.439551,0.339549,0.478323,0.425846,0.548023,0.421899,0.428920,0.539334,0.658660,0.660223,0.596877,0.4193
[...]
+0.422647,0.584568,0.538749,0.456432,0.446146,0.484458,0.450753,0.364749,0.452199,0.637368,0.537025,0.506547,0.487874,0.496665,0.570857,0.419667,0.566885,0.429099,0.432459,0.511695,0.466939,0.370312,0.478558,0.542463,0.583710,0.523997,0.449049,0.496690,0.691942,0.561235,0.460777,0.775672,0.554528,0.404096,0.430620,0.534400,0.426073,0.512157,0.598544,0.559555,0.603331,0.468003,0.681684,0.651818,0.422829,0.526235,0.596514,0.524766,0.481702,0.287822,0.444936,0.609147,0.560385,0.472377,0.4612
[...]
+0.495001,0.638178,0.523371,0.555292,0.480070,0.425783,0.290955,0.370908,0.456528,0.448157,0.511250,0.491739,0.604270,0.604761,0.480866,0.450348,0.556620,0.448400,0.391041,0.575122,0.560242,0.628516,0.548652,0.432348,0.520165,0.713901,0.421883,0.537972,0.508513,0.564109,0.589849,0.501467,0.435937,0.476449,0.541387,0.488868,0.430654,0.464136,0.564238,0.466150,0.493161,0.462334,0.519127,0.560541,0.421819,0.530675,0.442529,0.500205,0.538452,0.460254,0.468737,0.604597,0.500027,0.524924,0.4595
[...]
+0.344415,0.508217,0.430220,0.217548,0.374968,0.519287,0.569978,0.399657,0.461395,0.482120,0.537995,0.371053,0.566650,0.426186,0.368349,0.581705,0.477617,0.436927,0.485898,0.489258,0.645816,0.535179,0.532058,0.512644,0.503370,0.481449,0.591718,0.632651,0.607965,0.533137,0.586027,0.363173,0.503529,0.512710,0.581795,0.668873,0.693587,0.504023,0.480352,0.626736,0.357351,0.448472,0.383988,0.428113,0.515132,0.463287,0.498190,0.563239,0.481619,0.546711,0.358953,0.524036,0.439619,0.225896,0.3817
[...]
+0.426790,0.622763,0.549726,0.541441,0.559304,0.580575,0.468956,0.492094,0.390449,0.548385,0.598981,0.543657,0.452907,0.378462,0.508859,0.415797,0.495754,0.577147,0.527113,0.414861,0.497465,0.414849,0.427954,0.434620,0.471401,0.540354,0.467981,0.557656,0.331712,0.561765,0.497974,0.538275,0.611576,0.532986,0.472014,0.609107,0.552978,0.482931,0.518352,0.425411,0.558490,0.411199,0.437959,0.524134,0.499303,0.413687,0.475210,0.469987,0.613968,0.551682,0.388035,0.567282,0.511504,0.494930,0.5126
[...]
+0.437850,0.564703,0.529215,0.541465,0.625983,0.391904,0.533930,0.413493,0.486141,0.418267,0.548298,0.594630,0.560234,0.697742,0.616674,0.479541,0.404317,0.360240,0.564868,0.523023,0.413474,0.443025,0.598711,0.496449,0.577375,0.455003,0.537353,0.450266,0.403968,0.493887,0.510115,0.327276,0.551285,0.391849,0.529840,0.528622,0.553604,0.450265,0.522037,0.400191,0.402739,0.494758,0.436586,0.491841,0.355803,0.520302,0.538612,0.572486,0.515539,0.596130,0.400743,0.515075,0.488290,0.496818,0.5752
[...]
+0.473827,0.495039,0.450332,0.532157,0.536166,0.502557,0.461272,0.543496,0.447897,0.626269,0.566522,0.376099,0.389453,0.435868,0.504174,0.447618,0.495197,0.711383,0.429322,0.603777,0.499038,0.458111,0.345123,0.467357,0.593158,0.546612,0.477041,0.683334,0.575160,0.303047,0.458825,0.574088,0.459515,0.437025,0.536195,0.611456,0.596735,0.553600,0.527185,0.543907,0.480580,0.631948,0.488325,0.513947,0.629324,0.513830,0.372935,0.435416,0.552112,0.636293,0.533584,0.558395,0.518499,0.601861,0.6035
[...]
+0.374207,0.403949,0.526343,0.244185,0.528193,0.555913,0.496328,0.399437,0.351462,0.603851,0.449138,0.510296,0.505564,0.646414,0.452255,0.454127,0.401514,0.473468,0.483788,0.513998,0.554965,0.540235,0.523459,0.644116,0.644508,0.443629,0.585785,0.557221,0.538836,0.475847,0.481348,0.532752,0.520312,0.564166,0.509192,0.453757,0.474740,0.403492,0.507998,0.478487,0.456073,0.439831,0.442355,0.521777,0.598871,0.498273,0.438860,0.468806,0.626400,0.523556,0.384177,0.413451,0.537530,0.251481,0.5429
[...]
diff --git
a/src/test/resources/component/transformers/multi_attention_layer/output_dvalue_test6.csv
b/src/test/resources/component/transformers/multi_attention_layer/output_dvalue_test6.csv
new file mode 100644
index 0000000000..695ab823d8
--- /dev/null
+++
b/src/test/resources/component/transformers/multi_attention_layer/output_dvalue_test6.csv
@@ -0,0 +1 @@
+0.896445
diff --git
a/src/test/scripts/applications/nn/component/multi_attention_backward.dml
b/src/test/scripts/applications/nn/component/multi_attention_backward.dml
new file mode 100644
index 0000000000..441060562a
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/multi_attention_backward.dml
@@ -0,0 +1,66 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/layers/multi_attention.dml") as multi_attention
+
+batch_size = as.integer($1)
+seq_length = as.integer($2)
+heads = as.integer($3)
+embedding_dim = as.integer($4)
+debug = as.logical(as.integer($5))
+
+query = matrix(read($6, format="csv"), rows=batch_size,
cols=seq_length*heads*embedding_dim)
+key = matrix(read($7, format="csv"), rows=batch_size,
cols=seq_length*heads*embedding_dim)
+value = matrix(read($8, format="csv"), rows=batch_size,
cols=seq_length*heads*embedding_dim)
+
+dcontext = matrix(read($9, format="csv"), rows=batch_size,
cols=seq_length*heads*embedding_dim)
+attention = matrix(read($10, format="csv"), rows=batch_size,
cols=heads*seq_length*seq_length)
+
+expected_dquery = matrix(read($11, format="csv"), rows=batch_size,
cols=seq_length*heads*embedding_dim)
+expected_dkey = matrix(read($12, format="csv"), rows=batch_size,
cols=seq_length*heads*embedding_dim)
+expected_dvalue = matrix(read($13, format="csv"), rows=batch_size,
cols=seq_length*heads*embedding_dim)
+
+dropout_mask = matrix(0, rows=1, cols=1)
+
+[dquery, dkey, dvalue] = multi_attention::backward(dcontext, dropout_mask,
attention, query, key, value, heads, seq_length, embedding_dim, 0.0)
+
+if (debug) {
+ print(toString(dquery))
+ print(toString(expected_dquery))
+ print(toString(dkey))
+ print(toString(expected_dkey))
+ print(toString(dvalue))
+ print(toString(expected_dvalue))
+}
+
+dquery_error = max(abs(expected_dquery - dquery))
+dkey_error = max(abs(expected_dkey - dkey))
+dvalue_error = max(abs(expected_dvalue - dvalue))
+
+if (debug) {
+ print(dquery_error)
+ print(dkey_error)
+ print(dvalue_error)
+}
+
+write(dquery_error, $14, format="text")
+write(dkey_error, $15, format="text")
+write(dvalue_error, $16, format="text")
diff --git
a/src/test/scripts/applications/nn/component/multi_attention_forward.dml
b/src/test/scripts/applications/nn/component/multi_attention_forward.dml
new file mode 100644
index 0000000000..96b161053e
--- /dev/null
+++ b/src/test/scripts/applications/nn/component/multi_attention_forward.dml
@@ -0,0 +1,53 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("scripts/nn/layers/multi_attention.dml") as multi_attention
+
+batch_size = as.integer($1)
+seq_length = as.integer($2)
+heads = as.integer($3)
+embedding_dim = as.integer($4)
+debug = as.logical(as.integer($5))
+
+query = matrix(read($6, format="csv"), rows=batch_size,
cols=seq_length*heads*embedding_dim)
+key = matrix(read($7, format="csv"), rows=batch_size,
cols=seq_length*heads*embedding_dim)
+value = matrix(read($8, format="csv"), rows=batch_size,
cols=seq_length*heads*embedding_dim)
+
+expected_context = read($9, format="csv")
+expected_attention = read($10, format="csv")
+
+[context, attention, dropout_mask] = multi_attention::forward(query, key,
value, heads, seq_length, embedding_dim, 0.0)
+
+if (debug) {
+ print(toString(context))
+ print(toString(attention))
+}
+
+context_error = max(abs(expected_context - context))
+attention_error = max(abs(expected_attention - attention))
+
+if (debug) {
+ print(context_error)
+ print(attention_error)
+}
+
+write(context_error, $11, format="text")
+write(attention_error, $12, format="text")
