This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 8f1a402645 [SYSTEMDS-3701] New Scuro data representation search
primitive
8f1a402645 is described below
commit 8f1a402645185baaf8c92bed42b03c89b9050312
Author: Christina Dionysio <diony...@tu-berlin.de>
AuthorDate: Wed Aug 21 19:15:47 2024 +0200
[SYSTEMDS-3701] New Scuro data representation search primitive
Closes #2068.
---
.../python/systemds/scuro/aligner/dr_search.py | 124 +++++++++++++++++++++
src/main/python/systemds/scuro/aligner/task.py | 58 ++++++++++
src/main/python/systemds/scuro/main.py | 30 +++--
.../systemds/scuro/modality/video_modality.py | 2 +-
4 files changed, 204 insertions(+), 10 deletions(-)
diff --git a/src/main/python/systemds/scuro/aligner/dr_search.py
b/src/main/python/systemds/scuro/aligner/dr_search.py
new file mode 100644
index 0000000000..4bdc7da4a2
--- /dev/null
+++ b/src/main/python/systemds/scuro/aligner/dr_search.py
@@ -0,0 +1,124 @@
+# -------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# -------------------------------------------------------------
+import itertools
+from typing import List
+
+from aligner.task import Task
+from modality.aligned_modality import AlignedModality
+from modality.modality import Modality
+from representations.representation import Representation
+
+
+def get_modalities_by_name(modalities, name):
+ for modality in modalities:
+ if modality.name == name:
+ return modality
+
+ raise 'Modality ' + name + 'not in modalities'
+
+
+class DRSearch:
+ def __init__(self, modalities: List[Modality], task: Task,
representations: List[Representation]):
+ """
+ The DRSearch primitive finds the best uni- or multimodal data
representation for the given modalities for
+ a specific task
+ :param modalities: List of uni-modal modalities
+ :param task: custom task
+ :param representations: List of representations to be evaluated
+ """
+ self.modalities = modalities
+ self.task = task
+ self.representations = representations
+ self.scores = {}
+ self.best_modalities = None
+ self.best_representation = None
+ self.best_score = -1
+
+ def set_best_params(self, modality_name: str, representation:
Representation,
+ score: float, modality_names: List[str]):
+ """
+ Updates the best parameters for given modalities, representation, and
score
+ :param modality_name: The name of the aligned modality
+ :param representation: The representation used to retrieve the current
score
+ :param score: achieved score for the set of modalities and
representation
+ :param modality_names: List of modality names used in this setting
+ :return:
+ """
+
+ # check if modality name is already in dictionary
+ if modality_name not in self.scores.keys():
+ # if not add it to dictionary
+ self.scores[modality_name] = {}
+
+ # set score for representation
+ self.scores[modality_name][representation] = score
+
+ # compare current score with best score
+ if score > self.best_score:
+ self.best_score = score
+ self.best_representation = representation
+ self.best_modalities = modality_names
+
+ def fit(self):
+ """
+ This method finds the best representation out of a given List of
uni-modal modalities and
+ representations
+ :return: The best parameters found in the search procedure
+ """
+
+ for M in range(1, len(self.modalities) + 1):
+ for combination in itertools.combinations(self.modalities, M):
+ if len(combination) == 1:
+ modality = combination[0]
+ score =
self.task.run(modality.representation.scale_data(modality.data,
self.task.train_indices))
+ self.set_best_params(modality.name,
modality.representation.name, score, [modality.name])
+ self.scores[modality] = score
+ else:
+ for representation in self.representations:
+ modality = AlignedModality(representation,
list(combination)) # noqa
+ modality.combine(self.task.train_indices)
+
+ score = self.task.run(modality.data)
+ self.set_best_params(modality.name, representation,
score, modality.get_modality_names())
+
+ return self.best_representation, self.best_score, self.best_modalities
+
+ def transform(self, modalities: List[Modality]):
+ """
+ The transform method takes a list of uni-modal modalities and creates
an aligned representation
+ by using the best parameters found during the fitting step
+ :param modalities: List of uni-modal modalities
+ :return: aligned data
+ """
+
+ if self.best_score == -1:
+ raise 'Please fit representations first!'
+
+ used_modalities = []
+
+ for modality_name in self.best_modalities:
+ used_modalities.append(get_modalities_by_name(modalities,
modality_name))
+
+ modality = AlignedModality(self.best_representation, used_modalities)
# noqa
+ modality.combine(self.task.train_indices)
+
+ return modality.data
+
\ No newline at end of file
diff --git a/src/main/python/systemds/scuro/aligner/task.py
b/src/main/python/systemds/scuro/aligner/task.py
new file mode 100644
index 0000000000..79f9690e65
--- /dev/null
+++ b/src/main/python/systemds/scuro/aligner/task.py
@@ -0,0 +1,58 @@
+# -------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# -------------------------------------------------------------
+from typing import List
+
+from models.model import Model
+
+
+class Task:
+ def __init__(self, name: str, model: Model, labels, train_indices: List,
val_indices: List):
+ """
+ Parent class for the prediction task that is performed on top of the
aligned representation
+ :param name: Name of the task
+ :param model: ML-Model to run
+ :param labels: Labels used for prediction
+ :param train_indices: Indices to extract training data
+ :param val_indices: Indices to extract validation data
+ """
+ self.name = name
+ self.model = model
+ self.labels = labels
+ self.train_indices = train_indices
+ self.val_indices = val_indices
+
+ def get_train_test_split(self, data):
+ X_train = [data[i] for i in self.train_indices]
+ y_train = [self.labels[i] for i in self.train_indices]
+ X_test = [data[i] for i in self.val_indices]
+ y_test = [self.labels[i] for i in self.val_indices]
+
+ return X_train, y_train, X_test, y_test
+
+ def run(self, data):
+ """
+ The run method need to be implemented by every task class
+ It handles the training and validation procedures for the specific task
+ :param data: The aligned data used in the prediction process
+ :return: the validation accuracy
+ """
+ pass
+
\ No newline at end of file
diff --git a/src/main/python/systemds/scuro/main.py
b/src/main/python/systemds/scuro/main.py
index 398db9921c..22477eb549 100644
--- a/src/main/python/systemds/scuro/main.py
+++ b/src/main/python/systemds/scuro/main.py
@@ -23,16 +23,31 @@ import json
from datetime import datetime
from representations.average import Averaging
+from representations.concatenation import Concatenation
from modality.aligned_modality import AlignedModality
from modality.text_modality import TextModality
from modality.video_modality import VideoModality
from modality.audio_modality import AudioModality
from representations.unimodal import Pickle, JSON, HDF5, NPY
from models.discrete_model import DiscreteModel
+from aligner.task import Task
+from aligner.dr_search import DRSearch
+
+
+class CustomTask(Task):
+ def __init__(self, model, labels, train_indices, val_indices):
+ super().__init__('CustomTask', model, labels, train_indices,
val_indices)
+
+ def run(self, data):
+ X_train, y_train, X_test, y_test = self.get_train_test_split(data)
+ self.model.fit(X_train, y_train, X_test, y_test)
+ score = self.model.test(X_test, y_test)
+ return score
labels = []
train_indices = []
+val_indices = []
video_path = ''
audio_path = ''
@@ -47,15 +62,12 @@ video.read_all()
audio.read_all()
text.read_all()
-combined_modality = AlignedModality(Averaging(), [text, video, audio])
-combined_modality.combine()
-
-# create train-val split
-train_X, train_y = None, None
-val_X, val_y = None, None
+modalities = [text, audio, video]
model = DiscreteModel()
-model.fit(train_X, train_y)
-model.test(val_X, val_y)
-
+custom_task = CustomTask(model, labels, train_indices, val_indices)
+representations = [Concatenation(), Averaging()]
+dr_search = DRSearch(modalities, custom_task, representations)
+best_representation, best_score, best_modalities = dr_search.fit()
+aligned_representation = dr_search.transform(modalities)
diff --git a/src/main/python/systemds/scuro/modality/video_modality.py
b/src/main/python/systemds/scuro/modality/video_modality.py
index ff7eebc9cc..8062c26a89 100644
--- a/src/main/python/systemds/scuro/modality/video_modality.py
+++ b/src/main/python/systemds/scuro/modality/video_modality.py
@@ -49,5 +49,5 @@ class VideoModality(Modality):
def read_chunk(self):
pass
- def read_all(self, indices):
+ def read_all(self, indices=None):
self.data = self.representation.parse_all(self.file_path,
indices=indices) # noqa
