This is an automated email from the ASF dual-hosted git repository.
rpardomeza pushed a commit to branch profile-db
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
The following commit(s) were added to refs/heads/profile-db by this push:
new fdb7ac3 [WAYANG-32] More general corrections, comments, and Readme
update
fdb7ac3 is described below
commit fdb7ac352c5fb7eb69a436a5b4dc42c8d9d0a268
Author: rodrigopardomeza <[email protected]>
AuthorDate: Tue Aug 31 00:00:51 2021 +0200
[WAYANG-32] More general corrections, comments, and Readme update
---
wayang-commons/pom.xml | 3 +-
.../wayang-utils/wayang-profile-db/readme.md | 31 ++++-
.../src/main/java/profiledb/ProfileDB.java | 44 +++++++
.../src/main/java/profiledb/storage/Storage.java | 46 +++++--
.../src/test/java/profiledb/ProfileDBTest.java | 135 +++++++++------------
5 files changed, 172 insertions(+), 87 deletions(-)
diff --git a/wayang-commons/pom.xml b/wayang-commons/pom.xml
index 4dab9d5..2caa960 100644
--- a/wayang-commons/pom.xml
+++ b/wayang-commons/pom.xml
@@ -214,7 +214,7 @@
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
- <version>2.2.4</version>
+ <version>2.8.8</version>
</dependency>
</dependencies>
</dependencyManagement>
@@ -222,6 +222,7 @@
<modules>
<module>wayang-core</module>
<module>wayang-basic</module>
+ <module>wayang-utils</module>
</modules>
</project>
\ No newline at end of file
diff --git a/wayang-commons/wayang-utils/wayang-profile-db/readme.md
b/wayang-commons/wayang-utils/wayang-profile-db/readme.md
index 8c14afc..0e1fe47 100644
--- a/wayang-commons/wayang-utils/wayang-profile-db/readme.md
+++ b/wayang-commons/wayang-utils/wayang-profile-db/readme.md
@@ -1,3 +1,30 @@
-Base on
+This code is based on the implementation you can find in the following
repository:
-https://github.com/sekruse/profiledb-java.git
\ No newline at end of file
+- https://github.com/sekruse/profiledb-java.git
+
+The code there does not have regular maintenance. Wayang will require new
functionalities to deal with serialization of UDFs and storage in other
platforms.
+
+The classes below has not been modified:
+
+ MeasurementDeserializer
+ MeasurementSerializer
+ Experiment
+ Measurement
+ Subject
+ TimeMeasurement
+ Type
+
+The classes below has been added/modified to provide an abstraction over
different Storage methods:
+
+ ProfileDB
+ Storage
+ FileStorage
+ JDBCStorage
+
+The code that is based on the mentioned repository starts and ends with the
commits indicated below:
+
+- start:
[5344336f68bb9038e701435e9859321b6e8cbcfc](https://github.com/apache/incubator-wayang/commit/5344336f68bb9038e701435e9859321b6e8cbcfc)
+
+- end:
+
+All the code that has been added after those commits is totally independent of
the mentioned repository.
\ No newline at end of file
diff --git
a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/ProfileDB.java
b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/ProfileDB.java
index 21490e9..3020233 100644
---
a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/ProfileDB.java
+++
b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/ProfileDB.java
@@ -40,30 +40,74 @@ public class ProfileDB {
*/
private Gson gson;
+ /**
+ * Receive an array of {@link Experiment}s and persist them
+ *
+ * @param experiments Array of {@link Experiment}s to be persisted
+ * @throws IOException
+ */
public void save(Experiment... experiments) throws IOException {
this.storage.save(experiments);
}
+ /**
+ * Receive a Collection of {@link Experiment}s and persist them
+ *
+ * @param experiments Collection of {@link Experiment}s to be persisted
+ * @throws IOException
+ */
public void save(Collection<Experiment> experiments) throws IOException {
this.storage.save(experiments);
}
+ /**
+ * Receive a Collection of {@link Experiment}s and persist them
+ *
+ * @param experiments Collection of {@link Experiment}s to be persisted
+ * @param outputStream Indicates where the data must to be written
+ * @throws IOException
+ */
public void save(Collection<Experiment> experiments, OutputStream
outputStream) throws IOException {
this.storage.save(experiments, outputStream);
}
+ /**
+ * Related to file based storage, Receive an array of {@link Experiment}s
and persist them at the end of a file
+ *
+ * @param experiments Array of {@link Experiment}s to be persisted
+ * @throws IOException
+ */
public void append(Experiment... experiments) throws IOException {
this.storage.append(experiments);
}
+ /**
+ * Related to file based storage, Receive a Collection of {@link
Experiment}s and persist them at the end of a file
+ *
+ * @param experiments Collection of {@link Experiment}s to be persisted
+ * @throws IOException
+ */
public void append(Collection<Experiment> experiments) throws IOException {
this.storage.append(experiments);
}
+ /**
+ * Bring {@link Experiment}s from current Storage to local variable
+ *
+ * @return Collection of {@link Experiment}s
+ * @throws IOException
+ */
public Collection<Experiment> load() throws IOException {
return this.storage.load();
}
+ /**
+ * Bring {@link Experiment}s from current Storage to local variable
+ *
+ * @param inputStream Data to be read
+ * @return Collection of {@link Experiment}s
+ * @throws IOException
+ */
public Collection<Experiment> load(InputStream inputStream) throws
IOException {
return this.storage.load(inputStream);
}
diff --git
a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/storage/Storage.java
b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/storage/Storage.java
index b27aefc..f89f9c6 100644
---
a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/storage/Storage.java
+++
b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/storage/Storage.java
@@ -32,6 +32,8 @@ public abstract class Storage {
this.storageFile = uri;
}
+
+
/**
* Sets the ProfileDB for this instance that manages all the Measurement
subclasses
* */
@@ -47,18 +49,48 @@ public abstract class Storage {
this.storageFile = uri;
}
- public void save(Experiment... experiments) throws IOException {
- System.out.println("llegue");
- }
+ /**
+ * Receive an array of {@link Experiment}s and persist them
+ *
+ * @param experiments Array of {@link Experiment}s to be persisted
+ * @throws IOException
+ */
+ public abstract void save(Experiment... experiments) throws IOException;
- public void save(Collection<Experiment> experiments) throws IOException {}
+ /**
+ * Receive a Collection of {@link Experiment}s and persist them
+ *
+ * @param experiments Collection of {@link Experiment}s to be persisted
+ * @throws IOException
+ */
+ public abstract void save(Collection<Experiment> experiments) throws
IOException;
- public void append(Experiment... experiments) throws IOException {}
+ /**
+ * Related to file based storage, Receive an array of {@link Experiment}s
and persist them at the end of a file
+ *
+ * @param experiments Array of {@link Experiment}s to be persisted
+ * @throws IOException
+ */
+ public abstract void append(Experiment... experiments) throws IOException;
- public void append(Collection<Experiment> experiments) throws IOException
{}
+ /**
+ * Related to file based storage, Receive a Collection of {@link
Experiment}s and persist them at the end of a file
+ *
+ * @param experiments Collection of {@link Experiment}s to be persisted
+ * @throws IOException
+ */
+ public abstract void append(Collection<Experiment> experiments) throws
IOException ;
+
+ /**
+ * Bring {@link Experiment}s from current Storage to local variable
+ *
+ * @return Collection of {@link Experiment}s
+ * @throws IOException
+ */
+ public abstract Collection<Experiment> load() throws IOException;
- public Collection<Experiment> load() throws IOException { return null; }
+ //TODO The following methods should be moved to file storage implementation
/**
* Write {@link Experiment}s to an {@link OutputStream}.
*
diff --git
a/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/ProfileDBTest.java
b/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/ProfileDBTest.java
index ea93bc7..b002c65 100644
---
a/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/ProfileDBTest.java
+++
b/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/ProfileDBTest.java
@@ -15,7 +15,9 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.net.URL;
import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
@@ -25,32 +27,28 @@ public class ProfileDBTest {
public void testPolymorphSaveAndLoad() throws IOException {
try {
- URI uri = new
URI("file:///Users/rodrigopardomeza/Desktop/random/myfile.txt");
+ Path temp = Files.createTempFile("", ".tmp");
+
+ String absolutePath = temp.toString();
+ System.out.println("Temp file : " + absolutePath);
+
+ URI uri = new URI("my-file4");
+
FileStorage store = new FileStorage(uri);
ProfileDB profileDB = new ProfileDB(store)
.registerMeasurementClass(TestMemoryMeasurement.class)
.registerMeasurementClass(TestTimeMeasurement.class);
- /**
- * Esto es lo que se espera del codigo del cliente
- * Tiene que usar la API para registrar medidas
- */
- // crea un experimento falso
final Experiment experiment = new Experiment("test-xp", new
Subject("PageRank", "1.0"), "test experiment");
- // Agrega medidas falsas hardcoded
Measurement timeMeasurement = new TestTimeMeasurement("exec-time",
12345L);
Measurement memoryMeasurement = new
TestMemoryMeasurement("exec-time", System.currentTimeMillis(), 54321L);
- /*Agrega las medidas al experimento*/
experiment.addMeasurement(timeMeasurement);
experiment.addMeasurement(memoryMeasurement);
// Save the experiment.
- /**
- * Guarda el experimento en memoria
- */
byte[] buffer;
ByteArrayOutputStream bos = new ByteArrayOutputStream();
profileDB.save(Collections.singleton(experiment), bos);
@@ -59,9 +57,6 @@ public class ProfileDBTest {
System.out.println("Buffer contents: " + new String(buffer,
"UTF-8"));
// Load the experiment.
- /**
- * Lee el experimento desde el buffer en memoria
- */
ByteArrayInputStream bis = new ByteArrayInputStream(buffer);
Collection<Experiment> loadedExperiments = profileDB.load(bis);
@@ -86,7 +81,7 @@ public class ProfileDBTest {
@Test
public void testRecursiveSaveAndLoad() throws IOException {
try {
- URI uri = new
URI("file:///Users/rodrigopardomeza/Desktop/random/myfile.txt");
+ URI uri = new URI("my-file2");
FileStorage store = new FileStorage(uri);
ProfileDB profileDB = new ProfileDB(store)
@@ -128,78 +123,64 @@ public class ProfileDBTest {
@Test
public void testFileOperations() throws IOException {
-
- try {
- URI uri = new
URI("file:///Users/rodrigopardomeza/Desktop/random/myfile.txt");
- FileStorage store = new FileStorage(uri);
-
- ProfileDB profileDB = new ProfileDB(store)
- .registerMeasurementClass(TestMemoryMeasurement.class)
- .registerMeasurementClass(TestTimeMeasurement.class);
-
- // Create example experiments.
- final Experiment experiment1 = new Experiment("xp1", new
Subject("PageRank", "1.0"), "test experiment 1");
- experiment1.addMeasurement(new TestTimeMeasurement("exec-time",
1L));
- final Experiment experiment2 = new Experiment("xp2", new
Subject("KMeans", "1.1"), "test experiment 2");
- experiment2.addMeasurement(new TestTimeMeasurement("exec-time",
2L));
- final Experiment experiment3 = new Experiment("xp3", new
Subject("Apriori", "2.0"), "test experiment 3");
- experiment3.addMeasurement(new TestMemoryMeasurement("ram",
System.currentTimeMillis(), 3L));
-
- // Save the experiments.
- File tempDir = Files.createTempDirectory("profiledb").toFile();
- //File dir =
Files.createTempDirectory(Paths.get("/Users/rodrigopardomeza/Desktop/random/"),
"profiledb").toFile();
- //File dir =
Paths.get("/Users/rodrigopardomeza/Desktop/random/").toFile();
- File file = new File(tempDir, "profiledb.json");
- file.createNewFile();
- profileDB.save(experiment1);
- profileDB.append(experiment2, experiment3);
-
- System.out.println("File plat" + file.toPath().toUri().toString());
- Files.lines(file.toPath()).forEach(System.out::println);
-
- // Load and compare.
- final Set<Experiment> loadedExperiments = new
HashSet<>(profileDB.load());
- final List<Experiment> expectedExperiments =
Arrays.asList(experiment1, experiment2, experiment3);
- Assert.assertEquals(expectedExperiments.size(),
loadedExperiments.size());
- Assert.assertEquals(new HashSet<>(expectedExperiments), new
HashSet<>(loadedExperiments));
- } catch (URISyntaxException e) {
- e.printStackTrace();
- }
+ File tempDir = Files.createTempDirectory("profiledb").toFile();
+ File file = new File(tempDir, "profiledb.json");
+ file.createNewFile();
+ FileStorage store = new FileStorage(file.toURI());
+
+ ProfileDB profileDB = new ProfileDB(store)
+ .registerMeasurementClass(TestMemoryMeasurement.class)
+ .registerMeasurementClass(TestTimeMeasurement.class);
+
+ // Create example experiments.
+ final Experiment experiment1 = new Experiment("xp1", new
Subject("PageRank", "1.0"), "test experiment 1");
+ experiment1.addMeasurement(new TestTimeMeasurement("exec-time", 1L));
+ final Experiment experiment2 = new Experiment("xp2", new
Subject("KMeans", "1.1"), "test experiment 2");
+ experiment2.addMeasurement(new TestTimeMeasurement("exec-time", 2L));
+ final Experiment experiment3 = new Experiment("xp3", new
Subject("Apriori", "2.0"), "test experiment 3");
+ experiment3.addMeasurement(new TestMemoryMeasurement("ram",
System.currentTimeMillis(), 3L));
+
+ // Save the experiments.
+ profileDB.save(experiment1);
+ profileDB.append(experiment2, experiment3);
+
+ Files.lines(file.toPath()).forEach(System.out::println);
+
+ // Load and compare.
+ final Set<Experiment> loadedExperiments = new
HashSet<>(profileDB.load());
+ final List<Experiment> expectedExperiments =
Arrays.asList(experiment1, experiment2, experiment3);
+ Assert.assertEquals(expectedExperiments.size(),
loadedExperiments.size());
+ Assert.assertEquals(new HashSet<>(expectedExperiments), new
HashSet<>(loadedExperiments));
}
@Test
public void testAppendOnNonExistentFile() throws IOException {
- try {
- URI uri = new
URI("file:///Users/rodrigopardomeza/Desktop/random/myfile.txt");
- FileStorage store = new FileStorage(uri);
+ File tempDir = Files.createTempDirectory("profiledb").toFile();
+ File file = new File(tempDir, "new-profiledb.json");
+ file.createNewFile();
+ FileStorage store = new FileStorage(file.toURI());
- // This seems to be an issue on Linux.
- ProfileDB profileDB = new ProfileDB(store)
- .registerMeasurementClass(TestMemoryMeasurement.class)
- .registerMeasurementClass(TestTimeMeasurement.class);
+ // This seems to be an issue on Linux.
+ ProfileDB profileDB = new ProfileDB(store)
+ .registerMeasurementClass(TestMemoryMeasurement.class)
+ .registerMeasurementClass(TestTimeMeasurement.class);
- // Create example experiments.
- final Experiment experiment1 = new Experiment("xp1", new
Subject("PageRank", "1.0"), "test experiment 1");
- experiment1.addMeasurement(new TestTimeMeasurement("exec-time",
1L));
+ // Create example experiments.
+ final Experiment experiment1 = new Experiment("xp1", new
Subject("PageRank", "1.0"), "test experiment 1");
+ experiment1.addMeasurement(new TestTimeMeasurement("exec-time", 1L));
- // Save the experiments.
- File tempDir = Files.createTempDirectory("profiledb").toFile();
- File file = new File(tempDir, "new-profiledb.json");
- file.createNewFile();
- Assert.assertTrue(!file.exists() || file.delete());
- profileDB.append(experiment1);
+ // Save the experiments.
+ Assert.assertTrue(!file.exists() || file.delete());
+ profileDB.append(experiment1);
- Files.lines(file.toPath()).forEach(System.out::println);
+ Files.lines(file.toPath()).forEach(System.out::println);
- // Load and compare.
- final Set<Experiment> loadedExperiments = new
HashSet<>(profileDB.load());
- final List<Experiment> expectedExperiments =
Collections.singletonList(experiment1);
- Assert.assertEquals(expectedExperiments.size(),
loadedExperiments.size());
- Assert.assertEquals(new HashSet<>(expectedExperiments), new
HashSet<>(loadedExperiments));
- } catch (URISyntaxException e) {
- e.printStackTrace();
- }
+ // Load and compare.
+ final Set<Experiment> loadedExperiments = new
HashSet<>(profileDB.load());
+ final List<Experiment> expectedExperiments =
Collections.singletonList(experiment1);
+ Assert.assertEquals(expectedExperiments.size(),
loadedExperiments.size());
+ Assert.assertEquals(new HashSet<>(expectedExperiments), new
HashSet<>(loadedExperiments));
}
}
