This is an automated email from the ASF dual-hosted git repository. rpardomeza pushed a commit to branch profile-db in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit 5344336f68bb9038e701435e9859321b6e8cbcfc Author: rodrigopardomeza <[email protected]> AuthorDate: Mon Aug 30 13:00:09 2021 +0200 [WAYANG-32] Base structure for Wayang Experiments Storage functionalities --- wayang-commons/wayang-utils/pom.xml | 20 +++ .../wayang-utils/wayang-profile-db/pom.xml | 24 +++ .../wayang-utils/wayang-profile-db/readme.md | 3 + .../src/main/java/profiledb/ProfileDB.java | 105 +++++++++++ .../profiledb/json/MeasurementDeserializer.java | 40 +++++ .../java/profiledb/json/MeasurementSerializer.java | 23 +++ .../src/main/java/profiledb/model/Experiment.java | 163 +++++++++++++++++ .../src/main/java/profiledb/model/Measurement.java | 54 ++++++ .../src/main/java/profiledb/model/Subject.java | 69 +++++++ .../src/main/java/profiledb/model/Type.java | 13 ++ .../model/measurement/TimeMeasurement.java | 199 +++++++++++++++++++++ .../main/java/profiledb/storage/FileStorage.java | 101 +++++++++++ .../main/java/profiledb/storage/JDBCStorage.java | 90 ++++++++++ .../src/main/java/profiledb/storage/Storage.java | 122 +++++++++++++ .../src/test/java/profiledb/ProfileDBTest.java | 199 +++++++++++++++++++++ .../measurement/TestMemoryMeasurement.java | 62 +++++++ .../profiledb/measurement/TestTimeMeasurement.java | 56 ++++++ 17 files changed, 1343 insertions(+) diff --git a/wayang-commons/wayang-utils/pom.xml b/wayang-commons/wayang-utils/pom.xml new file mode 100644 index 0000000..7768f19 --- /dev/null +++ b/wayang-commons/wayang-utils/pom.xml @@ -0,0 +1,20 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>wayang-commons</artifactId> + <groupId>org.apache.wayang</groupId> + <version>0.6.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>wayang-utils</artifactId> + <packaging>pom</packaging> + + <modules> + <module>wayang-profile-db</module> + </modules> + + +</project> \ No newline at end of file diff --git a/wayang-commons/wayang-utils/wayang-profile-db/pom.xml b/wayang-commons/wayang-utils/wayang-profile-db/pom.xml new file mode 100644 index 0000000..45bfaf9 --- /dev/null +++ b/wayang-commons/wayang-utils/wayang-profile-db/pom.xml @@ -0,0 +1,24 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>wayang-utils</artifactId> + <groupId>org.apache.wayang</groupId> + <version>0.6.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>wayang-profile-db</artifactId> + + <dependencies> + <!-- https://mvnrepository.com/artifact/com.google.code.gson/gson --> + <dependency> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + </dependency> + + </dependencies> + + +</project> \ No newline at end of file diff --git a/wayang-commons/wayang-utils/wayang-profile-db/readme.md b/wayang-commons/wayang-utils/wayang-profile-db/readme.md new file mode 100644 index 0000000..8c14afc --- /dev/null +++ b/wayang-commons/wayang-utils/wayang-profile-db/readme.md @@ -0,0 +1,3 @@ +Base on + +https://github.com/sekruse/profiledb-java.git \ No newline at end of file diff --git a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/ProfileDB.java b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/ProfileDB.java new file mode 100644 index 0000000..cd90d40 --- /dev/null +++ b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/ProfileDB.java @@ -0,0 +1,105 @@ +package profiledb; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import profiledb.json.MeasurementDeserializer; +import profiledb.json.MeasurementSerializer; +import profiledb.model.Experiment; +import profiledb.model.Measurement; +import profiledb.storage.Storage; + +import java.io.*; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.function.Consumer; + +/** + * This class provides facilities to save and load {@link Experiment}s. + */ +public class ProfileDB { + + /** + * Maintains the full list of {@link Class}es for {@link Measurement}s. Which are required for deserialization. + */ + private List<Class<? extends Measurement>> measurementClasses = new LinkedList<>(); + + /** + * Controls how conducted experiments will be persisted and loaded + */ + private Storage storage; + + /** + * Maintains actions to preparate {@link Gson}. + */ + private List<Consumer<GsonBuilder>> gsonPreparationSteps = new LinkedList<>(); + + /** + * Maintains a {@link Gson} object for efficiency. It will be dropped on changes, though. + */ + private Gson gson; + + /** + * Creates a new instance. + */ + public ProfileDB(Storage storage) { + + this.storage = storage; + this.storage.setContext(this); + //this.measurementClasses.add(TimeMeasurement.class); + } + + /** + * To work with storage object provided to persist or load experiments + * + * @return Storage object proportioned for this instance + */ + public Storage getStorage() { + return storage; + } + + /** + * Register a {@link Measurement} type. This is required before being able to load that type. + * + * @param measurementClass the {@link Measurement} {@link Class} + * @return this instance + */ + public ProfileDB registerMeasurementClass(Class<? extends Measurement> measurementClass) { + this.measurementClasses.add(measurementClass); + this.gson = null; + return this; + } + + /** + * Apply any changes necessary to {@link Gson} so that it can be used for de/serialization of custom objects. + * + * @param preparation a preparatory step performed on a {@link GsonBuilder} + * @return this instance + */ + public ProfileDB withGsonPreparation(Consumer<GsonBuilder> preparation) { + this.gsonPreparationSteps.add(preparation); + this.gson = null; + return this; + } + + /** + * Provide a {@link Gson} object. + * + * @return the {@link Gson} object + */ + public Gson getGson() { + if (this.gson == null) { + MeasurementSerializer measurementSerializer = new MeasurementSerializer(); + MeasurementDeserializer measurementDeserializer = new MeasurementDeserializer(); + this.measurementClasses.forEach(measurementDeserializer::register); + final GsonBuilder gsonBuilder = new GsonBuilder() + .registerTypeAdapter(Measurement.class, measurementDeserializer) + .registerTypeAdapter(Measurement.class, measurementSerializer); + this.gsonPreparationSteps.forEach(step -> step.accept(gsonBuilder)); + this.gson = gsonBuilder.create(); + } + return this.gson; + } + +} diff --git a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/json/MeasurementDeserializer.java b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/json/MeasurementDeserializer.java new file mode 100644 index 0000000..cedecd8 --- /dev/null +++ b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/json/MeasurementDeserializer.java @@ -0,0 +1,40 @@ +package profiledb.json; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; +import profiledb.model.Measurement; + +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.Map; + +/** + * Custom deserializer for {@link Measurement}s + * Detects actual subclass of serialized instances and then delegates the deserialization to that subtype. + */ +public class MeasurementDeserializer implements JsonDeserializer<Measurement> { + + private final Map<String, Class<? extends Measurement>> measurementTypes = new HashMap<>(); + + public void register(Class<? extends Measurement> measurementClass) { + String typeName = Measurement.getTypeName(measurementClass); + this.measurementTypes.put(typeName, measurementClass); + } + + @Override + public Measurement deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext jsonDeserializationContext) throws JsonParseException { + final JsonElement typeElement = jsonElement.getAsJsonObject().get("type"); + if (typeElement == null) { + throw new IllegalArgumentException("Missing type in " + jsonElement); + } + final String typeName = typeElement.getAsString(); + final Class<? extends Measurement> measurementClass = this.measurementTypes.get(typeName); + if (measurementClass == null) { + throw new JsonParseException("Unknown measurement type: " + typeName); + } + return jsonDeserializationContext.deserialize(jsonElement, measurementClass); + } + +} diff --git a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/json/MeasurementSerializer.java b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/json/MeasurementSerializer.java new file mode 100644 index 0000000..4f820e8 --- /dev/null +++ b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/json/MeasurementSerializer.java @@ -0,0 +1,23 @@ +package profiledb.json; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; +import profiledb.model.Measurement; + +import java.lang.reflect.Type; + +/** + * Custom serializer for {@link Measurement}s + * Detects actual subclass of given instances, encodes this class membership, and then delegates serialization to that subtype. + */ +public class MeasurementSerializer implements JsonSerializer<Measurement> { + + @Override + public JsonElement serialize(Measurement measurement, Type type, JsonSerializationContext jsonSerializationContext) { + final JsonObject jsonObject = (JsonObject) jsonSerializationContext.serialize(measurement); + jsonObject.addProperty("type", measurement.getType()); + return jsonObject; + } +} diff --git a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/model/Experiment.java b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/model/Experiment.java new file mode 100644 index 0000000..1222a75 --- /dev/null +++ b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/model/Experiment.java @@ -0,0 +1,163 @@ +package profiledb.model; + +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedList; +import java.util.Objects; + +/** + * An experiment comprises {@link Measurement}s from one specific {@link Subject} execution. + */ +public class Experiment { + + /** + * Identifier for this instance. + */ + private String id; + + /** + * Description for this instance. (Optional) + */ + private String description; + + /** + * When this experiment has been started. + */ + private long startTime; + + /** + * Tags to group multiple Experiment instances. (Optional) + */ + private Collection<String> tags; + + /** + * {@link Measurement}s captured for this instance. + */ + private Collection<Measurement> measurements; + + /** + * The {@link Subject} being experimented with. + */ + private Subject subject; + + /** + * For deserialization. + */ + private Experiment() { + } + + /** + * Create a new instance that is starting right now. + * + * @param id Identifier for the new instance + * @param subject the {@link Subject} + * @param tags tags to group several experiments + */ + public Experiment(String id, Subject subject, String... tags) { + this(id, subject, System.currentTimeMillis(), tags); + } + + /** + * Create a new instance. + * + * @param id Identifier for the new instance + * @param subject the {@link Subject} of this experiment + * @param startTime start timestamp of this experiment + * @param tags tags to group several experiments + */ + public Experiment(String id, Subject subject, long startTime, String... tags) { + this.id = id; + this.subject = subject; + this.startTime = startTime; + this.tags = Arrays.asList(tags); + this.measurements = new LinkedList<>(); + } + + /** + * Adds a description for this instance. + * + * @param description the description + * @return this instance + */ + public Experiment withDescription(String description) { + this.description = description; + return this; + } + + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public long getStartTime() { + return startTime; + } + + public void setStartTime(long startTime) { + this.startTime = startTime; + } + + public Collection<String> getTags() { + return tags; + } + + public void setTags(Collection<String> tags) { + this.tags = tags; + } + + public void addMeasurement(Measurement measurement) { + this.measurements.add(measurement); + } + + public Collection<Measurement> getMeasurements() { + return measurements; + } + + public Subject getSubject() { + return this.subject; + } + + public void setSubject(Subject subject) { + this.subject = subject; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Experiment that = (Experiment) o; + return startTime == that.startTime && + Objects.equals(id, that.id) && + Objects.equals(description, that.description) && + Objects.equals(tags, that.tags) && + Objects.equals(subject, that.subject); + } + + @Override + public int hashCode() { + return Objects.hash(id, startTime); + } + + @Override + public String toString() { + return String.format( + "%s[%s, %d tags, %d measurements]", + this.getClass().getSimpleName(), + this.id, + this.tags.size(), + this.measurements.size() + ); + } +} diff --git a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/model/Measurement.java b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/model/Measurement.java new file mode 100644 index 0000000..23567a1 --- /dev/null +++ b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/model/Measurement.java @@ -0,0 +1,54 @@ +package profiledb.model; + +import java.util.Objects; + +/** + * + * Measurement captures the value of a metric at a specific time + */ +public abstract class Measurement { + + private String id; + + /** + * Returns implementation Class of this Measurement + */ + public static String getTypeName(Class<? extends Measurement> measurementClass) { + return measurementClass.getDeclaredAnnotation(Type.class).value(); + } + + /** + * Deserialization constructor. + */ + protected Measurement() { + } + + public Measurement(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getType() { + return getTypeName(this.getClass()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Measurement that = (Measurement) o; + return Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } +} diff --git a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/model/Subject.java b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/model/Subject.java new file mode 100644 index 0000000..cfb6710 --- /dev/null +++ b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/model/Subject.java @@ -0,0 +1,69 @@ +package profiledb.model; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * The subject of an {@link Experiment}, e.g., an application or algorithm. + */ +public class Subject { + + /** + * Identifier for the subject. + */ + private String id; + + /** + * Version of the subject. + */ + private String version; + + /** + * Configuration of this object. + */ + private Map<String, Object> configuration = new HashMap<>(); + + /** + * Creates a new instance. + * + * @param id Identifier for the subject + * @param version To distinguish different versions among instances with the same {@code id} + */ + public Subject(String id, String version) { + this.id = id; + this.version = version; + } + + /** + * Adds a configuration. + * + * @param key Key of the configuration entry + * @param value Value for the new configuration entry; must be JSON-compatible, e.g. {@link Integer} or {@link String} + * @return this instance + */ + public Subject addConfiguration(String key, Object value) { + this.configuration.put(key, value); + return this; + } + + @Override + public String toString() { + return String.format("%s[%s:%s]", this.getClass().getSimpleName(), this.id, this.version); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Subject subject = (Subject) o; + return Objects.equals(id, subject.id) && + Objects.equals(version, subject.version) && + Objects.equals(configuration, subject.configuration); + } + + @Override + public int hashCode() { + return Objects.hash(id, version, configuration); + } +} diff --git a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/model/Type.java b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/model/Type.java new file mode 100644 index 0000000..f96e5f2 --- /dev/null +++ b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/model/Type.java @@ -0,0 +1,13 @@ +package profiledb.model; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface Type { + + String value(); +} diff --git a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/model/measurement/TimeMeasurement.java b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/model/measurement/TimeMeasurement.java new file mode 100644 index 0000000..e73caa9 --- /dev/null +++ b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/model/measurement/TimeMeasurement.java @@ -0,0 +1,199 @@ +package profiledb.model.measurement; + +import profiledb.model.Measurement; +import profiledb.model.Type; + +import java.util.Collection; +import java.util.LinkedList; + +/** + * A {@link Measurement} that captures a certain amount of time in milliseconds. Instances can be nested within + * each other. + * <p>Besides storing those data, it also provides utility functionality to obtain measurements.</p> + */ +@Type("time") +public class TimeMeasurement extends Measurement { + + /** + * The measured time in milliseconds. + */ + private long millis = 0L; + + /** + * Keeps track on measurement starts. + */ + private transient long startTime = -1L; + + /** + * Sub-{@link TimeMeasurement}s of this instance. + */ + private Collection<TimeMeasurement> rounds = new LinkedList<>(); + + /** + * Serialization constructor. + */ + @SuppressWarnings("unused") + private TimeMeasurement() { + super(); + } + + /** + * Creates a new instance. + * + * @param id the ID of the new instance + */ + public TimeMeasurement(String id) { + super(id); + } + + /** + * Start measuring time for this instance. + */ + public void start() { + this.startTime = System.currentTimeMillis(); + } + + /** + * Ensure that this instance has its timer started. + */ + private void ensureStarted() { + if (this.startTime == -1L) { + this.startTime = System.currentTimeMillis(); + } + } + + /** + * Start a (potentially new) sub-{@link TimeMeasurement}. + * + * @param identifiers identifies the target {@link TimeMeasurement} as a path of IDs + * @return the started instance + */ + public TimeMeasurement start(String... identifiers) { + return this.start(identifiers, 0); + } + + /** + * Start a (potentially new) sub-{@link TimeMeasurement}. + * + * @param identifiers identifies the target {@link TimeMeasurement} as a path of IDs + * @param index the index of this instance within {@code identifiers} + * @return the started instance + */ + private TimeMeasurement start(String[] identifiers, int index) { + if (index >= identifiers.length) { + this.start(); + return this; + } else { + this.ensureStarted(); + TimeMeasurement round = this.getOrCreateRound(identifiers[index]); + return round.start(identifiers, index + 1); + } + } + + /** + * Retrieves an existing {@link TimeMeasurement} from {@link #rounds} with the given {@code id} or creates and stores a new one. + * + * @param id the ID of the {@link TimeMeasurement} + * @return the {@link TimeMeasurement} + */ + public TimeMeasurement getOrCreateRound(String id) { + TimeMeasurement round = this.getRound(id); + if (round != null) return round; + + round = new TimeMeasurement(id); + this.rounds.add(round); + return round; + } + + /** + * Retrieves an existing {@link TimeMeasurement} from {@link #rounds} with the given {@code id}. + * + * @param id the ID of the {@link TimeMeasurement} + * @return the {@link TimeMeasurement} or {@code null} if it does not exist + */ + private TimeMeasurement getRound(String id) { + for (TimeMeasurement round : this.rounds) { + if (id.equals(round.getId())) { + return round; + } + } + return null; + } + + /** + * Stop a measurement that has been started via {@link #start()} or derivatives. + */ + public void stop() { + this.stop(System.currentTimeMillis()); + } + + + /** + * Stop a measurement that has been started via {@link #start()} or derivatives. + * + * @param stopTime at which the measurement has been stopped + */ + private void stop(long stopTime) { + if (this.startTime != -1L) { + this.millis += (stopTime - this.startTime); + this.startTime = -1L; + } + for (TimeMeasurement round : this.rounds) { + round.stop(stopTime); + } + } + + /** + * Stop a measurement that has been started via {@link #start(String...)} or related. + * + * @param identfiers identify the target {@link TimeMeasurement} as a path of IDs + */ + public void stop(String... identfiers) { + long stopTime = System.currentTimeMillis(); + TimeMeasurement round = this; + for (String identfier : identfiers) { + round = round.getRound(identfier); + if (round == null) return; + } + round.stop(stopTime); + } + + public long getMillis() { + return millis; + } + + public void setMillis(long millis) { + this.millis = millis; + } + + public Collection<TimeMeasurement> getRounds() { + return rounds; + } + + public void addRounds(TimeMeasurement round) { + this.rounds.add(round); + } + + /** + * Formats the given milliseconds as {@code h:MM:ss.mmm}. + * + * @param millis the milliseconds to format + * @return the formatted milliseconds + */ + public static String formatDuration(long millis) { + if (millis < 0) return "-" + formatDuration(-millis); + long ms = millis % 1000; + millis /= 1000; + long s = millis % 60; + millis /= 60; + long m = millis % 60; + millis /= 60; + long h = millis % 60; + return String.format("%d:%02d:%02d.%03d", h, m, s, ms); + } + + @Override + public String toString() { + return String.format("%s[%s, %s, %d subs]", this.getClass().getSimpleName(), this.getId(), formatDuration(this.millis), this.rounds.size()); + } +} diff --git a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/storage/FileStorage.java b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/storage/FileStorage.java new file mode 100644 index 0000000..4182dc5 --- /dev/null +++ b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/storage/FileStorage.java @@ -0,0 +1,101 @@ +package profiledb.storage; + +import profiledb.model.Experiment; + +import java.io.*; +import java.net.URI; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedList; + +public class FileStorage extends Storage { + + /** + * File where {@link Experiment}s will be written + */ + private File file; + + /** + * Assigns File where {@link Experiment}s will be written regarding to given URI + * + * @param uri URI where experiments are persisted + */ + public FileStorage(URI uri) { + + super(uri); + this.file = new File(uri); + } + + /** + * To change target URI during execution + * + * @param uri determines new URI where {@link Experiment}s will be persisted + */ + @Override + public void changeLocation(URI uri){ + + super.changeLocation(uri); + this.file = new File(uri); + } + + /** + * Write {@link Experiment}s to a {@link File}. Existing file contents will be overwritten. + * + * @param experiments the {@link Experiment}s + * @throws IOException if the writing fails + */ + @Override + public void save(Collection<Experiment> experiments) throws IOException { + this.file.getAbsoluteFile().getParentFile().mkdirs(); + try (FileOutputStream fos = new FileOutputStream(this.file, false)) { + this.save(experiments, fos); + } + } + + /** + * Write {@link Experiment}s to a {@link File}. Existing file contents will be overwritten. + * + * @param experiments the {@link Experiment}s + * @throws IOException if the writing fails + */ + @Override + public void save(Experiment... experiments) throws IOException { + this.save(Arrays.asList(experiments)); + } + + /** + * Load {@link Experiment}s from a {@link File}. + * + * @return the {@link Experiment}s + */ + @Override + public Collection<Experiment> load() throws IOException { + return load(new FileInputStream(this.file)); + } + + /** + * Append {@link Experiment}s to a {@link File}. Existing file contents will be preserved. + * + * @param experiments the {@link Experiment}s + * @throws IOException if the writing fails + */ + @Override + public void append(Collection<Experiment> experiments) throws IOException { + this.file.getAbsoluteFile().getParentFile().mkdirs(); + try (FileOutputStream fos = new FileOutputStream(this.file, true)) { + this.save(experiments, fos); + } + } + + /** + * Append {@link Experiment}s to a {@link File}. Existing file contents will be preserved. + * + * @param experiments the {@link Experiment}s + * @throws IOException if the writing fails + */ + @Override + public void append(Experiment... experiments) throws IOException { + this.append(Arrays.asList(experiments)); + } + +} diff --git a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/storage/JDBCStorage.java b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/storage/JDBCStorage.java new file mode 100644 index 0000000..ba847c1 --- /dev/null +++ b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/storage/JDBCStorage.java @@ -0,0 +1,90 @@ +package profiledb.storage; + +import profiledb.model.Experiment; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URI; +import java.util.Arrays; +import java.util.Collection; + +public class JDBCStorage extends Storage { + + //TODO: Implement JDBC connection + + private File file; + + public JDBCStorage(URI uri) { + super(uri); + this.file = new File(uri); + } + + @Override + public void changeLocation(URI uri){ + + super.changeLocation(uri); + this.file = new File(uri); + } + + /** + * Write {@link Experiment}s to a {@link File}. Existing file contents will be overwritten. + * + * @param experiments the {@link Experiment}s + * @throws IOException if the writing fails + */ + @Override + public void save(Collection<Experiment> experiments) throws IOException { + this.file.getAbsoluteFile().getParentFile().mkdirs(); + try (FileOutputStream fos = new FileOutputStream(this.file, false)) { + this.save(experiments, fos); + } + } + + /** + * Write {@link Experiment}s to a {@link File}. Existing file contents will be overwritten. + * + * @param experiments the {@link Experiment}s + * @throws IOException if the writing fails + */ + @Override + public void save(Experiment... experiments) throws IOException { + this.save(Arrays.asList(experiments)); + } + + /** + * Load {@link Experiment}s from a {@link File}. + * + * @return the {@link Experiment}s + */ + @Override + public Collection<Experiment> load() throws IOException { + return load(new FileInputStream(this.file)); + } + + /** + * Append {@link Experiment}s to a {@link File}. Existing file contents will be preserved. + * + * @param experiments the {@link Experiment}s + * @throws IOException if the writing fails + */ + @Override + public void append(Collection<Experiment> experiments) throws IOException { + this.file.getAbsoluteFile().getParentFile().mkdirs(); + try (FileOutputStream fos = new FileOutputStream(this.file, true)) { + this.save(experiments, fos); + } + } + + /** + * Append {@link Experiment}s to a {@link File}. Existing file contents will be preserved. + * + * @param experiments the {@link Experiment}s + * @throws IOException if the writing fails + */ + @Override + public void append(Experiment... experiments) throws IOException { + this.append(Arrays.asList(experiments)); + } +} diff --git a/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/storage/Storage.java b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/storage/Storage.java new file mode 100644 index 0000000..cdda15b --- /dev/null +++ b/wayang-commons/wayang-utils/wayang-profile-db/src/main/java/profiledb/storage/Storage.java @@ -0,0 +1,122 @@ +package profiledb.storage; + +import com.google.gson.Gson; +import profiledb.ProfileDB; +import profiledb.model.Experiment; + +import java.io.*; +import java.net.URI; +import java.util.Collection; +import java.util.LinkedList; + +/** + * Controls how conducted experiments will be persisted and loaded + */ +public abstract class Storage { + + /** + * Object or URI where experiments are persisted + */ + private URI storageFile; + + /** + * To access profileDB general serialization functions + */ + private ProfileDB context; + + /** + * Creates a new instance. + * @param uri Object or URI where experiments are persisted + */ + public Storage(URI uri){ + this.storageFile = uri; + } + + /** + * Sets the ProfileDB for this instance that manages all the Measurement subclasses + * */ + public void setContext(ProfileDB context) { + this.context = context; + } + + /** + * Allows to change where future experiments will be persisted and loaded + * @param uri + */ + public void changeLocation(URI uri){ + this.storageFile = uri; + } + + public void save(Experiment... experiments) throws IOException {} + + public void save(Collection<Experiment> experiments) throws IOException {} + + public void append(Experiment... experiments) throws IOException {} + + public void append(Collection<Experiment> experiments) throws IOException {} + + public Collection<Experiment> load() throws IOException { return null; } + + /** + * Write {@link Experiment}s to an {@link OutputStream}. + * + * @param outputStream the {@link OutputStream} + */ + public void save(Collection<Experiment> experiments, OutputStream outputStream) throws IOException { + try { + BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(outputStream, "UTF-8")); + this.save(experiments, writer); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("Unexpectedly, UTF-8 is not supported."); + } + } + + /** + * Write {@link Experiment}s to a {@link Writer}. + * + * @param writer the {@link Writer} + */ + public void save(Collection<Experiment> experiments, Writer writer) throws IOException { + try { + Gson gson = context.getGson(); + for (Experiment experiment : experiments) { + gson.toJson(experiment, writer); + writer.append('\n'); + } + writer.flush(); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("Unexpectedly, UTF-8 is not supported."); + } + } + + /** + * Load {@link Experiment}s from an {@link InputStream}. + * + * @param inputStream the {@link InputStream} + * @return the {@link Experiment}s + */ + public Collection<Experiment> load(InputStream inputStream) throws IOException { + try { + return load(new BufferedReader(new InputStreamReader(inputStream, "UTF-8"))); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("Unexpectedly, UTF-8 is not supported."); + } + } + + /** + * Load {@link Experiment}s from an {@link Reader}. + * + * @param reader the {@link Reader} + * @return the {@link Experiment}s + */ + public Collection<Experiment> load(BufferedReader reader) throws IOException { + Collection<Experiment> experiments = new LinkedList<>(); + Gson gson = context.getGson(); + String line; + while ((line = reader.readLine()) != null) { + Experiment experiment = gson.fromJson(line, Experiment.class); + experiments.add(experiment); + } + return experiments; + } +} diff --git a/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/ProfileDBTest.java b/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/ProfileDBTest.java new file mode 100644 index 0000000..da05470 --- /dev/null +++ b/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/ProfileDBTest.java @@ -0,0 +1,199 @@ +package profiledb; + +import org.junit.Assert; +import org.junit.Test; +import profiledb.measurement.TestMemoryMeasurement; +import profiledb.measurement.TestTimeMeasurement; +import profiledb.model.Experiment; +import profiledb.model.Measurement; +import profiledb.model.Subject; +import profiledb.storage.FileStorage; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.util.*; + +public class ProfileDBTest { + + @Test + public void testPolymorphSaveAndLoad() throws IOException { + + try { + URI uri = new URI("file:///Users/rodrigopardomeza/Desktop/random/myfile.txt"); + FileStorage store = new FileStorage(uri); + + ProfileDB profileDB = new ProfileDB(store) + .registerMeasurementClass(TestMemoryMeasurement.class) + .registerMeasurementClass(TestTimeMeasurement.class); + + /** + * Esto es lo que se espera del codigo del cliente + * Tiene que usar la API para registrar medidas + */ + // crea un experimento falso + final Experiment experiment = new Experiment("test-xp", new Subject("PageRank", "1.0"), "test experiment"); + + // Agrega medidas falsas hardcoded + Measurement timeMeasurement = new TestTimeMeasurement("exec-time", 12345L); + Measurement memoryMeasurement = new TestMemoryMeasurement("exec-time", System.currentTimeMillis(), 54321L); + + /*Agrega las medidas al experimento*/ + experiment.addMeasurement(timeMeasurement); + experiment.addMeasurement(memoryMeasurement); + + // Save the experiment. + /** + * Guarda el experimento en memoria + */ + byte[] buffer; + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + profileDB.getStorage().save(Collections.singleton(experiment), bos); + bos.close(); + buffer = bos.toByteArray(); + System.out.println("Buffer contents: " + new String(buffer, "UTF-8")); + + // Load the experiment. + /** + * Lee el experimento desde el buffer en memoria + */ + ByteArrayInputStream bis = new ByteArrayInputStream(buffer); + Collection<Experiment> loadedExperiments = profileDB.getStorage().load(bis); + + // Compare the experiments. + Assert.assertEquals(1, loadedExperiments.size()); + Experiment loadedExperiment = loadedExperiments.iterator().next(); + Assert.assertEquals(experiment, loadedExperiment); + + // Compare the measurements. + Assert.assertEquals(2, loadedExperiment.getMeasurements().size()); + Set<Measurement> expectedMeasurements = new HashSet<>(2); + expectedMeasurements.add(timeMeasurement); + expectedMeasurements.add(memoryMeasurement); + Set<Measurement> loadedMeasurements = new HashSet<>(loadedExperiment.getMeasurements()); + Assert.assertEquals(expectedMeasurements, loadedMeasurements); + + } catch (URISyntaxException e) { + e.printStackTrace(); + } + } + + @Test + public void testRecursiveSaveAndLoad() throws IOException { + try { + URI uri = new URI("file:///Users/rodrigopardomeza/Desktop/random/myfile.txt"); + FileStorage store = new FileStorage(uri); + + ProfileDB profileDB = new ProfileDB(store) + .registerMeasurementClass(TestMemoryMeasurement.class) + .registerMeasurementClass(TestTimeMeasurement.class); + + // Create an example experiment. + final Experiment experiment = new Experiment("test-xp", new Subject("PageRank", "1.0"), "test experiment"); + TestTimeMeasurement topLevelMeasurement = new TestTimeMeasurement("exec-time", 12345L); + TestTimeMeasurement childMeasurement = new TestTimeMeasurement("sub-exec-time", 2345L); + topLevelMeasurement.addSubmeasurements(childMeasurement); + experiment.addMeasurement(topLevelMeasurement); + + // Save the experiment. + byte[] buffer; + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + profileDB.getStorage().save(Collections.singleton(experiment), bos); + bos.close(); + buffer = bos.toByteArray(); + System.out.println("Buffer contents: " + new String(buffer, "UTF-8")); + + // Load the experiment. + ByteArrayInputStream bis = new ByteArrayInputStream(buffer); + Collection<Experiment> loadedExperiments = profileDB.getStorage().load(bis); + + // Compare the experiments. + Assert.assertEquals(1, loadedExperiments.size()); + Experiment loadedExperiment = loadedExperiments.iterator().next(); + Assert.assertEquals(experiment, loadedExperiment); + + // Compare the measurements. + Assert.assertEquals(1, loadedExperiment.getMeasurements().size()); + final Measurement loadedMeasurement = loadedExperiment.getMeasurements().iterator().next(); + Assert.assertEquals(topLevelMeasurement, loadedMeasurement); + } catch (URISyntaxException e) { + e.printStackTrace(); + } + } + + @Test + public void testFileOperations() throws IOException { + + try { + URI uri = new URI("file:///Users/rodrigopardomeza/Desktop/random/myfile.txt"); + FileStorage store = new FileStorage(uri); + + ProfileDB profileDB = new ProfileDB(store) + .registerMeasurementClass(TestMemoryMeasurement.class) + .registerMeasurementClass(TestTimeMeasurement.class); + + // Create example experiments. + final Experiment experiment1 = new Experiment("xp1", new Subject("PageRank", "1.0"), "test experiment 1"); + experiment1.addMeasurement(new TestTimeMeasurement("exec-time", 1L)); + final Experiment experiment2 = new Experiment("xp2", new Subject("KMeans", "1.1"), "test experiment 2"); + experiment2.addMeasurement(new TestTimeMeasurement("exec-time", 2L)); + final Experiment experiment3 = new Experiment("xp3", new Subject("Apriori", "2.0"), "test experiment 3"); + experiment3.addMeasurement(new TestMemoryMeasurement("ram", System.currentTimeMillis(), 3L)); + + // Save the experiments. + File tempDir = Files.createTempDirectory("profiledb").toFile(); + File file = new File(tempDir, "profiledb.json"); + profileDB.getStorage().save(experiment1); + profileDB.getStorage().append(experiment2, experiment3); + + Files.lines(file.toPath()).forEach(System.out::println); + + // Load and compare. + final Set<Experiment> loadedExperiments = new HashSet<>(profileDB.getStorage().load()); + final List<Experiment> expectedExperiments = Arrays.asList(experiment1, experiment2, experiment3); + Assert.assertEquals(expectedExperiments.size(), loadedExperiments.size()); + Assert.assertEquals(new HashSet<>(expectedExperiments), new HashSet<>(loadedExperiments)); + } catch (URISyntaxException e) { + e.printStackTrace(); + } + } + + @Test + public void testAppendOnNonExistentFile() throws IOException { + + try { + URI uri = new URI("file:///Users/rodrigopardomeza/Desktop/random/myfile.txt"); + FileStorage store = new FileStorage(uri); + + // This seems to be an issue on Linux. + ProfileDB profileDB = new ProfileDB(store) + .registerMeasurementClass(TestMemoryMeasurement.class) + .registerMeasurementClass(TestTimeMeasurement.class); + + // Create example experiments. + final Experiment experiment1 = new Experiment("xp1", new Subject("PageRank", "1.0"), "test experiment 1"); + experiment1.addMeasurement(new TestTimeMeasurement("exec-time", 1L)); + + // Save the experiments. + File tempDir = Files.createTempDirectory("profiledb").toFile(); + File file = new File(tempDir, "new-profiledb.json"); + Assert.assertTrue(!file.exists() || file.delete()); + profileDB.getStorage().append(experiment1); + + Files.lines(file.toPath()).forEach(System.out::println); + + // Load and compare. + final Set<Experiment> loadedExperiments = new HashSet<>(profileDB.getStorage().load()); + final List<Experiment> expectedExperiments = Collections.singletonList(experiment1); + Assert.assertEquals(expectedExperiments.size(), loadedExperiments.size()); + Assert.assertEquals(new HashSet<>(expectedExperiments), new HashSet<>(loadedExperiments)); + } catch (URISyntaxException e) { + e.printStackTrace(); + } + } + +} diff --git a/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/measurement/TestMemoryMeasurement.java b/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/measurement/TestMemoryMeasurement.java new file mode 100644 index 0000000..200a661 --- /dev/null +++ b/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/measurement/TestMemoryMeasurement.java @@ -0,0 +1,62 @@ +package profiledb.measurement; + +import profiledb.model.Measurement; +import profiledb.model.Type; + +import java.util.Objects; + +/** + * {@link Measurement} implementation for test purposes. + */ +@Type("test-mem") +public class TestMemoryMeasurement extends Measurement { + + private long timestamp; + + private long usedMb; + + public TestMemoryMeasurement(String id, long timestamp, long usedMb) { + super(id); + this.timestamp = timestamp; + this.usedMb = usedMb; + } + + public long getTimestamp() { + return timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + public long getUsedMb() { + return usedMb; + } + + public void setUsedMb(long usedMb) { + this.usedMb = usedMb; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + TestMemoryMeasurement that = (TestMemoryMeasurement) o; + return timestamp == that.timestamp && + usedMb == that.usedMb; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), timestamp, usedMb); + } + + @Override + public String toString() { + return "TestMemoryMeasurement{" + + "timestamp=" + timestamp + + ", usedMb=" + usedMb + + '}'; + } +} \ No newline at end of file diff --git a/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/measurement/TestTimeMeasurement.java b/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/measurement/TestTimeMeasurement.java new file mode 100644 index 0000000..2f2b037 --- /dev/null +++ b/wayang-commons/wayang-utils/wayang-profile-db/src/test/java/profiledb/measurement/TestTimeMeasurement.java @@ -0,0 +1,56 @@ +package profiledb.measurement; + +import profiledb.model.Measurement; +import profiledb.model.Type; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.Objects; + +/** + * {@link Measurement} implementation for test purposes. + */ +@Type("test-time") +public class TestTimeMeasurement extends Measurement { + + private long millis; + + private Collection<Measurement> submeasurements; + + public TestTimeMeasurement(String id, long millis) { + super(id); + this.millis = millis; + this.submeasurements = new LinkedList<>(); + } + + public long getMillis() { + return millis; + } + + public void setMillis(long millis) { + this.millis = millis; + } + + public Collection<Measurement> getSubmeasurements() { + return submeasurements; + } + + public void addSubmeasurements(TestTimeMeasurement submeasurements) { + this.submeasurements.add(submeasurements); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + TestTimeMeasurement that = (TestTimeMeasurement) o; + return millis == that.millis && + Objects.equals(submeasurements, that.submeasurements); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), millis, submeasurements); + } +} \ No newline at end of file
