Ian Maxon has submitted this change and it was merged. Change subject: Merge changes necessary for asterix-experiments pkg ......................................................................
Merge changes necessary for asterix-experiments pkg Change-Id: I1256608bbb97d747747325c2050d5a2dc89beeb0 Reviewed-on: https://asterix-gerrit.ics.uci.edu/641 Tested-by: Jenkins <[email protected]> Reviewed-by: Ian Maxon <[email protected]> --- A hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExecutionTimeProfiler.java A hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExecutionTimeStopWatch.java A hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExperimentProfiler.java A hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExperimentProfilerUtils.java A hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/OperatorExecutionTimeProfiler.java A hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/SpatialIndexProfiler.java A hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/StopWatch.java 7 files changed, 481 insertions(+), 0 deletions(-) Approvals: Ian Maxon: Looks good to me, approved Jenkins: Verified diff --git a/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExecutionTimeProfiler.java b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExecutionTimeProfiler.java new file mode 100644 index 0000000..7673f2c --- /dev/null +++ b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExecutionTimeProfiler.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.api.util; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +public class ExecutionTimeProfiler { + + public static final boolean PROFILE_MODE = false; + public static final String INIT = "init"; + private FileOutputStream fos; + private String filePath; + private StringBuilder sb; + private int printInterval; + private int addCount; + private Object lock1 = new Object(); + + + // [Key: Job, Value: [Key: Operator, Value: Duration of each operators]] + private HashMap<String, LinkedHashMap<String, String>> spentTimePerJobMap; + + public ExecutionTimeProfiler(String filePath, int printInterval) { + this.filePath = new String(filePath); + this.sb = new StringBuilder(); + this.printInterval = printInterval; + this.spentTimePerJobMap = new HashMap<String, LinkedHashMap<String, String>>(); + } + + public void begin() { + try { + fos = ExperimentProfilerUtils.openOutputFile(filePath); + addCount = 0; + } catch (IOException e) { + e.printStackTrace(); + throw new IllegalStateException(e); + } + } + + public synchronized void add(String jobSignature, String operatorSignature, String message, boolean flushNeeded) { + + if (!spentTimePerJobMap.containsKey(jobSignature)) { + spentTimePerJobMap.put(jobSignature, new LinkedHashMap<String, String>()); + } + spentTimePerJobMap.get(jobSignature).put(operatorSignature, message); + + if (flushNeeded) { + flush(jobSignature); + } + } + + public synchronized void flush(String jobSignature) { + try { + synchronized (lock1) { + sb.append("\n\n"); + for (Map.Entry<String, String> entry : spentTimePerJobMap.get(jobSignature).entrySet()) { + sb.append(entry.getValue()); + } + fos.write(sb.toString().getBytes()); + fos.flush(); + spentTimePerJobMap.get(jobSignature).clear(); + sb.setLength(0); + } + } catch (IOException e) { + e.printStackTrace(); + throw new IllegalStateException(e); + } + } + + public void clear() { + spentTimePerJobMap.clear(); + sb.setLength(0); + } + + public void clear(String jobSignature) { + spentTimePerJobMap.get(jobSignature).clear(); + sb.setLength(0); + } + + public synchronized void end() { + try { + if (fos != null) { + fos.flush(); + fos.close(); + fos = null; + } + } catch (IOException e) { + e.printStackTrace(); + throw new IllegalStateException(e); + } + } +} diff --git a/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExecutionTimeStopWatch.java b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExecutionTimeStopWatch.java new file mode 100644 index 0000000..4613fd5 --- /dev/null +++ b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExecutionTimeStopWatch.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.api.util; + +import java.text.SimpleDateFormat; + +public class ExecutionTimeStopWatch { + private long startTime = 0; + private long stopTime = 0; + private long elapsedTime = 0; + + private long elapsedTimeBetweenTimeStamp = 0; + + // starting timestamp of an operator + private long startTimeStamp = 0; + + // ending timestamp + private long endTimeStamp = 0; + + // The timer has started? + private boolean isStarted = false; + + private String message; + + public void start() { + elapsedTime = 0; + startTime = System.currentTimeMillis(); + startTimeStamp = startTime; + isStarted = true; + message = ""; + } + + public void suspend() { + stopTime = System.currentTimeMillis(); + elapsedTime += stopTime - startTime; + } + + public void resume() { + startTime = System.currentTimeMillis(); + } + + public void finish() { + endTimeStamp = stopTime; + elapsedTimeBetweenTimeStamp = endTimeStamp - startTimeStamp; + } + + // elapsed time in milliseconds + public long getElapsedTime() { + return elapsedTime; + } + + // elapsed time in seconds + public double getElapsedTimeSecs() { + return (double) elapsedTime / 1000; + } + + // elapsed time in milliseconds + public long getElapsedTimeStamp() { + return elapsedTimeBetweenTimeStamp; + } + + // elapsed time in seconds + public double getElapsedTimeStampSecs() { + return (double) elapsedTimeBetweenTimeStamp / 1000; + } + + public String getMessage(String operatorName, long timeStamp) { + message = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS").format(timeStamp) + "\t" + operatorName + "\t" + + this.getElapsedTime() + "\t" + this.getElapsedTimeSecs() + "\t" + this.getElapsedTimeStamp() + "\t" + + this.getElapsedTimeStampSecs() + "\n"; + return message; + } + + public long getStartTimeStamp() { + return startTimeStamp; + } + + public long getEndTimeStamp() { + return endTimeStamp; + } + +} \ No newline at end of file diff --git a/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExperimentProfiler.java b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExperimentProfiler.java new file mode 100644 index 0000000..b98d7d3 --- /dev/null +++ b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExperimentProfiler.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.api.util; + +import java.io.FileOutputStream; +import java.io.IOException; + +public class ExperimentProfiler { + + public static final boolean PROFILE_MODE = false; + private FileOutputStream fos; + private String filePath; + private StringBuilder sb; + private int printInterval; + private int addCount; + + public ExperimentProfiler(String filePath, int printInterval) { + this.filePath = new String(filePath); + this.sb = new StringBuilder(); + this.printInterval = printInterval; + } + + public void begin() { + try { + fos = ExperimentProfilerUtils.openOutputFile(filePath); + addCount = 0; + } catch (IOException e) { + e.printStackTrace(); + throw new IllegalStateException(e); + } + } + + public synchronized void add(String s) { + sb.append(s); + if (printInterval > 0 && ++addCount % printInterval == 0) { + flush(); + addCount = 0; + } + } + + public synchronized void flush() { + try { + fos.write(sb.toString().getBytes()); + fos.flush(); + sb.setLength(0); + } catch (IOException e) { + e.printStackTrace(); + throw new IllegalStateException(e); + } + } + + public synchronized void end() { + try { + if (fos != null) { + fos.flush(); + fos.close(); + fos = null; + } + } catch (IOException e) { + e.printStackTrace(); + throw new IllegalStateException(e); + } + } +} diff --git a/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExperimentProfilerUtils.java b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExperimentProfilerUtils.java new file mode 100644 index 0000000..2305573 --- /dev/null +++ b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExperimentProfilerUtils.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.api.util; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +public class ExperimentProfilerUtils { + public static void printToOutputFile(StringBuffer sb, FileOutputStream fos) throws IllegalStateException, + IOException { + fos.write(sb.toString().getBytes()); + } + + public static FileOutputStream openOutputFile(String filepath) throws IOException { + File file = new File(filepath); + if (file.exists()) { + file.delete(); + } + file.createNewFile(); + return new FileOutputStream(file); + } + + public static void closeOutputFile(OutputStream os) throws IOException { + os.flush(); + os.close(); + os = null; + } +} diff --git a/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/OperatorExecutionTimeProfiler.java b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/OperatorExecutionTimeProfiler.java new file mode 100644 index 0000000..b5d632b --- /dev/null +++ b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/OperatorExecutionTimeProfiler.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.api.util; + +import java.net.Inet4Address; +import java.net.UnknownHostException; + +public class OperatorExecutionTimeProfiler { + public static final OperatorExecutionTimeProfiler INSTANCE = new OperatorExecutionTimeProfiler(); + public ExecutionTimeProfiler executionTimeProfiler; + + private OperatorExecutionTimeProfiler() { + String profileHomeDir = SpatialIndexProfiler.PROFILE_HOME_DIR; + if (ExecutionTimeProfiler.PROFILE_MODE) { + //SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS"); + try { + executionTimeProfiler = new ExecutionTimeProfiler(profileHomeDir + "executionTime-" + + Inet4Address.getLocalHost().getHostAddress() + ".txt", 1); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + executionTimeProfiler.begin(); + } + } +} diff --git a/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/SpatialIndexProfiler.java b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/SpatialIndexProfiler.java new file mode 100644 index 0000000..a1b40d0 --- /dev/null +++ b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/SpatialIndexProfiler.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.api.util; + +import java.net.Inet4Address; +import java.net.UnknownHostException; + +public class SpatialIndexProfiler { + public static final SpatialIndexProfiler INSTANCE = new SpatialIndexProfiler(); + public static final String PROFILE_HOME_DIR = "/data/seok.kim/spatial-index-experiment/asterix-instance/logs/"; + // public static final String PROFILE_HOME_DIR = "/Volumes/MyPassport/workspace/spatial-index-experiment/asterix-instance/logs/"; + public ExperimentProfiler falsePositivePerQuery; + public ExperimentProfiler cacheMissPerQuery; + + private SpatialIndexProfiler() { + if (ExperimentProfiler.PROFILE_MODE) { + //SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH:mm:ss.SSS"); + try { + falsePositivePerQuery = new ExperimentProfiler(PROFILE_HOME_DIR + "falsePositivePerQuery-" + + Inet4Address.getLocalHost().getHostAddress() + ".txt", 1); + } catch (UnknownHostException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + falsePositivePerQuery.begin(); + try { + cacheMissPerQuery = new ExperimentProfiler(PROFILE_HOME_DIR + "cacheMissPerQuery-" + + Inet4Address.getLocalHost().getHostAddress() + ".txt", 1); + } catch (UnknownHostException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + cacheMissPerQuery.begin(); + } + } +} diff --git a/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/StopWatch.java b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/StopWatch.java new file mode 100644 index 0000000..e752cb3 --- /dev/null +++ b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/StopWatch.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hyracks.api.util; + +public class StopWatch { + private long startTime = 0; + private long stopTime = 0; + private long elapsedTime = 0; + + public void start() { + elapsedTime = 0; + startTime = System.currentTimeMillis(); + } + + public void stop() { + stopTime = System.currentTimeMillis(); + elapsedTime += stopTime - startTime; + } + + public void resume() { + startTime = System.currentTimeMillis(); + } + + //elaspsed time in milliseconds + public long getElapsedTime() { + return elapsedTime; + } + + //elaspsed time in seconds + public long getElapsedTimeSecs() { + return elapsedTime / 1000; + } +} \ No newline at end of file -- To view, visit https://asterix-gerrit.ics.uci.edu/641 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I1256608bbb97d747747325c2050d5a2dc89beeb0 Gerrit-PatchSet: 7 Gerrit-Project: hyracks Gerrit-Branch: master Gerrit-Owner: Ian Maxon <[email protected]> Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Ildar Absalyamov <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: Young-Seok Kim <[email protected]>
