asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ima...@apache.org
Subject incubator-asterixdb-hyracks git commit: Merge changes necessary for asterix-experiments pkg
Date Wed, 16 Mar 2016 00:34:41 GMT
Repository: incubator-asterixdb-hyracks
Updated Branches:
  refs/heads/master 7dabc19f7 -> 1a2e95f07


Merge changes necessary for asterix-experiments pkg

Change-Id: I1256608bbb97d747747325c2050d5a2dc89beeb0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/641
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <imaxon@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/commit/1a2e95f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/tree/1a2e95f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/diff/1a2e95f0

Branch: refs/heads/master
Commit: 1a2e95f075b0714f6f98468819af520bef4e6dcc
Parents: 7dabc19
Author: Ian Maxon <imaxon@apache.org>
Authored: Thu Feb 18 13:51:55 2016 -0800
Committer: Ian Maxon <imaxon@apache.org>
Committed: Tue Mar 15 17:29:26 2016 -0700

----------------------------------------------------------------------
 .../hyracks/api/util/ExecutionTimeProfiler.java | 111 +++++++++++++++++++
 .../api/util/ExecutionTimeStopWatch.java        |  98 ++++++++++++++++
 .../hyracks/api/util/ExperimentProfiler.java    |  81 ++++++++++++++
 .../api/util/ExperimentProfilerUtils.java       |  47 ++++++++
 .../api/util/OperatorExecutionTimeProfiler.java |  41 +++++++
 .../hyracks/api/util/SpatialIndexProfiler.java  |  53 +++++++++
 .../org/apache/hyracks/api/util/StopWatch.java  |  50 +++++++++
 7 files changed, 481 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a2e95f0/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExecutionTimeProfiler.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExecutionTimeProfiler.java
b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExecutionTimeProfiler.java
new file mode 100644
index 0000000..7673f2c
--- /dev/null
+++ b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExecutionTimeProfiler.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.util;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class ExecutionTimeProfiler {
+
+    public static final boolean PROFILE_MODE = false;
+    public static final String INIT = "init";
+    private FileOutputStream fos;
+    private String filePath;
+    private StringBuilder sb;
+    private int printInterval;
+    private int addCount;
+    private Object lock1 = new Object();
+
+
+    // [Key: Job, Value: [Key: Operator, Value: Duration of each operators]]
+    private HashMap<String, LinkedHashMap<String, String>> spentTimePerJobMap;
+
+    public ExecutionTimeProfiler(String filePath, int printInterval) {
+        this.filePath = new String(filePath);
+        this.sb = new StringBuilder();
+        this.printInterval = printInterval;
+        this.spentTimePerJobMap = new HashMap<String, LinkedHashMap<String, String>>();
+    }
+
+    public void begin() {
+        try {
+            fos = ExperimentProfilerUtils.openOutputFile(filePath);
+            addCount = 0;
+        } catch (IOException e) {
+            e.printStackTrace();
+            throw new IllegalStateException(e);
+        }
+    }
+
+    public synchronized void add(String jobSignature, String operatorSignature, String message,
boolean flushNeeded) {
+
+        if (!spentTimePerJobMap.containsKey(jobSignature)) {
+            spentTimePerJobMap.put(jobSignature, new LinkedHashMap<String, String>());
+        }
+        spentTimePerJobMap.get(jobSignature).put(operatorSignature, message);
+
+        if (flushNeeded) {
+            flush(jobSignature);
+        }
+    }
+
+    public synchronized void flush(String jobSignature) {
+        try {
+            synchronized (lock1) {
+                sb.append("\n\n");
+                for (Map.Entry<String, String> entry : spentTimePerJobMap.get(jobSignature).entrySet())
{
+                    sb.append(entry.getValue());
+                }
+                fos.write(sb.toString().getBytes());
+                fos.flush();
+                spentTimePerJobMap.get(jobSignature).clear();
+                sb.setLength(0);
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+            throw new IllegalStateException(e);
+        }
+    }
+
+    public void clear() {
+        spentTimePerJobMap.clear();
+        sb.setLength(0);
+    }
+
+    public void clear(String jobSignature) {
+        spentTimePerJobMap.get(jobSignature).clear();
+        sb.setLength(0);
+    }
+
+    public synchronized void end() {
+        try {
+            if (fos != null) {
+                fos.flush();
+                fos.close();
+                fos = null;
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+            throw new IllegalStateException(e);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a2e95f0/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExecutionTimeStopWatch.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExecutionTimeStopWatch.java
b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExecutionTimeStopWatch.java
new file mode 100644
index 0000000..4613fd5
--- /dev/null
+++ b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExecutionTimeStopWatch.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.util;
+
+import java.text.SimpleDateFormat;
+
+public class ExecutionTimeStopWatch {
+    private long startTime = 0;
+    private long stopTime = 0;
+    private long elapsedTime = 0;
+
+    private long elapsedTimeBetweenTimeStamp = 0;
+
+    // starting timestamp of an operator
+    private long startTimeStamp = 0;
+
+    // ending timestamp
+    private long endTimeStamp = 0;
+
+    // The timer has started?
+    private boolean isStarted = false;
+
+    private String message;
+
+    public void start() {
+        elapsedTime = 0;
+        startTime = System.currentTimeMillis();
+        startTimeStamp = startTime;
+        isStarted = true;
+        message = "";
+    }
+
+    public void suspend() {
+        stopTime = System.currentTimeMillis();
+        elapsedTime += stopTime - startTime;
+    }
+
+    public void resume() {
+        startTime = System.currentTimeMillis();
+    }
+
+    public void finish() {
+        endTimeStamp = stopTime;
+        elapsedTimeBetweenTimeStamp = endTimeStamp - startTimeStamp;
+    }
+
+    // elapsed time in milliseconds
+    public long getElapsedTime() {
+        return elapsedTime;
+    }
+
+    // elapsed time in seconds
+    public double getElapsedTimeSecs() {
+        return (double) elapsedTime / 1000;
+    }
+
+    // elapsed time in milliseconds
+    public long getElapsedTimeStamp() {
+        return elapsedTimeBetweenTimeStamp;
+    }
+
+    // elapsed time in seconds
+    public double getElapsedTimeStampSecs() {
+        return (double) elapsedTimeBetweenTimeStamp / 1000;
+    }
+
+    public String getMessage(String operatorName, long timeStamp) {
+        message = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS").format(timeStamp) + "\t"
+ operatorName + "\t"
+                + this.getElapsedTime() + "\t" + this.getElapsedTimeSecs() + "\t" + this.getElapsedTimeStamp()
+ "\t"
+                + this.getElapsedTimeStampSecs() + "\n";
+        return message;
+    }
+
+    public long getStartTimeStamp() {
+        return startTimeStamp;
+    }
+
+    public long getEndTimeStamp() {
+        return endTimeStamp;
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a2e95f0/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExperimentProfiler.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExperimentProfiler.java
b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExperimentProfiler.java
new file mode 100644
index 0000000..b98d7d3
--- /dev/null
+++ b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExperimentProfiler.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.api.util;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+public class ExperimentProfiler {
+
+    public static final boolean PROFILE_MODE = false;
+    private FileOutputStream fos;
+    private String filePath;
+    private StringBuilder sb;
+    private int printInterval;
+    private int addCount;
+
+    public ExperimentProfiler(String filePath, int printInterval) {
+        this.filePath = new String(filePath);
+        this.sb = new StringBuilder();
+        this.printInterval = printInterval;
+    }
+
+    public void begin() {
+        try {
+            fos = ExperimentProfilerUtils.openOutputFile(filePath);
+            addCount = 0;
+        } catch (IOException e) {
+            e.printStackTrace();
+            throw new IllegalStateException(e);
+        }
+    }
+
+    public synchronized void add(String s) {
+        sb.append(s);
+        if (printInterval > 0 && ++addCount % printInterval == 0) {
+            flush();
+            addCount = 0;
+        }
+    }
+
+    public synchronized void flush() {
+        try {
+            fos.write(sb.toString().getBytes());
+            fos.flush();
+            sb.setLength(0);
+        } catch (IOException e) {
+            e.printStackTrace();
+            throw new IllegalStateException(e);
+        }
+    }
+
+    public synchronized void end() {
+        try {
+            if (fos != null) {
+                fos.flush();
+                fos.close();
+                fos = null;
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+            throw new IllegalStateException(e);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a2e95f0/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExperimentProfilerUtils.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExperimentProfilerUtils.java
b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExperimentProfilerUtils.java
new file mode 100644
index 0000000..2305573
--- /dev/null
+++ b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/ExperimentProfilerUtils.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.api.util;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class ExperimentProfilerUtils {
+    public static void printToOutputFile(StringBuffer sb, FileOutputStream fos) throws IllegalStateException,
+            IOException {
+        fos.write(sb.toString().getBytes());
+    }
+
+    public static FileOutputStream openOutputFile(String filepath) throws IOException {
+        File file = new File(filepath);
+        if (file.exists()) {
+            file.delete();
+        }
+        file.createNewFile();
+        return new FileOutputStream(file);
+    }
+
+    public static void closeOutputFile(OutputStream os) throws IOException {
+        os.flush();
+        os.close();
+        os = null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a2e95f0/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/OperatorExecutionTimeProfiler.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/OperatorExecutionTimeProfiler.java
b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/OperatorExecutionTimeProfiler.java
new file mode 100644
index 0000000..b5d632b
--- /dev/null
+++ b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/OperatorExecutionTimeProfiler.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.util;
+
+import java.net.Inet4Address;
+import java.net.UnknownHostException;
+
+public class OperatorExecutionTimeProfiler {
+    public static final OperatorExecutionTimeProfiler INSTANCE = new OperatorExecutionTimeProfiler();
+    public ExecutionTimeProfiler executionTimeProfiler;
+
+    private OperatorExecutionTimeProfiler() {
+        String profileHomeDir = SpatialIndexProfiler.PROFILE_HOME_DIR;
+        if (ExecutionTimeProfiler.PROFILE_MODE) {
+            //SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS");
+            try {
+                executionTimeProfiler = new ExecutionTimeProfiler(profileHomeDir + "executionTime-"
+                        + Inet4Address.getLocalHost().getHostAddress() + ".txt", 1);
+            } catch (UnknownHostException e) {
+                e.printStackTrace();
+            }
+            executionTimeProfiler.begin();
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a2e95f0/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/SpatialIndexProfiler.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/SpatialIndexProfiler.java
b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/SpatialIndexProfiler.java
new file mode 100644
index 0000000..a1b40d0
--- /dev/null
+++ b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/SpatialIndexProfiler.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.api.util;
+
+import java.net.Inet4Address;
+import java.net.UnknownHostException;
+
+public class SpatialIndexProfiler {
+    public static final SpatialIndexProfiler INSTANCE = new SpatialIndexProfiler();
+    public static final String PROFILE_HOME_DIR = "/data/seok.kim/spatial-index-experiment/asterix-instance/logs/";
+    //    public static final String PROFILE_HOME_DIR = "/Volumes/MyPassport/workspace/spatial-index-experiment/asterix-instance/logs/";
+    public ExperimentProfiler falsePositivePerQuery;
+    public ExperimentProfiler cacheMissPerQuery;
+
+    private SpatialIndexProfiler() {
+        if (ExperimentProfiler.PROFILE_MODE) {
+            //SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH:mm:ss.SSS");
+            try {
+                falsePositivePerQuery = new ExperimentProfiler(PROFILE_HOME_DIR + "falsePositivePerQuery-"
+                        + Inet4Address.getLocalHost().getHostAddress() + ".txt", 1);
+            } catch (UnknownHostException e) {
+                // TODO Auto-generated catch block
+                e.printStackTrace();
+            }
+            falsePositivePerQuery.begin();
+            try {
+                cacheMissPerQuery = new ExperimentProfiler(PROFILE_HOME_DIR + "cacheMissPerQuery-"
+                        + Inet4Address.getLocalHost().getHostAddress() + ".txt", 1);
+            } catch (UnknownHostException e) {
+                // TODO Auto-generated catch block
+                e.printStackTrace();
+            }
+            cacheMissPerQuery.begin();
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a2e95f0/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/StopWatch.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/StopWatch.java
b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/StopWatch.java
new file mode 100644
index 0000000..e752cb3
--- /dev/null
+++ b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/StopWatch.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.api.util;
+
+public class StopWatch {
+    private long startTime = 0;
+    private long stopTime = 0;
+    private long elapsedTime = 0;
+
+    public void start() {
+        elapsedTime = 0;
+        startTime = System.currentTimeMillis();
+    }
+
+    public void stop() {
+        stopTime = System.currentTimeMillis();
+        elapsedTime += stopTime - startTime;
+    }
+
+    public void resume() {
+        startTime = System.currentTimeMillis();
+    }
+
+    //elaspsed time in milliseconds
+    public long getElapsedTime() {
+        return elapsedTime;
+    }
+
+    //elaspsed time in seconds
+    public long getElapsedTimeSecs() {
+        return elapsedTime / 1000;
+    }
+}
\ No newline at end of file


Mime
View raw message