Recent round of additions for latest benchmark tests. - New support queries. - New performance logging script. - Bug fixes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/commit/679a9f6b Tree: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/tree/679a9f6b Diff: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/diff/679a9f6b Branch: refs/heads/prestonc/benchmark Commit: 679a9f6ba2351108165eee75645fd766a7b65c7c Parents: a82fa16 Author: Preston Carman <[email protected]> Authored: Tue May 6 12:59:39 2014 -0700 Committer: Preston Carman <[email protected]> Committed: Thu May 8 14:15:35 2014 -0700 ---------------------------------------------------------------------- .../noaa-ghcn-daily/queries/q07_tmax_values.xq | 24 ++++++++++++++ .../noaa-ghcn-daily/queries/q07_tmin_self.xq | 33 ++++++++++++++++++ .../noaa-ghcn-daily/queries/q07_tmin_values.xq | 24 ++++++++++++++ .../noaa-ghcn-daily/scripts/run_group_test.sh | 35 ++++++++++++++++++++ .../scripts/weather_benchmark.py | 2 +- .../src/main/resources/util/log_top.sh | 17 ++++++++++ 6 files changed, 134 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/679a9f6b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/queries/q07_tmax_values.xq ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/queries/q07_tmax_values.xq b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/queries/q07_tmax_values.xq new file mode 100644 index 0000000..2b94827 --- /dev/null +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/queries/q07_tmax_values.xq @@ -0,0 +1,24 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(: XQuery Join Query :) +(: Find the all the records for TMAX. :) +let $sensor_collection_max := "/tmp/1.0_partition_ghcnd_all_xml/sensors" +for $r_max in collection($sensor_collection_max)/dataCollection/data + +where $r_max/dataType eq "TMAX" +return fn:concat($r_max/station, ",", $r_max/date, ",", $r_max/dataType) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/679a9f6b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/queries/q07_tmin_self.xq ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/queries/q07_tmin_self.xq b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/queries/q07_tmin_self.xq new file mode 100644 index 0000000..6eb9f70 --- /dev/null +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/queries/q07_tmin_self.xq @@ -0,0 +1,33 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(: XQuery Self Join Query :) +(: Self join with all stations finding the difference in min and max :) +(: temperature and get the average. :) +fn:count( + let $sensor_collection_min := "/tmp/1.0_partition_ghcnd_all_xml/sensors" + for $r_min in collection($sensor_collection_min)/dataCollection/data + + let $sensor_collection_max := "/tmp/1.0_partition_ghcnd_all_xml/sensors" + for $r_max in collection($sensor_collection_max)/dataCollection/data + + where $r_min/station eq $r_max/station + and $r_min/date eq $r_max/date + and $r_min/dataType eq "TMIN" + and $r_max/dataType eq "TMIN" + return $r_max/station +) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/679a9f6b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/queries/q07_tmin_values.xq ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/queries/q07_tmin_values.xq b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/queries/q07_tmin_values.xq new file mode 100644 index 0000000..fa390a0 --- /dev/null +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/queries/q07_tmin_values.xq @@ -0,0 +1,24 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(: XQuery Join Query :) +(: Find the all the records for TMIN. :) +let $sensor_collection_min := "/tmp/1.0_partition_ghcnd_all_xml/sensors" +for $r_min in collection($sensor_collection_min)/dataCollection/data + +where $r_min/dataType eq "TMIN" +return fn:concat($r_min/station, ",", $r_min/date, ",", $r_min/dataType) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/679a9f6b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/run_group_test.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/run_group_test.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/run_group_test.sh new file mode 100644 index 0000000..b6ff22f --- /dev/null +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/run_group_test.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cluster_ip=${1} + +for n in 2 1 0 +do + for t in "batch_scale_out" "speed_up" + do + for p in 2 1 + do + for c in 4 + do + echo " ==== node ${n} test ${t} partition ${p} cores ${c} ====" + sh noaa-ghcn-daily/scripts/run_benchmark_cluster.sh weather_data/dataset-small-d2/queries/${t}/${n}nodes/d2_p${p}/ ${n} "-client-net-ip-address ${cluster_ip} -available-processors ${c}" + done + done + done +done http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/679a9f6b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py index 1b44155..daae6b2 100644 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py @@ -188,7 +188,7 @@ class WeatherBenchmark: if test in self.BENCHMARK_LOCAL_TESTS: for i in self.partitions: scheme = self.get_local_partition_scheme(test, i) - self.build_data_links_scheme(scheme) + self.build_data_links_scheme(scheme, False) if 1 in self.partitions and len(self.base_paths) > 1: scheme = self.build_data_links_local_zero_partition(test) self.build_data_links_scheme(scheme, False) http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/679a9f6b/vxquery-benchmark/src/main/resources/util/log_top.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/util/log_top.sh b/vxquery-benchmark/src/main/resources/util/log_top.sh new file mode 100644 index 0000000..6e54b0c --- /dev/null +++ b/vxquery-benchmark/src/main/resources/util/log_top.sh @@ -0,0 +1,17 @@ +#!/bin/bash +LOG_FILE=logs/top.log + +# Reset counters +iostat >> /dev/null +sar -n DEV 1 1 >> /dev/null + +# Save IO, CPU and Network snapshot to a log file. +while (sleep 10) +do + echo "---------------------------------------------" >> ${LOG_FILE} + date >> ${LOG_FILE} + echo >> ${LOG_FILE} + iostat >> ${LOG_FILE} + top -n 1 -b | head -11 | tail -6 >> ${LOG_FILE} + sar -n DEV 1 1 >> ${LOG_FILE} +done; \ No newline at end of file
