Added scripts to help manage MRQL testing.
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/837a0efb Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/837a0efb Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/837a0efb Branch: refs/heads/prestonc/exrt_benchmark_queries Commit: 837a0efb642bd06a5d268c44753e0fab43bf679d Parents: 630b7ab Author: Preston Carman <[email protected]> Authored: Fri Aug 22 14:08:06 2014 -0700 Committer: Preston Carman <[email protected]> Committed: Mon Sep 22 20:15:07 2014 -0700 ---------------------------------------------------------------------- .../other_systems/mrql_scripts/clear.sh | 19 +++++++++ .../mrql_scripts/run_mrql_tests.sh | 43 ++++++++++++++++++++ .../other_systems/mrql_scripts/start.sh | 21 ++++++++++ .../other_systems/mrql_scripts/stop.sh | 21 ++++++++++ .../scripts/weather_benchmark.py | 6 +-- .../scripts/weather_data_files.py | 14 +------ 6 files changed, 109 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/837a0efb/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh new file mode 100755 index 0000000..b5ea487 --- /dev/null +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +bin/hadoop namenode -format http://git-wip-us.apache.org/repos/asf/vxquery/blob/837a0efb/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh new file mode 100755 index 0000000..831f424 --- /dev/null +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +NODES=${2} +REPEAT=${3} +DATASET="hcn" + + +for j in $(find ${1} -name '*q??.mrql') +do + date + echo "Running MRQL query: ${j}" + time for i in {1..${REPEAT}}; do ~/mrql/incubator-mrql/bin/mrql -dist -nodes ${NODES} ${j} >> ~/disk1/weather_data/mrql/query_logs/$(basename "${j}").log 2>&1; done; +done + + +if which programname >/dev/null; +then + echo "Sending out e-mail notification." + SUBJECT="MRQL Tests Finished (${DATASET})" + EMAIL="[email protected]" + /bin/mail -s "${SUBJECT}" "${EMAIL}" <<EOM + Completed all MRQL tests on ${DATASET}. + EOM +else + echo "No mail command to use." +fi; http://git-wip-us.apache.org/repos/asf/vxquery/blob/837a0efb/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/start.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/start.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/start.sh new file mode 100755 index 0000000..c09e7e7 --- /dev/null +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/start.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +bin/start-dfs.sh +bin/start-mapred.sh http://git-wip-us.apache.org/repos/asf/vxquery/blob/837a0efb/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/stop.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/stop.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/stop.sh new file mode 100755 index 0000000..c4259b8 --- /dev/null +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/stop.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +bin/stop-mapred.sh +bin/stop-dfs.sh http://git-wip-us.apache.org/repos/asf/vxquery/blob/837a0efb/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py index 8021b2c..783a332 100644 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py @@ -120,8 +120,8 @@ class WeatherBenchmark: def get_local_partition_scheme(self, test, partition): scheme = [] virtual_partitions = get_local_virtual_disk_partitions(self.partitions) - data_schemes = get_partition_scheme(0, virtual_partitions, self.base_paths) - link_base_schemes = get_partition_scheme(0, virtual_partitions, self.base_paths, self.DATA_LINKS_FOLDER + test) + data_schemes = get_disk_partition_scheme(0, virtual_partitions, self.base_paths) + link_base_schemes = get_disk_partition_scheme(0, virtual_partitions, self.base_paths, self.DATA_LINKS_FOLDER + test) # Match link paths to real data paths. group_size = len(data_schemes) / len(link_base_schemes) @@ -218,7 +218,7 @@ class WeatherBenchmark: '''Build a scheme for all data in one symbolically linked folder. (0 partition)''' scheme = [] index = 0 - link_base_schemes = get_partition_scheme(0, 1, self.base_paths, self.DATA_LINKS_FOLDER + test) + link_base_schemes = get_disk_partition_scheme(0, 1, self.base_paths, self.DATA_LINKS_FOLDER + test) for link_node, link_disk, link_virtual, link_index, link_path in link_base_schemes: if test == "local_batch_scale_out" and index > 0: continue http://git-wip-us.apache.org/repos/asf/vxquery/blob/837a0efb/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_data_files.py ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_data_files.py b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_data_files.py index 4877120..c8b0fa5 100644 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_data_files.py +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_data_files.py @@ -96,7 +96,7 @@ class WeatherDataFiles: return # Initialize the partition paths. - partition_paths = get_partition_paths(0, partitions, base_paths) + partition_paths = get_disk_partition_paths(0, partitions, base_paths) for path in partition_paths: # Make sure the xml folder is available. prepare_path(path, reset) @@ -150,7 +150,7 @@ class WeatherDataFiles: XML_START = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>" - partition_paths = get_partition_paths(0, partitions, base_paths) + partition_paths = get_disk_partition_paths(0, partitions, base_paths) import fnmatch import os @@ -375,16 +375,6 @@ PARTITION_INDEX_VIRTUAL = 2 PARTITION_INDEX = 3 PARTITION_INDEX_PATH = 4 PARTITION_HEADER = ("Node", "Disk", "Virtual", "Index", "Path") - -def get_partition_paths(node_id, partitions, base_paths, key="partitions"): - partition_paths = [] - for scheme in get_partition_scheme(node_id, partitions, base_paths, key): - partition_paths.append(scheme[PARTITION_INDEX_PATH]) - return partition_paths - -def get_partition_scheme(node_id, virtual_partitions, base_paths, key="partitions"): - partitions_per_disk = virtual_partitions / len(base_paths) - return get_disk_partition_scheme(node_id, partitions_per_disk, base_paths, key) def get_disk_partition_paths(node_id, partitions, base_paths, key="partitions"): partition_paths = []
