MRQL script for FLINK.
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/0b7dfa06 Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/0b7dfa06 Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/0b7dfa06 Branch: refs/heads/master Commit: 0b7dfa06d40f69befa36baff618f2aa0f3247b2d Parents: 32f9fcd Author: Preston Carman <[email protected]> Authored: Wed Oct 22 13:50:54 2014 -0700 Committer: Eldon Carman <[email protected]> Committed: Thu Jun 18 13:13:19 2015 -0700 ---------------------------------------------------------------------- .../other_systems/mrql_scripts/clear.sh | 24 ------------- .../other_systems/mrql_scripts/clear_hadoop.sh | 24 +++++++++++++ .../other_systems/mrql_scripts/clear_hadoop2.sh | 22 ++++++++++++ .../mrql_scripts/run_group_test.sh | 36 +++++++++++++------- .../mrql_scripts/run_mrql_tests.sh | 7 ++-- 5 files changed, 71 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/0b7dfa06/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh deleted file mode 100755 index 9a912b2..0000000 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Remove data -rm -rf disk1/hadoop/data -rm -rf disk2/hadoop/data -rm -rf disk1/hadoop/tmp -rm -rf disk1/hadoop/logs http://git-wip-us.apache.org/repos/asf/vxquery/blob/0b7dfa06/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear_hadoop.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear_hadoop.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear_hadoop.sh new file mode 100755 index 0000000..9a912b2 --- /dev/null +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear_hadoop.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Remove data +rm -rf disk1/hadoop/data +rm -rf disk2/hadoop/data +rm -rf disk1/hadoop/tmp +rm -rf disk1/hadoop/logs http://git-wip-us.apache.org/repos/asf/vxquery/blob/0b7dfa06/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear_hadoop2.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear_hadoop2.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear_hadoop2.sh new file mode 100755 index 0000000..fbc15b5 --- /dev/null +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/clear_hadoop2.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Remove data +rm -rf disk1/hadoop2/hdfs +rm -rf disk2/hadoop2/hdfs http://git-wip-us.apache.org/repos/asf/vxquery/blob/0b7dfa06/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh index 84028e5..9c2f3ad 100755 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_group_test.sh @@ -33,13 +33,21 @@ DATASET=${1} NODES=${2} REPEAT=1 DATA_FILES=${NODES} -#DATA_FILES=8 # Start Hadoop -sh saved/hadoop/hadoop-1.2.1/bin/start-all.sh +# sh saved/hadoop/hadoop-1.2.1/bin/start-all.sh +sh saved/hadoop/hadoop-2.5.1/sbin/hadoop-daemon.sh start namenode +sh saved/hadoop/hadoop-2.5.1/sbin/hadoop-daemons.sh start datanode +sh saved/hadoop/hadoop-2.5.1/sbin/yarn-daemon.sh start resourcemanager +sh saved/hadoop/hadoop-2.5.1/sbin/yarn-daemons.sh start nodemanager +sh saved/hadoop/hadoop-2.5.1/sbin/mr-jobhistory-daemon.sh start historyserver sleep 10 +# Start Flink +sh saved/flink/flink-yarn-0.6.1-incubating/bin/yarn-session.sh -n $((4*${NODES})) -tm 1024 & +FLINK_PID=$! + # Prepare hadoop file system hadoop fs -mkdir ${DATASET} hadoop fs -ls @@ -47,19 +55,12 @@ hadoop fs -mkdir ${DATASET}/sensors hadoop fs -mkdir ${DATASET}/stations hadoop fs -ls ${DATASET} -# Prepare hadoop file system 2 -hadoop fs -mkdir ${DATASET}2 -hadoop fs -ls -hadoop fs -mkdir ${DATASET}2/sensors -hadoop fs -mkdir ${DATASET}2/stations -hadoop fs -ls ${DATASET}2 - hadoop balancer # Upload test data COUNTER=0 -while [ ${COUNTER} -lt ${NODES} ]; +while [ ${COUNTER} -lt ${DATA_FILES} ]; do sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/load_node_file.sh ${DATASET} ${COUNTER} let COUNTER=COUNTER+1 @@ -67,9 +68,18 @@ done # Start test -sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh \ - vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/ ${NODES} ${REPEAT} ${DATASET} +sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql/ ${NODES} ${REPEAT} ${DATASET} + +# Stop Flink +kill ${FLINK_PID} +jobs -p +kill $(jobs -p) # Stop Hadoop -sh saved/hadoop/hadoop-1.2.1/bin/stop-all.sh +# sh saved/hadoop/hadoop-1.2.1/bin/stop-all.sh +sh saved/hadoop/hadoop-2.5.1/sbin/mr-jobhistory-daemon.sh stop historyserver +sh saved/hadoop/hadoop-2.5.1/sbin/yarn-daemons.sh stop nodemanager +sh saved/hadoop/hadoop-2.5.1/sbin/yarn-daemon.sh stop resourcemanager +sh saved/hadoop/hadoop-2.5.1/sbin/hadoop-daemons.sh stop datanode +sh saved/hadoop/hadoop-2.5.1/sbin/hadoop-daemon.sh stop namenode http://git-wip-us.apache.org/repos/asf/vxquery/blob/0b7dfa06/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh index 912cd3b..c000727 100755 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/other_systems/mrql_scripts/run_mrql_tests.sh @@ -23,19 +23,16 @@ NODES=${2} REPEAT=${3} DATASET=${4} -THREADS=$((4*${NODES})) # Make log folder mkdir -p ~/disk1/weather_data/mrql/query_logs/${NODES}nodes/ -for j in $(find ${1} -name '*q?7.mrql') +for j in $(find ${1} -name '*q??.mrql') do date echo "Running MRQL query: ${j}" - time for i in {1..${REPEAT}}; do ~/mrql/incubator-mrql/bin/mrql -dist -nodes ${THREADS} ${j} \ - ${DATASET}/sensors/ ${DATASET}/stations/ ${DATASET}2/sensors/ ${DATASET}2/stations/ \ - >> ~/disk1/weather_data/mrql/query_logs/${NODES}nodes/$(basename "${j}").log 2>&1; done; + time for i in {1..${REPEAT}}; do ~/mrql/incubator-mrql/bin/mrql.flink -dist -nodes ${NODES} ${j} ${DATASET}/sensors/ ${DATASET}/stations/ >> ~/disk1/weather_data/mrql/query_logs/${NODES}nodes/$(basename "${j}").log 2>&1; done; done
