Preparing for local speed up tests. I removed the queries that are currently not working. Also fixed a bug referencing files directly in the query instead of through the data links. Removed some debug code.
Project: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/commit/575bcfd5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/tree/575bcfd5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/diff/575bcfd5 Branch: refs/heads/prestonc/benchmarks_staging Commit: 575bcfd58afe64e96d240f9356fc03d03b06f5a2 Parents: 4d6fde6 Author: Preston Carman <[email protected]> Authored: Thu Feb 27 14:21:00 2014 -0800 Committer: Preston Carman <[email protected]> Committed: Thu Feb 27 14:21:00 2014 -0800 ---------------------------------------------------------------------- .../noaa-ghcn-daily/scripts/weather_benchmark.py | 16 +++++++--------- .../noaa-ghcn-daily/scripts/weather_example.xml | 2 ++ 2 files changed, 9 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/575bcfd5/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py index 6d9301e..f0d93f8 100644 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py @@ -35,8 +35,9 @@ class WeatherBenchmark: QUERY_REPLACEMENT_KEY = "/tmp/1.0_partition_ghcnd_all_xml/" QUERY_MASTER_FOLDER = "../queries/" - QUERY_FILE_LIST = ["q00.xq", "q01.xq", "q02.xq", "q03.xq", "q04.xq", "q05.xq"] - BENCHMARK_LOCAL_TESTS = ["local_speed_up"] + QUERY_FILE_LIST = ["q00.xq", "q01.xq", "q02.xq", "q03.xq"] +# QUERY_FILE_LIST = ["q00.xq", "q01.xq", "q02.xq", "q03.xq", "q04.xq", "q05.xq"] + BENCHMARK_LOCAL_TESTS = ["local_speed_up", "local_batch_scale_out"] BENCHMARK_CLUSTER_TESTS = ["speed_up", "batch_scale_out"] QUERY_COLLECTIONS = ["sensors", "stations"] @@ -68,12 +69,8 @@ class WeatherBenchmark: # Match link paths to real data paths. offset = 0 group_size = len(data_paths) / len(link_base_paths) - print "g " + str(group_size) - print link_base_paths - print data_paths for link_index, link_path in enumerate(link_base_paths): for data_index, data_path in enumerate(data_paths): - print index, offset, group_size, link_index, data_index if offset <= data_index and data_index < offset + group_size: self.add_collection_links_for(data_path, link_path, data_index) offset += group_size @@ -161,8 +158,8 @@ class WeatherBenchmark: os.makedirs(query_path) # Copy query files. - node_partitions = get_partition_paths(partitions, self.base_paths, "data_links/" + test + "/" + str(i) + "nodes") - self.copy_and_replace_query(query_path, node_partitions) + partition_paths = get_partition_paths(partitions, self.base_paths, "data_links/" + test + "/" + str(i) + "nodes") + self.copy_and_replace_query(query_path, partition_paths) def copy_local_query_files(self, test): for i in self.partitions: @@ -172,7 +169,8 @@ class WeatherBenchmark: os.makedirs(query_path) # Copy query files. - self.copy_and_replace_query(query_path, get_partition_paths(i, self.base_paths)) + partition_paths = get_partition_paths(i, self.base_paths, "data_links/" + test) + self.copy_and_replace_query(query_path, partition_paths) def copy_and_replace_query(self, query_path, replacement_list): '''Copy the query files over to the query_path and replace the path http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/575bcfd5/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml index 4f31dff..daf8f20 100644 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml @@ -35,6 +35,8 @@ <name>small</name> <test>local_speed_up</test> <save_path>/data</save_path> + <partitions_per_path>1</partitions_per_path> + <partitions_per_path>2</partitions_per_path> <partitions_per_path>4</partitions_per_path> <partitions_per_path>8</partitions_per_path> </dataset>
