More tweaks to make the benchmark cluster queries. * Updated the data links to real partitions. * Updated the query copy process. * Simplified the example config file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/commit/231cdf25 Tree: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/tree/231cdf25 Diff: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/diff/231cdf25 Branch: refs/heads/prestonc/benchmarks_staging Commit: 231cdf25cb75614418ad1bdcae01133c57df9de1 Parents: c72b074 Author: Preston Carman <[email protected]> Authored: Sat Mar 1 23:15:26 2014 -0800 Committer: Preston Carman <[email protected]> Committed: Sat Mar 1 23:15:26 2014 -0800 ---------------------------------------------------------------------- .../noaa-ghcn-daily/scripts/weather_benchmark.py | 18 +++++++++++------- .../noaa-ghcn-daily/scripts/weather_example.xml | 11 +---------- .../metadata/VXQueryCollectionDataSource.java | 1 - 3 files changed, 12 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/231cdf25/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py index f0d93f8..4813bd9 100644 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_benchmark.py @@ -130,11 +130,11 @@ class WeatherBenchmark: def add_collection_links_for(self, real_path, link_path, index): for collection in self.QUERY_COLLECTIONS: - collection_path = link_path + "/" + collection + collection_path = link_path + collection + "/" if not os.path.isdir(collection_path): os.makedirs(collection_path) if index >= 0: - os.symlink(real_path + "/" + collection, collection_path + "/index" + str(index)) + os.symlink(real_path + collection + "/", collection_path + "index" + str(index)) def get_partition_folders(self, base_path): glob.glob(base_path + "partitions/d*_p*_i*") @@ -158,7 +158,7 @@ class WeatherBenchmark: os.makedirs(query_path) # Copy query files. - partition_paths = get_partition_paths(partitions, self.base_paths, "data_links/" + test + "/" + str(i) + "nodes") + partition_paths = get_cluster_link_paths_for_node(i, self.base_paths, "data_links/" + test) self.copy_and_replace_query(query_path, partition_paths) def copy_local_query_files(self, test): @@ -205,12 +205,16 @@ class WeatherBenchmark: def get_cluster_link_paths(nodes, base_paths, key="partitions"): link_paths = [] for i in range(0, nodes): - for j in range(0, len(base_paths)): - new_link_path = base_paths[j] + key + "/" + str(i) + "nodes/" - link_paths.append(new_link_path) + new_link_path = get_cluster_link_paths_for_node(i, base_paths, key) + link_paths.extend(new_link_path) return link_paths - +def get_cluster_link_paths_for_node(node_id, base_paths, key="partitions"): + link_paths = [] + for j in range(0, len(base_paths)): + new_link_path = base_paths[j] + key + "/" + str(node_id) + "nodes/" + link_paths.append(new_link_path) + return link_paths def get_local_query_path(base_paths, test, partition): return base_paths[0] + "queries/" + test + "/" + get_local_query_folder(len(base_paths), partition) + "/" http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/231cdf25/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml ---------------------------------------------------------------------- diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml index daf8f20..bc9f07f 100644 --- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml +++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml @@ -23,16 +23,7 @@ <cluster_ip>127.0.0.1</cluster_ip> </node> <dataset> - <name>tiny</name> - <test>local_speed_up</test> - <save_path>/data</save_path> - <partitions_per_path>1</partitions_per_path> - <partitions_per_path>2</partitions_per_path> - <partitions_per_path>4</partitions_per_path> - <partitions_per_path>8</partitions_per_path> - </dataset> - <dataset> - <name>small</name> + <name>tiny-example</name> <test>local_speed_up</test> <save_path>/data</save_path> <partitions_per_path>1</partitions_per_path> http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/231cdf25/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java index 1f9f0a7..e48d4ab 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionDataSource.java @@ -16,7 +16,6 @@ */ package org.apache.vxquery.metadata; -import java.io.Serializable; import java.util.ArrayList; import java.util.List;
