copied aggregate tests for hdfs and added another option in test options for hdfs
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/02fdcc19 Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/02fdcc19 Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/02fdcc19 Branch: refs/heads/steven/hdfs Commit: 02fdcc195e2893e1f9ab0d0af6b163ac11ab4ee0 Parents: dacf1bc Author: efikalti <[email protected]> Authored: Thu Jun 11 15:21:38 2015 +0300 Committer: efikalti <[email protected]> Committed: Thu Jun 11 15:21:38 2015 +0300 ---------------------------------------------------------------------- .../org/apache/vxquery/hdfs2/HDFSFunctions.java | 69 +++++++++++++++----- .../java/org/apache/vxquery/xtest/XTest.java | 4 ++ .../org/apache/vxquery/xtest/XTestOptions.java | 3 + .../HDFS/Partition-1/avgHDFS.txt | 1 + .../HDFS/Partition-1/countHDFS.txt | 1 + .../HDFS/Partition-1/maxHDFS.txt | 1 + .../HDFS/Partition-1/minHDFS.txt | 1 + .../HDFS/Partition-1/sumHDFS.txt | 1 + .../Queries/XQuery/HDFS/Partition-1/avgHDFS.xq | 25 +++++++ .../XQuery/HDFS/Partition-1/countHDFS.xq | 25 +++++++ .../Queries/XQuery/HDFS/Partition-1/maxHDFS.xq | 25 +++++++ .../Queries/XQuery/HDFS/Partition-1/minHDFS.xq | 25 +++++++ .../Queries/XQuery/HDFS/Partition-1/sumHDFS.xq | 25 +++++++ .../src/test/resources/VXQueryCatalog.xml | 15 +++++ .../cat/HDFSAggregatePartition1Queries.xml | 48 ++++++++++++++ 15 files changed, 251 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/02fdcc19/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFunctions.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFunctions.java b/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFunctions.java index 46ef557..5b1ebd8 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFunctions.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFunctions.java @@ -24,14 +24,12 @@ import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; -import sun.tools.tree.ThisExpression; - public class HDFSFunctions { - private final Configuration conf; + private Configuration conf; private FileSystem fs; private String conf_path; - private final String conf_folder; + private final String conf_folder = "/etc/hadoop/"; /** * Create the configuration and add the paths for core-site and hdfs-site as resources. @@ -40,17 +38,21 @@ public class HDFSFunctions { */ public HDFSFunctions() { - this.conf_folder = "/etc/hadoop/"; - locateConf(); - System.out.println(this.conf_path); - this.conf = new Configuration(); - conf.addResource(new Path(this.conf_path + "core-site.xml")); - conf.addResource(new Path(this.conf_path + "hdfs-site.xml")); - try { - fs = FileSystem.get(conf); - } catch (IOException ex) { - System.err.println(ex); - } + if(locateConf()) + { + this.conf = new Configuration(); + conf.addResource(new Path(this.conf_path + "core-site.xml")); + conf.addResource(new Path(this.conf_path + "hdfs-site.xml")); + try { + fs = FileSystem.get(conf); + } catch (IOException ex) { + System.err.println(ex); + } + } + else + { + //print error cannot locate configuration folder for HDFS + } } /** @@ -106,23 +108,54 @@ public class HDFSFunctions { return null; } - private void locateConf() - { + /** + * Search in the system environment variables for the Hadoop/HDFS home folder to get the path to the configuration. + * Variables it is checking are: HADOOP_HOME,HADOOP_PREFIX,HADOOP_CONF_DIR,HADOOP_HDFS_HOME. + * @return true if is successfully finds the Hadoop/HDFS home directory + */ + private boolean locateConf() + {//HADOOP_HOME String conf = System.getenv("HADOOP_HOME"); if (conf == null) - { + {//HADOOP_PREFIX conf = System.getenv("HADOOP_PREFIX"); if (conf != null) { this.conf_path = conf + this.conf_folder; } + else + {//HADOOP_CONF_DIR + conf = System.getenv("HADOOP_CONF_DIR"); + if (conf != null) + { + this.conf_path = conf + this.conf_folder; + } + else + {//HADOOP_HDFS_HOME + conf = System.getenv("HADOOP_HDFS_HOME"); + if (conf != null) + { + this.conf_path = conf + this.conf_folder; + } + else + { + return false; + } + } + } } else { this.conf_path = conf + this.conf_folder;; } + return true; } + /** + * Get instance of the hdfs file system if it is configured correctly. + * Return null if there is no instance. + * @return + */ public FileSystem getFileSystem() { if (this.conf_path != null) http://git-wip-us.apache.org/repos/asf/vxquery/blob/02fdcc19/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java index 0e2a6e0..861b301 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java @@ -61,6 +61,10 @@ public class XTest { if (opts.diffable != null) { reporters.add(new LineFileReporterImpl(new File(opts.diffable))); } + if (opts.hdfs != null) + { + //run tests for HDFS + } reporters.add(new ResultReporter() { @Override public void close() { http://git-wip-us.apache.org/repos/asf/vxquery/blob/02fdcc19/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java index 2b7b508..b273931 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java @@ -52,4 +52,7 @@ public class XTestOptions { @Option(name = "-htmlreport", required = false, usage = "HTML Report output file") String htmlReport; + + @Option(name = "-hdfs", required = false, usage = "run HDFS tests") + String hdfs; } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/02fdcc19/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/avgHDFS.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/avgHDFS.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/avgHDFS.txt new file mode 100644 index 0000000..7ef6ffe --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/avgHDFS.txt @@ -0,0 +1 @@ +12.5 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/02fdcc19/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/countHDFS.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/countHDFS.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/countHDFS.txt new file mode 100644 index 0000000..d8263ee --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/countHDFS.txt @@ -0,0 +1 @@ +2 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/02fdcc19/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/maxHDFS.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/maxHDFS.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/maxHDFS.txt new file mode 100644 index 0000000..dc7b54a --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/maxHDFS.txt @@ -0,0 +1 @@ +33 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/02fdcc19/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/minHDFS.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/minHDFS.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/minHDFS.txt new file mode 100644 index 0000000..ea1acb6 --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/minHDFS.txt @@ -0,0 +1 @@ +11.25 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/02fdcc19/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/sumHDFS.txt ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/sumHDFS.txt b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/sumHDFS.txt new file mode 100644 index 0000000..2b82dfe --- /dev/null +++ b/vxquery-xtest/src/test/resources/ExpectedTestResults/HDFS/Partition-1/sumHDFS.txt @@ -0,0 +1 @@ +60 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/02fdcc19/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/avgHDFS.xq ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/avgHDFS.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/avgHDFS.xq new file mode 100644 index 0000000..1d1847c --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/avgHDFS.xq @@ -0,0 +1,25 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(: XQuery Aggregate Query :) +(: Find the average minimum temperature. :) +fn:avg( + let $collection := "ghcnd" + for $r in collection($collection)/dataCollection/data + where $r/dataType eq "TMIN" + return $r/value +) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/02fdcc19/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/countHDFS.xq ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/countHDFS.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/countHDFS.xq new file mode 100644 index 0000000..fbc1b2f --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/countHDFS.xq @@ -0,0 +1,25 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(: XQuery Aggregate Query :) +(: Find the number of wind sensor readings. :) +fn:count( + let $collection := "ghcnd" + for $r in collection($collection)/dataCollection/data + where $r/dataType eq "AWND" + return $r/value +) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/02fdcc19/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/maxHDFS.xq ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/maxHDFS.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/maxHDFS.xq new file mode 100644 index 0000000..8de59b4 --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/maxHDFS.xq @@ -0,0 +1,25 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(: XQuery Aggregate Query :) +(: Find the highest max temperature. :) +fn:max( + let $collection := "ghcnd" + for $r in collection($collection)/dataCollection/data + where $r/dataType eq "TMAX" + return $r/value +) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/02fdcc19/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/minHDFS.xq ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/minHDFS.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/minHDFS.xq new file mode 100644 index 0000000..a841d49 --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/minHDFS.xq @@ -0,0 +1,25 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(: XQuery Aggregate Query :) +(: Find the lowest min temperature. :) +fn:min( + let $collection := "ghcnd" + for $r in collection($collection)/dataCollection/data + where $r/dataType eq "TMIN" + return $r/value +) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/02fdcc19/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/sumHDFS.xq ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/sumHDFS.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/sumHDFS.xq new file mode 100644 index 0000000..e769d0a --- /dev/null +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/sumHDFS.xq @@ -0,0 +1,25 @@ +(: Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. :) + +(: XQuery Aggregate Query :) +(: Find the total precipitation. :) +fn:sum( + let $collection := "ghcnd" + for $r in collection($collection)/dataCollection/data + where $r/dataType eq "PRCP" + return $r/value +) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/02fdcc19/vxquery-xtest/src/test/resources/VXQueryCatalog.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/VXQueryCatalog.xml b/vxquery-xtest/src/test/resources/VXQueryCatalog.xml index 773c276..9a79b77 100644 --- a/vxquery-xtest/src/test/resources/VXQueryCatalog.xml +++ b/vxquery-xtest/src/test/resources/VXQueryCatalog.xml @@ -39,6 +39,8 @@ <!ENTITY SingleQuery SYSTEM "cat/SingleQuery.xml"> <!ENTITY SingleAlternateQuery SYSTEM "cat/SingleAlternateQuery.xml"> +<!ENTITY HDFSAggregatePartition1Queries SYSTEM "cat/HDFSAggregatePartition1Queries.xml"> + ]> <test-suite xmlns="http://www.w3.org/2005/02/query-test-XQTSCatalog"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; @@ -176,4 +178,17 @@ &GhcndRecordsPartition4Queries; </test-group> </test-group> + <test-group name="HDFSAggregatePartitionQueries" featureOwner="Efi Kaltirimidou"> + <GroupInfo> + <title>Aggregate Partition Queries in HDFS</title> + <description/> + </GroupInfo> + <test-group name="AggregateParallelExecutionTests" featureOwner="Efi Kaltirimidou"> + <GroupInfo> + <title>Aggregate HDFS Execution Tests</title> + <description/> + </GroupInfo> + &HDFSAggregatePartition1Queries; + </test-group> + </test-group> </test-suite> http://git-wip-us.apache.org/repos/asf/vxquery/blob/02fdcc19/vxquery-xtest/src/test/resources/cat/HDFSAggregatePartition1Queries.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/cat/HDFSAggregatePartition1Queries.xml b/vxquery-xtest/src/test/resources/cat/HDFSAggregatePartition1Queries.xml new file mode 100644 index 0000000..fb420f6 --- /dev/null +++ b/vxquery-xtest/src/test/resources/cat/HDFSAggregatePartition1Queries.xml @@ -0,0 +1,48 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<test-group xmlns="http://www.w3.org/2005/02/query-test-XQTSCatalog"; name="AggregatePartition1Queries" featureOwner="VXQuery"> + <GroupInfo> + <title>HDFS Aggregate Partition 1</title> + <description/> + </GroupInfo> + <test-case name="hdfs-aggregate-partition-1-avg" FilePath="HDFS/Partition-1/" Creator="Efi Kaltirimidou"> + <description>Count records in HDFS returned for q02 from the weather benchmark with 1 partition.</description> + <query name="avgHDFS" date="2015-06-11"/> + <output-file compare="Text">avgHDFS.txt</output-file> + </test-case> + <test-case name="hdfs-aggregate-partition-1-count" FilePath="HDFS/Partition-1/" Creator="Efi Kaltirimidou"> + <description>Count records in HDFS returned for q02 from the weather benchmark with 1 partition.</description> + <query name="countHDFS" date="2015-06-11"/> + <output-file compare="Text">countHDFS.txt</output-file> + </test-case> + <test-case name="hdfs-aggregate-partition-1-min" FilePath="HDFS/Partition-1/" Creator="Efi Kaltirimidou"> + <description>Count records in HDFS returned for q05 from the weather benchmark with 1 partition.</description> + <query name="minHDFS" date="2015-06-11"/> + <output-file compare="Text">minHDFS.txt</output-file> + </test-case> + <test-case name="hdfs-aggregate-partition-1-max" FilePath="HDFS/Partition-1/" Creator="Efi Kaltirimidou"> + <description>Count records in HDFS returned for q07 from the weather benchmark with 1 partition.</description> + <query name="maxHDFS" date="2015-06-11"/> + <output-file compare="Text">maxHDFS.txt</output-file> + </test-case> + <test-case name="hdfs-aggregate-partition-1-sum" FilePath="HDFS/Partition-1/" Creator="Efi Kaltirimidou"> + <description>Count records in HDFS returned for q03 from the weather benchmark with 1 partition.</description> + <query name="sumHDFS" date="2015-06-11"/> + <output-file compare="Text">sumHDFS.txt</output-file> + </test-case> +</test-group> \ No newline at end of file
