changes to folder names and created function to upload folder to hdfs
Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/d9288065 Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/d9288065 Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/d9288065 Branch: refs/heads/steven/hdfs Commit: d9288065a50cea6dd26da347d0109da0b9e4a1e2 Parents: 02fdcc1 Author: efikalti <[email protected]> Authored: Thu Jun 11 16:02:57 2015 +0300 Committer: efikalti <[email protected]> Committed: Thu Jun 11 16:02:57 2015 +0300 ---------------------------------------------------------------------- .../org/apache/vxquery/hdfs2/HDFSFunctions.java | 31 ++++++++++++++++++++ .../VXQueryCollectionOperatorDescriptor.java | 2 ++ .../java/org/apache/vxquery/xtest/XTest.java | 7 ++++- .../org/apache/vxquery/xtest/XTestOptions.java | 2 +- .../Queries/XQuery/HDFS/Partition-1/avgHDFS.xq | 2 +- .../XQuery/HDFS/Partition-1/countHDFS.xq | 2 +- .../Queries/XQuery/HDFS/Partition-1/maxHDFS.xq | 2 +- .../Queries/XQuery/HDFS/Partition-1/minHDFS.xq | 2 +- .../Queries/XQuery/HDFS/Partition-1/sumHDFS.xq | 2 +- .../src/test/resources/VXQueryCatalog.xml | 3 ++ 10 files changed, 48 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/d9288065/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFunctions.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFunctions.java b/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFunctions.java index 5b1ebd8..a475b45 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFunctions.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFunctions.java @@ -152,6 +152,37 @@ public class HDFSFunctions { } /** + * Upload a file/directory to HDFS.Filepath is the path in the local file system.dir is the destination path. + * @param filepath + * @param dir + * @return + */ + public boolean put(String filepath,String dir) + { + if (this.fs != null) + { + Path path = new Path(filepath); + Path dest = new Path(dir); + try { + if (fs.exists(dest)) + { + fs.delete(dest, true); //recursive delete + try { + fs.copyFromLocalFile(path, dest); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + return false; + } + + /** * Get instance of the hdfs file system if it is configured correctly. * Return null if there is no instance. * @return http://git-wip-us.apache.org/repos/asf/vxquery/blob/d9288065/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java index f2f26c8..0d78cf5 100644 --- a/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java +++ b/vxquery-core/src/main/java/org/apache/vxquery/metadata/VXQueryCollectionOperatorDescriptor.java @@ -104,6 +104,7 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO //check if directory is in the local file system if(collectionDirectory.exists()) { + System.out.println("searching in local for file : " + collectionDirectory.getName()); // Go through each tuple. if (collectionDirectory.isDirectory()) { for (int tupleIndex = 0; tupleIndex < fta.getTupleCount(); ++tupleIndex) { @@ -126,6 +127,7 @@ public class VXQueryCollectionOperatorDescriptor extends AbstractSingleActivityO //else check in HDFS file system else { + System.out.println("searching in hdfs for file : " + collectionDirectory.getName()); HDFSFunctions hdfs = new HDFSFunctions(); FileSystem fs = hdfs.getFileSystem(); if (fs != null) http://git-wip-us.apache.org/repos/asf/vxquery/blob/d9288065/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java index 861b301..6d19f06 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTest.java @@ -23,6 +23,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import org.apache.vxquery.hdfs2.HDFSFunctions; import org.mortbay.jetty.Server; public class XTest { @@ -61,9 +62,13 @@ public class XTest { if (opts.diffable != null) { reporters.add(new LineFileReporterImpl(new File(opts.diffable))); } - if (opts.hdfs != null) + if (opts.hdfs) { //run tests for HDFS + //upload sources to hdfs + HDFSFunctions function = new HDFSFunctions(); + System.getProperty("user.dir"); + function.put("src/test/resources/TestSources/ghcnd", "vxquery-hdfs-test"); } reporters.add(new ResultReporter() { @Override http://git-wip-us.apache.org/repos/asf/vxquery/blob/d9288065/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java index b273931..c73697d 100644 --- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java +++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/XTestOptions.java @@ -54,5 +54,5 @@ public class XTestOptions { String htmlReport; @Option(name = "-hdfs", required = false, usage = "run HDFS tests") - String hdfs; + boolean hdfs; } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/vxquery/blob/d9288065/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/avgHDFS.xq ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/avgHDFS.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/avgHDFS.xq index 1d1847c..25b1b9e 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/avgHDFS.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/avgHDFS.xq @@ -18,7 +18,7 @@ (: XQuery Aggregate Query :) (: Find the average minimum temperature. :) fn:avg( - let $collection := "ghcnd" + let $collection := "vxquery-hdfs-test" for $r in collection($collection)/dataCollection/data where $r/dataType eq "TMIN" return $r/value http://git-wip-us.apache.org/repos/asf/vxquery/blob/d9288065/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/countHDFS.xq ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/countHDFS.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/countHDFS.xq index fbc1b2f..f6c59d8 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/countHDFS.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/countHDFS.xq @@ -18,7 +18,7 @@ (: XQuery Aggregate Query :) (: Find the number of wind sensor readings. :) fn:count( - let $collection := "ghcnd" + let $collection := "vxquery-hdfs-test" for $r in collection($collection)/dataCollection/data where $r/dataType eq "AWND" return $r/value http://git-wip-us.apache.org/repos/asf/vxquery/blob/d9288065/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/maxHDFS.xq ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/maxHDFS.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/maxHDFS.xq index 8de59b4..9fce33c 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/maxHDFS.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/maxHDFS.xq @@ -18,7 +18,7 @@ (: XQuery Aggregate Query :) (: Find the highest max temperature. :) fn:max( - let $collection := "ghcnd" + let $collection := "vxquery-hdfs-test" for $r in collection($collection)/dataCollection/data where $r/dataType eq "TMAX" return $r/value http://git-wip-us.apache.org/repos/asf/vxquery/blob/d9288065/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/minHDFS.xq ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/minHDFS.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/minHDFS.xq index a841d49..4b896c9 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/minHDFS.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/minHDFS.xq @@ -18,7 +18,7 @@ (: XQuery Aggregate Query :) (: Find the lowest min temperature. :) fn:min( - let $collection := "ghcnd" + let $collection := "vxquery-hdfs-test" for $r in collection($collection)/dataCollection/data where $r/dataType eq "TMIN" return $r/value http://git-wip-us.apache.org/repos/asf/vxquery/blob/d9288065/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/sumHDFS.xq ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/sumHDFS.xq b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/sumHDFS.xq index e769d0a..f318e25 100644 --- a/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/sumHDFS.xq +++ b/vxquery-xtest/src/test/resources/Queries/XQuery/HDFS/Partition-1/sumHDFS.xq @@ -18,7 +18,7 @@ (: XQuery Aggregate Query :) (: Find the total precipitation. :) fn:sum( - let $collection := "ghcnd" + let $collection := "vxquery-hdfs-test" for $r in collection($collection)/dataCollection/data where $r/dataType eq "PRCP" return $r/value http://git-wip-us.apache.org/repos/asf/vxquery/blob/d9288065/vxquery-xtest/src/test/resources/VXQueryCatalog.xml ---------------------------------------------------------------------- diff --git a/vxquery-xtest/src/test/resources/VXQueryCatalog.xml b/vxquery-xtest/src/test/resources/VXQueryCatalog.xml index 9a79b77..ab7c155 100644 --- a/vxquery-xtest/src/test/resources/VXQueryCatalog.xml +++ b/vxquery-xtest/src/test/resources/VXQueryCatalog.xml @@ -84,6 +84,9 @@ <source ID="ghcnd_quarter_4" FileName="TestSources/ghcnd/half_2/quarter_4" Creator="Preston Carman"> <description last-mod="2014-04-02">Collection of files</description> </source> + <source ID="hdfs" FileName="TestSources/hdfs" Creator="Efi Kaltirimidou"> + <description last-mod="2015-06-11">Collection of files</description> + </source> </sources> <test-group name="SingleQuery" featureOwner="Preston Carman"> <GroupInfo>
