Repository: vxquery Updated Branches: refs/heads/steven/hdfs [created] 66ff50ae9
dependencies for hdfs and hdfs2 package with helper functions Project: http://git-wip-us.apache.org/repos/asf/vxquery/repo Commit: http://git-wip-us.apache.org/repos/asf/vxquery/commit/5e86d37d Tree: http://git-wip-us.apache.org/repos/asf/vxquery/tree/5e86d37d Diff: http://git-wip-us.apache.org/repos/asf/vxquery/diff/5e86d37d Branch: refs/heads/steven/hdfs Commit: 5e86d37d9ea823d1d5ae777dab89910055b5870e Parents: 32f9fcd Author: efikalti <[email protected]> Authored: Thu Jun 4 18:07:51 2015 +0300 Committer: efikalti <[email protected]> Committed: Thu Jun 4 18:07:51 2015 +0300 ---------------------------------------------------------------------- vxquery-core/pom.xml | 16 +++ .../apache/vxquery/hdfs2/HDFSFileFunctions.java | 100 +++++++++++++++++++ 2 files changed, 116 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/vxquery/blob/5e86d37d/vxquery-core/pom.xml ---------------------------------------------------------------------- diff --git a/vxquery-core/pom.xml b/vxquery-core/pom.xml index 968e5c7..1bed170 100644 --- a/vxquery-core/pom.xml +++ b/vxquery-core/pom.xml @@ -259,6 +259,22 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>2.7.0</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>2.7.0</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <version>2.7.0</version> + </dependency> </dependencies> <reporting> http://git-wip-us.apache.org/repos/asf/vxquery/blob/5e86d37d/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFileFunctions.java ---------------------------------------------------------------------- diff --git a/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFileFunctions.java b/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFileFunctions.java new file mode 100644 index 0000000..c0b43ca --- /dev/null +++ b/vxquery-core/src/main/java/org/apache/vxquery/hdfs2/HDFSFileFunctions.java @@ -0,0 +1,100 @@ +package org.apache.vxquery.hdfs2; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; + +public class HDFSFileFunctions { + + private final Configuration conf; + private FileSystem fs; + + /** + * Create the configuration and add the paths for core-site and hdfs-site as resources. + * Initialize an instance a hdfs FileSystem for this configuration. + * @param hadoop_conf_filepath + */ + public HDFSFileFunctions(String hadoop_conf_filepath) + { + this.conf = new Configuration(); + conf.addResource(new Path(hadoop_conf_filepath + "core-site.xml")); + conf.addResource(new Path(hadoop_conf_filepath + "hdfs-site.xml")); + + try { + fs = FileSystem.get(conf); + } catch (IOException ex) { + System.err.println(ex); + } + } + + /** + * Returns true if the file path exists or it is located somewhere in the home directory of the user that called the function. + * Searches in subdirectories of the home directory too. + * @param filename + * @return + */ + public boolean isLocatedInHDFS(String filename) throws IOException + { + //search file path + if (fs.exists(new Path(filename))) + { + return true; + } + //Search every file and folder in the home directory + if (searchInDirectory(fs.getHomeDirectory(), filename) != null) + { + return true; + } + return false; + } + + /** + * Searches the given directory and subdirectories for the file. + * @param directory to search + * @param filename of file we want + * @return path if file exists in this directory.else return null. + */ + public Path searchInDirectory(Path directory, String filename) + { + //Search every folder in the directory + try { + RemoteIterator<LocatedFileStatus> it = fs.listFiles(directory, true); + String[] parts; + Path path; + while(it.hasNext()) + { + path = it.next().getPath(); + parts = path.toString().split("/"); + if(parts[parts.length-1].equals(filename)) + { + return path; + } + } + } catch (IOException ex) { + System.err.println(ex); + } + return null; + } + + public void readFile(String filename) throws IOException + { + Path path = this.searchInDirectory(fs.getHomeDirectory(), filename); + if (path != null) + { + BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(path))); + String line; + line = br.readLine(); + while (line != null) { + System.out.println(line); + line = br.readLine(); + } + fs.close(); + } + } +}
