When you want to upload bigfile by using hbase.

[Tost]

Hi.

Actually, I tried to find out uploading soluction by using hbase.
but, it was impossible because of memory ( hbase is byte stream based
module.)

So, if you want to upload big file, use hadoop directly.
This is sample source.

I'm working hbase-util moduls. hbase is nice project for me. but it's not
easy.
so i'll make it like apache-dbutils.

after I finish, i'll share that modules. Thanks.


-------------------------------------------------------------------------------------
package com.hadoop;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
 * Hadoop File upload test
 * (If you want to upload big file, don't use hbase. just use hadoop
directly.)
 * @author nca...@gmail.com
 *
 */
public class HadoopTest {
private Configuration conf = null;
public HadoopTest(){
conf = new Configuration();
}
 private FileSystem getDFS() throws IOException {
return FileSystem.get(conf);
}
 public void close(OutputStream os){
if(os==null) return;
try { os.close(); } catch (IOException e) {}
}
public void close(InputStream is){
if(is==null) return;
try { is.close(); } catch (IOException e) {}
}
 public boolean uploadFile(Path directory,String filename, InputStream is,
boolean overwrite){
boolean isSuccess=false;
FSDataOutputStream fos=null;
try {
Path p  = new Path(directory, new Path(filename));
FileSystem fs = getDFS();
if(fs.getFileStatus(directory).isDir()==false) {
throw new IOException(directory+" isn't directory.");
}else if(fs.exists(p)){
if(overwrite) {
delete(p,true);
}else{
throw new IOException(p+" already exist.");
}
}
 fos = fs.create(p);
write2Stream(is,fos);
 isSuccess= true;
} catch (IOException e) {
e.printStackTrace();
} finally{
close(fos);
}
return isSuccess;
}
 /**
 * InputStream => OutputStream
 * after finished. you must close InputStream and OutputStream.
 * @param is
 * @param os
 * @throws IOException
 */
public void write2Stream(InputStream is, OutputStream os) throws
IOException{
BufferedInputStream bis = new BufferedInputStream(is);
byte[] buf = new byte[809600]; //800k
 int length=0;
long current = 0;
long total = bis.available();
long progress = 0;
while((length=bis.read(buf))!=(-1)){
os.write(buf,0, length);
current+=length;
long now = current*100/total;
if(progress!=now) {
log("progress=",now,"%");
progress = now;
}
}
}
 public void delete(Path p, boolean recursive) throws IOException{
FileSystem fs = getDFS();
fs.delete(p, recursive);
}
 public static final void log(Object ... os){
StringBuilder sb = new StringBuilder();
for(Object s:os){
sb.append(s);
}
System.out.println(sb.toString());
}
 /**
 * @param args
 * @throws IOException
 */
public static void main(String[] args) throws IOException {
HadoopTest ht = new HadoopTest();
 File file = new File("doc/movie.avi");
FileInputStream fis = new FileInputStream(file);
Path rootPath = new Path("/files/");
String filename = "movie.avi";
 long start = System.currentTimeMillis();
boolean isUploaded = ht.uploadFile(rootPath,filename,fis,true);
long end = System.currentTimeMillis();
log("uploaded=",isUploaded," size=",file.length()," bytes.
time=",(end-start)," ms");
 }

}