[ https://issues.apache.org/jira/browse/MAPREDUCE-2257?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13025035#comment-13025035 ]
Rosie Li commented on MAPREDUCE-2257: ------------------------------------- [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:59: warning: [deprecation] org.apache.hadoop.mapred.FileSplit in org.apache.hadoop.mapred has been deprecated [javac] import org.apache.hadoop.mapred.FileSplit; [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:60: warning: [deprecation] org.apache.hadoop.mapred.InputFormat in org.apache.hadoop.mapred has been deprecated [javac] import org.apache.hadoop.mapred.InputFormat; [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:61: warning: [deprecation] org.apache.hadoop.mapred.InputSplit in org.apache.hadoop.mapred has been deprecated [javac] import org.apache.hadoop.mapred.InputSplit; [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:63: warning: [deprecation] org.apache.hadoop.mapred.JobClient in org.apache.hadoop.mapred has been deprecated [javac] import org.apache.hadoop.mapred.JobClient; [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:64: warning: [deprecation] org.apache.hadoop.mapred.JobConf in org.apache.hadoop.mapred has been deprecated [javac] import org.apache.hadoop.mapred.JobConf; [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:66: warning: [deprecation] org.apache.hadoop.mapred.Mapper in org.apache.hadoop.mapred has been deprecated [javac] import org.apache.hadoop.mapred.Mapper; [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:211: warning: [deprecation] org.apache.hadoop.mapred.JobConf in org.apache.hadoop.mapred has been deprecated [javac] private JobConf conf; [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:738: warning: [deprecation] org.apache.hadoop.mapred.JobConf in org.apache.hadoop.mapred has been deprecated [javac] private static void checkSrcPath(JobConf jobConf, List<Path> srcPaths) [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:831: warning: [deprecation] org.apache.hadoop.mapred.JobConf in org.apache.hadoop.mapred has been deprecated [javac] static private void finalize(Configuration conf, JobConf jobconf, [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:1096: warning: [deprecation] org.apache.hadoop.mapred.JobConf in org.apache.hadoop.mapred has been deprecated [javac] private static int setMapCount(long totalBytes, JobConf job) [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:1120: warning: [deprecation] org.apache.hadoop.mapred.JobConf in org.apache.hadoop.mapred has been deprecated [javac] private static JobConf createJobConf(Configuration conf) { [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:1148: warning: [deprecation] org.apache.hadoop.mapred.JobConf in org.apache.hadoop.mapred has been deprecated [javac] private static void setReplication(Configuration conf, JobConf jobConf, [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:1190: warning: [deprecation] org.apache.hadoop.mapred.JobConf in org.apache.hadoop.mapred has been deprecated [javac] static boolean setup(Configuration conf, JobConf jobConf, [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:1562: warning: [deprecation] org.apache.hadoop.mapred.JobConf in org.apache.hadoop.mapred has been deprecated [javac] FileSystem jobfs, Path jobdir, JobConf jobconf, Configuration conf [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:257: warning: [deprecation] org.apache.hadoop.mapred.InputFormat in org.apache.hadoop.mapred has been deprecated [javac] static class CopyInputFormat implements InputFormat<Text, Text> { [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:265: warning: [deprecation] org.apache.hadoop.mapred.JobConf in org.apache.hadoop.mapred has been deprecated [javac] public InputSplit[] getSplits(JobConf job, int numSplits) [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:265: warning: [deprecation] org.apache.hadoop.mapred.InputSplit in org.apache.hadoop.mapred has been deprecated [javac] public InputSplit[] getSplits(JobConf job, int numSplits) [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:316: warning: [deprecation] org.apache.hadoop.mapred.InputSplit in org.apache.hadoop.mapred has been deprecated [javac] public RecordReader<Text, Text> getRecordReader(InputSplit split, [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:317: warning: [deprecation] org.apache.hadoop.mapred.JobConf in org.apache.hadoop.mapred has been deprecated [javac] JobConf job, Reporter reporter) throws IOException { [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:326: warning: [deprecation] org.apache.hadoop.mapred.Mapper in org.apache.hadoop.mapred has been deprecated [javac] implements Mapper<LongWritable, FilePair, WritableComparable<?>, Text> { [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:337: warning: [deprecation] org.apache.hadoop.mapred.JobConf in org.apache.hadoop.mapred has been deprecated [javac] private JobConf job; [javac] ^ [javac] /data/users/rosieli/hadoop_jira/hadoop-mapred-trunk/src/tools/org/apache/hadoop/tools/DistCp.java:617: warning: [deprecation] org.apache.hadoop.mapred.JobConf in org.apache.hadoop.mapred has been deprecated [javac] public void configure(JobConf job) For the warning added, they are all from using deprecated class. > distcp can copy blocks in parallel > ---------------------------------- > > Key: MAPREDUCE-2257 > URL: https://issues.apache.org/jira/browse/MAPREDUCE-2257 > Project: Hadoop Map/Reduce > Issue Type: Improvement > Components: distcp > Affects Versions: 0.21.0 > Reporter: dhruba borthakur > Assignee: dhruba borthakur > Attachments: MAPREDUCE-2257.patch > > > The minimum unit of work for a distcp task is a file. We have files that are > greater than 1 TB with a block size of 1 GB. If we use distcp to copy these > files, the tasks either take a long long long time or finally fails. A better > way for distcp would be to copy all the source blocks in parallel, and then > stich the blocks back to files at the destination via the HDFS Concat API > (HDFS-222) -- This message is automatically generated by JIRA. For more information on JIRA, see: http://www.atlassian.com/software/jira