It definitely has nothing to do with HBase - I switched to use MySQL and I am still having the exact same problem - freezing in the exact same spot.
The new thread dump is similar to this one: http://lucene.472066.n3.nabble.com/Nutch-frozen-but-not-exiting-td604954.html He reported a similar freezing issue and unfortunately there was no resolution found (at least not listed in that thread). Maybe it is something to do with a java memory setting so I may start playing around with those next. Here is my full thread dump now: bash-3.2$ jstack -F 29017 >> generator_dump_mysql_F.log Attaching to process ID 29017, please wait... Debugger attached successfully. Server compiler detected. JVM version is 23.1-b03 Deadlock Detection: No deadlocks found. Thread 29110: (state = BLOCKED) Thread 29058: (state = BLOCKED) - sun.misc.Unsafe.park(boolean, long) @bci=0 (Interpreted frame) - java.util.concurrent.locks.LockSupport.park(java.lang.Object) @bci=14, line=186 (Interpreted frame) - java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await() @bci=42, line=2043 (Interpreted frame) - org.apache.hadoop.mapred.MapTask$MapOutputBuffer$SpillThread.run() @bci=55, line=1345 (Interpreted frame) Thread 29047: (state = BLOCKED) - java.lang.Object.wait(long) @bci=0 (Interpreted frame) - org.apache.hadoop.mapred.Task$TaskReporter.run() @bci=45, line=658 (Interpreted frame) - java.lang.Thread.run() @bci=11, line=722 (Interpreted frame) Thread 29046: (state = IN_NATIVE) - java.io.UnixFileSystem.getSpace(java.io.File, int) @bci=0 (Interpreted frame) - java.io.File.getUsableSpace() @bci=34, line=1758 (Interpreted frame) - org.apache.hadoop.fs.DF.getAvailable() @bci=4, line=79 (Interpreted frame) - org.apache.hadoop.fs.LocalDirAllocator$AllocatorPerContext.getLocalPathForWrite(java.lang.String, long, org.apache.hadoop.conf.Configuration, boolean) @bci=239, line=367 (Interpreted frame) - org.apache.hadoop.fs.LocalDirAllocator.getLocalPathForWrite(java.lang.String, long, org.apache.hadoop.conf.Configuration, boolean) @bci=18, line=146 (Interpreted frame) - org.apache.hadoop.fs.LocalDirAllocator.getLocalPathForWrite(java.lang.String, long, org.apache.hadoop.conf.Configuration) @bci=6, line=127 (Interpreted frame) - org.apache.hadoop.mapred.MapOutputFile.getSpillFileForWrite(int, long) @bci=33, line=121 (Interpreted frame) - org.apache.hadoop.mapred.MapTask$MapOutputBuffer.sortAndSpill() @bci=75, line=1397 (Interpreted frame) - org.apache.hadoop.mapred.MapTask$MapOutputBuffer.flush() @bci=102, line=1303 (Interpreted frame) - org.apache.hadoop.mapred.MapTask$NewOutputCollector.close(org.apache.hadoop.mapreduce.TaskAttemptContext) @bci=4, line=698 (Interpreted frame) - org.apache.hadoop.mapred.MapTask.runNewMapper(org.apache.hadoop.mapred.JobConf, org.apache.hadoop.mapreduce.split.JobSplit$TaskSplitIndex, org.apache.hadoop.mapred.TaskUmbilicalProtocol, org.apache.hadoop.mapred.Task$TaskReporter) @bci=324, line=767 (Interpreted frame) - org.apache.hadoop.mapred.MapTask.run(org.apache.hadoop.mapred.JobConf, org.apache.hadoop.mapred.TaskUmbilicalProtocol) @bci=100, line=364 (Interpreted frame) - org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable.run() @bci=221, line=223 (Interpreted frame) - java.util.concurrent.Executors$RunnableAdapter.call() @bci=4, line=471 (Interpreted frame) - java.util.concurrent.FutureTask$Sync.innerRun() @bci=29, line=334 (Interpreted frame) - java.util.concurrent.FutureTask.run() @bci=4, line=166 (Interpreted frame) - java.util.concurrent.ThreadPoolExecutor.runWorker(java.util.concurrent.ThreadPoolExecutor$Worker) @bci=46, line=1110 (Interpreted frame) - java.util.concurrent.ThreadPoolExecutor$Worker.run() @bci=5, line=603 (Interpreted frame) - java.lang.Thread.run() @bci=11, line=722 (Interpreted frame) Thread 29045: (state = BLOCKED) - sun.misc.Unsafe.park(boolean, long) @bci=0 (Interpreted frame) - java.util.concurrent.locks.LockSupport.parkNanos(java.lang.Object, long) @bci=20, line=226 (Interpreted frame) - java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(long) @bci=68, line=2082 (Interpreted frame) - java.util.concurrent.ThreadPoolExecutor.awaitTermination(long, java.util.concurrent.TimeUnit) @bci=68, line=1433 (Interpreted frame) - org.apache.hadoop.mapred.LocalJobRunner$Job.run() @bci=202, line=341 (Interpreted frame) Thread 29040: (state = BLOCKED) Thread 29039: (state = BLOCKED) - java.lang.Object.wait(long) @bci=0 (Interpreted frame) - java.lang.ref.ReferenceQueue.remove(long) @bci=44, line=135 (Interpreted frame) - java.lang.ref.ReferenceQueue.remove() @bci=2, line=151 (Interpreted frame) - java.lang.ref.Finalizer$FinalizerThread.run() @bci=3, line=177 (Interpreted frame) Thread 29038: (state = BLOCKED) - java.lang.Object.wait(long) @bci=0 (Interpreted frame) - java.lang.Object.wait() @bci=2, line=503 (Interpreted frame) - java.lang.ref.Reference$ReferenceHandler.run() @bci=46, line=133 (Interpreted frame) Thread 29036: (state = BLOCKED) - java.lang.Thread.sleep(long) @bci=0 (Interpreted frame) - org.apache.hadoop.mapred.JobClient.monitorAndPrintJob(org.apache.hadoop.mapred.JobConf, org.apache.hadoop.mapred.RunningJob) @bci=80, line=1387 (Interpreted frame) - org.apache.hadoop.mapreduce.Job.waitForCompletion(boolean) @bci=30, line=583 (Interpreted frame) - org.apache.nutch.util.NutchJob.waitForCompletion(boolean) @bci=2, line=50 (Interpreted frame) - org.apache.nutch.crawl.GeneratorJob.run(java.util.Map) @bci=361, line=199 (Interpreted frame) - org.apache.nutch.crawl.GeneratorJob.generate(long, long, boolean, boolean) @bci=224, line=223 (Interpreted frame) - org.apache.nutch.crawl.GeneratorJob.run(java.lang.String[]) @bci=386, line=279 (Interpreted frame) - org.apache.hadoop.util.ToolRunner.run(org.apache.hadoop.conf.Configuration, org.apache.hadoop.util.Tool, java.lang.String[]) @bci=38, line=65 (Interpreted frame) - org.apache.nutch.crawl.GeneratorJob.main(java.lang.String[]) @bci=11, line=287 (Interpreted frame) -- View this message in context: http://lucene.472066.n3.nabble.com/Nutch-2-2-1-Freezing-Deadlocked-During-Generator-Job-tp4078894p4080155.html Sent from the Nutch - User mailing list archive at Nabble.com.
