[
https://issues.apache.org/jira/browse/HAMA-642?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13459189#comment-13459189
]
Hudson commented on HAMA-642:
-----------------------------
Integrated in Hama-Nightly #680 (See
[https://builds.apache.org/job/Hama-Nightly/680/])
[HAMA-642]: Make GraphRunner disk based (Revision 1387533)
Result = FAILURE
tjungblut :
Files :
* /hama/trunk/CHANGES.txt
* /hama/trunk/core/pom.xml
* /hama/trunk/core/src/main/java/org/apache/hama/bsp/BSPMessageBundle.java
* /hama/trunk/core/src/main/java/org/apache/hama/bsp/LocalBSPRunner.java
* /hama/trunk/graph/pom.xml
* /hama/trunk/graph/src/main/java/org/apache/hama/graph/AbstractAggregator.java
* /hama/trunk/graph/src/main/java/org/apache/hama/graph/GraphJob.java
* /hama/trunk/graph/src/main/java/org/apache/hama/graph/GraphJobRunner.java
* /hama/trunk/graph/src/main/java/org/apache/hama/graph/Vertex.java
*
/hama/trunk/graph/src/main/java/org/apache/hama/graph/VertexWritableSerialization.java
* /hama/trunk/graph/src/main/java/org/apache/hama/graph/WritableComparator.java
*
/hama/trunk/graph/src/main/java/org/apache/hama/graph/WritableSerialization.java
* /hama/trunk/jdbm
* /hama/trunk/jdbm/pom.xml
* /hama/trunk/jdbm/src
* /hama/trunk/jdbm/src/main
* /hama/trunk/jdbm/src/main/java
* /hama/trunk/jdbm/src/main/java/org
* /hama/trunk/jdbm/src/main/java/org/apache
* /hama/trunk/jdbm/src/main/java/org/apache/hama
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm
*
/hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/AdvancedObjectInputStream.java
*
/hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/AdvancedObjectOutputStream.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/BTree.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/BTreeLazyRecord.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/BTreeMap.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/BTreeNode.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/BTreeSet.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/DB.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/DBAbstract.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/DBCache.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/DBCacheMRU.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/DBCacheRef.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/DBMaker.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/DBStore.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/DataInputOutput.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/HTree.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/HTreeBucket.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/HTreeDirectory.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/HTreeSet.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/JDBMUtils.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/LinkedList.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/LogicalRowIdManager.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/LongHashMap.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/LongPacker.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/Magic.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/PageFile.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/PageIo.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/PageManager.java
*
/hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/PageTransactionManager.java
*
/hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/PhysicalFreeRowIdManager.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/PhysicalRowIdManager.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/RecordHeader.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/RecordListener.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/SerialClassInfo.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/Serialization.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/SerializationHeader.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/Serializer.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/Storage.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/StorageDisk.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/StorageDiskMapped.java
* /hama/trunk/jdbm/src/main/java/org/apache/hama/jdbm/StorageMemory.java
* /hama/trunk/jdbm/src/main/resources
* /hama/trunk/jdbm/src/test
* /hama/trunk/jdbm/src/test/java
* /hama/trunk/jdbm/src/test/java/org
* /hama/trunk/jdbm/src/test/java/org/apache
* /hama/trunk/jdbm/src/test/java/org/apache/hama
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/BTreeBench.java
*
/hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/BTreeKeyCompressionTest.java
*
/hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/BTreeLeadingValuePackTest.java
*
/hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/BTreeMapNavigable2Test.java
*
/hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/BTreeMapNavigableSubMapExclusiveTest.java
*
/hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/BTreeMapNavigableSubMapInclusiveTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/BTreeMapNavigableTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/BTreeNodeTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/BTreeSetTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/ByteArrayComparator.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/CompactTest.java
*
/hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/ConcurrentBTreeReadTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DBCacheMRUTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DBCacheTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DBMakerTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DBTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DataInputOutputTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/DefragTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/FileHeaderTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/FileLockTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/HTreeBucketTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/HTreeDirectoryTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/HTreeSetTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/HTreeTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/LinkedListTest.java
*
/hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/LogicalRowIdManagerTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/LongHashMapTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/LongTreeMap.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/MapInterfaceTest.java
*
/hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/ObjectOutputStreamTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/PageFileTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/PageIoTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/PageManagerTest.java
*
/hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/PageTransactionManagerTest.java
*
/hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/PhysicalFreeRowIdManagerTest.java
*
/hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/PhysicalRowIdManagerTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/RecordHeaderTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/RollbackTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/SerialClassInfoTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/Serialization2Bean.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/Serialization2Test.java
*
/hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/SerializationHeaderTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/SerializationTest.java
*
/hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/Serialized2DerivedBean.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/StorageDiskMappedTest.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/StreamCorrupted.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/TestCaseWithTestFile.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/TestInsertPerf.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/TestInsertUpdate.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/TestIssues.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/TestLargeData.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/TestLazyRecordsInTree.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/TestRollback.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/TestStress.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/UtilTT.java
* /hama/trunk/jdbm/src/test/java/org/apache/hama/jdbm/UtilsTest.java
* /hama/trunk/jdbm/src/test/resources
* /hama/trunk/pom.xml
* /hama/trunk/src/assemble/bin.xml
> Make GraphRunner disk based
> ---------------------------
>
> Key: HAMA-642
> URL: https://issues.apache.org/jira/browse/HAMA-642
> Project: Hama
> Issue Type: Improvement
> Components: graph
> Affects Versions: 0.5.0
> Reporter: Thomas Jungblut
> Assignee: Thomas Jungblut
> Fix For: 0.6.0
>
> Attachments: HAMA-642_unix_1.patch, HAMA-642_unix_2.patch,
> HAMA-scale_1.patch, HAMA-scale_2.patch, HAMA-scale_3.patch, HAMA-scale_4.patch
>
>
> To improve scalability we can improve the graph runner to be disk based.
> Which basically means:
> - We have just a single Vertex instance that get's refilled.
> - We directly write vertices to disk after partitioning
> - In every superstep we iterate over the vertices on disk, fill the vertex
> instance and call the users compute functions
> Problems:
> - State other than vertex value can't be stored easy
> - How do we deal with random access after messages have arrived?
> So I think we should make the graph runner more hybrid, like using the queues
> we have implemented in the messaging. So the graphrunner can be configured to
> run completely on disk, in cached mode or in in-memory mode.
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
