I am using ignite hadoop accelerate to improve my map-reduce job. I am always encountered "java heap size" problem, though I have offer more than 80GB to ignite. The map-reduce job will write more than 150GB data to IGFS. When IGFS will sync data to HDFS? When processing a map-reduce job, do I have to make IGFS big enough to hold all the output data from my map-reduce job?
Here is my configuration: <?xml version="1.0" encoding="UTF-8"?> <beans xmlns="http://www.springframework.org/schema/beans"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xmlns:util="http://www.springframework.org/schema/util"; xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd";> <description> Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled. Ignite node will start with this configuration by default. </description> <bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer"> <property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/> <property name="searchSystemEnvironment" value="true"/> </bean> <bean id="igfsCfgBase" class="org.apache.ignite.configuration.FileSystemConfiguration" abstract="true"> <property name="blockSize" value="#{2 * 1024L * 1024L}"/> <property name="perNodeBatchSize" value="4"/> <property name="perNodeParallelBatchCount" value="2"/> <property name="prefetchBlocks" value="4"/> </bean> <bean class="org.apache.ignite.configuration.CacheConfiguration"> <property name="memoryMode" value="OFFHEAP_TIERED"/> <property name="offHeapMaxMemory" value="30106127360"/> <property name="evictionPolicy"> <bean class="org.apache.ignite.cache.eviction.igfs.IgfsPerBlockLruEvictionPolicy"> <property name="maxSize" value="30106127360"/> </bean> </property> <property name="swapEnabled" value="true"/> </bean> <bean id="dataCacheCfgBase" class="org.apache.ignite.configuration.CacheConfiguration" abstract="true"> <property name="cacheMode" value="PARTITIONED"/> <property name="atomicityMode" value="TRANSACTIONAL"/> <property name="writeSynchronizationMode" value="FULL_SYNC"/> <property name="backups" value="0"/> <property name="affinityMapper"> <bean class="org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper"> <constructor-arg value="512"/> </bean> </property> </bean> <bean id="metaCacheCfgBase" class="org.apache.ignite.configuration.CacheConfiguration" abstract="true"> <property name="cacheMode" value="REPLICATED"/> <property name="atomicityMode" value="TRANSACTIONAL"/> <property name="writeSynchronizationMode" value="FULL_SYNC"/> </bean> <bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"> <property name="hadoopConfiguration"> <bean class="org.apache.ignite.configuration.HadoopConfiguration"> <property name="finishedJobInfoTtl" value="300000"/> </bean> </property> <property name="connectorConfiguration"> <bean class="org.apache.ignite.configuration.ConnectorConfiguration"> <property name="host" value="10.8.12.18"/> <property name="port" value="11211"/> <property name="idleTimeout" value="300000"/> </bean> </property> <property name="fileSystemConfiguration"> <list> <bean class="org.apache.ignite.configuration.FileSystemConfiguration" parent="igfsCfgBase"> <property name="name" value="igfs"/> <property name="metaCacheName" value="igfs-meta"/> <property name="dataCacheName" value="igfs-data"/> <property name="ipcEndpointConfiguration"> <bean class="org.apache.ignite.igfs.IgfsIpcEndpointConfiguration"> <property name="type" value="TCP" /> <property name="host" value="0.0.0.0" /> <property name="port" value="10500" /> </bean> </property> <property name="secondaryFileSystem"> <bean class="org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem"> <property name="fileSystemFactory"> <bean class="org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory"> <property name="uri" value="hdfs://10.8.12.16:9000"/> <property name="configPaths"> <list> <value>/home/appadmin/hadoop-2.7.2/etc/hadoop/core-site.xml</value> </list> </property> </bean> </property> </bean> </property> </bean> </list> </property> <property name="cacheConfiguration"> <list> <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="metaCacheCfgBase"> <property name="name" value="igfs-meta"/> </bean> <bean class="org.apache.ignite.configuration.CacheConfiguration" parent="dataCacheCfgBase"> <property name="name" value="igfs-data"/> </bean> </list> </property> <property name="includeEventTypes"> <list> <util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_FAILED"/> <util:constant static-field="org.apache.ignite.events.EventType.EVT_TASK_FINISHED"/> <util:constant static-field="org.apache.ignite.events.EventType.EVT_JOB_MAPPED"/> </list> </property> <property name="communicationSpi"> <bean class="org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi"> <property name="socketWriteTimeout" value="1200000"/> </bean> </property> </bean> </beans> JVM option: if [ -z "$JVM_OPTS" ] ; then if [[ `"$JAVA" -version 2>&1 | egrep "1\.[7]\."` ]]; then JVM_OPTS="-Xms1g -Xmx1g -server -XX:+AggressiveOpts -XX:MaxPermSize=256m" else JVM_OPTS="-server -Xms20g -Xmx80g -server -XX:+AggressiveOpts -XX:MaxMetaspaceSize=10g -XX:NewSize=512m -XX:SurvivorRatio=6 -XX:+AlwaysPreTouch -XX:+UseG1GC -XX:MaxGCPauseMillis=2000 -XX:GCTimeRatio=4 -XX:InitiatingHeapOccupancyPercent=30 -XX:G1HeapRegionSize=8M -XX:ConcGCThreads=16 -XX:G1HeapWastePercent=10 -XX:+UseTLAB -XX:+ScavengeBeforeFullGC -XX:+DisableExplicitGC" fi fi -- View this message in context: http://apache-ignite-users.70518.x6.nabble.com/How-IGFS-keep-sync-with-HDFS-tp9258.html Sent from the Apache Ignite Users mailing list archive at Nabble.com.
