Index: memory_optimization/src/uk/me/parabola/splitter/BinaryMapParser.java =================================================================== --- memory_optimization/src/uk/me/parabola/splitter/BinaryMapParser.java (revision 185) +++ memory_optimization/src/uk/me/parabola/splitter/BinaryMapParser.java (working copy) @@ -11,22 +11,24 @@ private static final int WAY_STATUS_UPDATE_THRESHOLD = 1000000; private static final int RELATION_STATUS_UPDATE_THRESHOLD = 100000; - + private long nodeCount; private long wayCount; private long relationCount; - + BinaryMapParser(MapProcessor processor) { this.processor = processor; } MapProcessor processor; - - public void complete() { - // End of map is sent when all input files are processed. - // So do nothing. - } - + + @Override + public void complete() { + // End of map is sent when all input files are processed. + // So do nothing. + } + // Per-block state for parsing, set when processing the header of a block; + @Override protected void parseDense(Osmformat.DenseNodes nodes) { long last_id = 0, last_lat = 0, last_lon = 0; int j = 0; @@ -34,11 +36,11 @@ long lat = nodes.getLat(i)+last_lat; last_lat = lat; long lon = nodes.getLon(i)+last_lon; last_lon = lon; long id = nodes.getId(i)+last_id; last_id = id; - double latf = parseLat(lat), lonf = parseLon(lon); - + double latf = parseLat(lat), lonf = parseLon(lon); + Node tmp = new Node(); tmp.set(id, latf, lonf); - + if (!processor.isStartNodeOnly()) { if (nodes.getKeysValsCount() > 0) { while (nodes.getKeysVals(j) != 0) { @@ -51,10 +53,11 @@ } } processor.processNode(tmp); - processNodes(tmp); + CountNode(tmp.getId()); } } + @Override protected void parseNodes(List nodes) { for (Osmformat.Node i : nodes) { Node tmp = new Node(); @@ -66,15 +69,33 @@ tmp.set(id, latf, lonf); processor.processNode(tmp); - processNodes(tmp); + CountNode(tmp.getId()); } } - + + @Override protected void parseWays(List ways) { - for (Osmformat.Way i : ways) { - Way tmp = new Way(); - if (!processor.isStartNodeOnly()) { + + long numways = ways.size(); + if (numways == 0) + return; + if (processor.isStartNodeOnly()) { + // we just count the ways, so no need to iterate through the list + if (wayCount > 0) { + long x = (numways + wayCount) % WAY_STATUS_UPDATE_THRESHOLD; + // get and report the id that hits the threshold value + if (x <= numways) { + x = numways - x; + Osmformat.Way w = ways.get((int)(x - 1)); + System.out.println(Utils.format(wayCount+x) + " ways processed... id=" + w.getId()); + } + } + wayCount += numways; + } + else { + for (Osmformat.Way i : ways) { + Way tmp = new Way(); for (int j=0 ; j < i.getKeysCount(); j++) tmp.addTag(getStringById(i.getKeys(j)),getStringById(i.getVals(j))); @@ -83,16 +104,19 @@ tmp.addRef(j+last_id); last_id = j+last_id; } - } - long id = i.getId(); - tmp.set(id); + long id = i.getId(); + tmp.set(id); - processor.processWay(tmp); - processWays(tmp); + processor.processWay(tmp); + countWay(i.getId()); + } } } + + + @Override protected void parseRelations(List rels) { for (Osmformat.Relation i : rels) { Relation tmp = new Relation(); @@ -101,14 +125,14 @@ long id = i.getId(); tmp.set(id); - + long last_mid=0; for (int j =0; j < i.getMemidsCount() ; j++) { long mid = last_mid + i.getMemids(j); last_mid = mid; String role = getStringById(i.getRolesSid(j)); String etype=null; - + if (i.getTypes(j) == Osmformat.Relation.MemberType.NODE) etype = "node"; else if (i.getTypes(j) == Osmformat.Relation.MemberType.WAY) @@ -121,17 +145,18 @@ tmp.addMember(etype,mid,role); } processor.processRelation(tmp); - processRelations(tmp); + countRelation(tmp.getId()); } } + @Override public void parse(Osmformat.HeaderBlock block) { for (String s : block.getRequiredFeaturesList()) { - if (s.equals("OsmSchema-V0.6")) continue; // OK. - if (s.equals("DenseNodes")) continue; // OK. - throw new UnknownFeatureException(s); - } + if (s.equals("OsmSchema-V0.6")) continue; // OK. + if (s.equals("DenseNodes")) continue; // OK. + throw new UnknownFeatureException(s); + } if (block.hasBbox()) { final double multiplier = .000000001; @@ -151,25 +176,25 @@ } } - private void processNodes(Node tmp) { + private void CountNode(long id) { nodeCount++; if (nodeCount % NODE_STATUS_UPDATE_THRESHOLD == 0) { - System.out.println(Utils.format(nodeCount) + " nodes processed... id="+tmp.getId()); + System.out.println(Utils.format(nodeCount) + " nodes processed... id=" + id); } } - private void processWays(Way tmp) { + private void countWay(long id) { wayCount++; if (wayCount % WAY_STATUS_UPDATE_THRESHOLD == 0) { - System.out.println(Utils.format(wayCount) + " ways processed... id="+tmp.getId()); + System.out.println(Utils.format(wayCount) + " ways processed... id=" + id); } } - private void processRelations(Relation tmp) { + private void countRelation(long id) { relationCount++; if (relationCount % RELATION_STATUS_UPDATE_THRESHOLD == 0) { - System.out.println(Utils.format(relationCount) + " relations processed... id="+tmp.getId()); + System.out.println(Utils.format(relationCount) + " relations processed... id=" + id); } } Index: memory_optimization/src/uk/me/parabola/splitter/DensityMapCollector.java =================================================================== --- memory_optimization/src/uk/me/parabola/splitter/DensityMapCollector.java (revision 185) +++ memory_optimization/src/uk/me/parabola/splitter/DensityMapCollector.java (working copy) @@ -50,9 +50,8 @@ @Override public void processNode(Node n) { - if (n.getId()> maxNodeId) { + if (n.getId()> maxNodeId) maxNodeId = n.getId(); - } int glat = n.getMapLat(); int glon = n.getMapLon(); densityMap.addNode(glat, glon); @@ -60,7 +59,10 @@ } @Override - public void processWay(Way w) {} + public void processWay(Way w) { + if (w.getId()> maxNodeId) + maxNodeId = w.getId(); + } @Override public void processRelation(Relation r) {} Index: memory_optimization/src/uk/me/parabola/splitter/Node2AreaMap.java =================================================================== --- memory_optimization/src/uk/me/parabola/splitter/Node2AreaMap.java (revision 185) +++ memory_optimization/src/uk/me/parabola/splitter/Node2AreaMap.java (working copy) @@ -1,107 +1,106 @@ -/* - * Copyright (c) 2011. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - */ - -package uk.me.parabola.splitter; - -import it.unimi.dsi.fastutil.objects.ObjectArrayList; -import java.util.BitSet; -import java.util.HashMap; - -/// a map that stores all areas in which a node occurs. -public class Node2AreaMap { - private int size; - private final short unassigned = -1; - - private SparseLong2ShortMapFunction map; - private final ObjectArrayList dictionary; - private final HashMap index; - - public Node2AreaMap(int numareas, long nodeCount, long maxNodeId, boolean optimizeMem) { - if (optimizeMem) { - System.out.println("Making Node2AreaMap using SparseLong2ShortMapInline for (estimated) " + nodeCount + " Nodes"); - map = new SparseLong2ShortMapInline (nodeCount, maxNodeId); - } - else { - System.out.println("Making Node2AreaMap using SparseLong2ShortMapFix for highest NodeId " + maxNodeId); - map = new SparseLong2ShortMapFix (nodeCount, maxNodeId); - } - - - map.defaultReturnValue(unassigned); - dictionary = new ObjectArrayList(); - index = new HashMap(); - - // init the dictionary - for (short i=0;i <= numareas; i++){ - BitSet b = new BitSet(); - b.set (i); - dictionary.add(b); - index.put( b, i); - } - } - - public void addTo(long key, BitSet out) { - int idx = map.get (key); - if (idx == unassigned) - return; - out.or(dictionary.get(idx)); - //System.out.println(key+":"+out.toString()); -} - - - public void put(long key, short val) { - //System.out.println("p"+key+","+val); - - int idx = map.get (key); - BitSet bnew; - if (idx == unassigned) { - size++; - if (size %1000000 == 0) { - stats(); - } - // node is new -> val is equal index - map.put (key,val); - } - else { - // unlikely: node or way belongs to multiple areas - bnew = (BitSet) dictionary.get(idx).clone(); - bnew.set(val); - - Short combiIndex = index.get(bnew); - if (combiIndex == null){ - // this is a new combination of areas, create new entry - // in the dictionary - combiIndex = (short) dictionary.size(); - dictionary.add(bnew); - index.put(bnew, combiIndex); - //System.out.println("new combination " + combiIndex + " " + bnew.toString()); - } - map.put (key, (short) combiIndex); - } - - } - - public boolean contains (long key) { - return map.containsKey(key); - } - - - public int size() { - return size; - } - - public void stats() { - System.out.println("MAP occupancy: " + Utils.format(size) + ", number of area dictionary entries: " + dictionary.size()); - map.stats(); - } -} +/* + * Copyright (c) 2011. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 3 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +package uk.me.parabola.splitter; + +import it.unimi.dsi.fastutil.objects.ObjectArrayList; +import java.util.BitSet; +import java.util.HashMap; + +/// a map that stores all areas in which a node occurs. +public class Node2AreaMap { + private int size; + private final short unassigned = -1; + + private SparseLong2ShortMapFunction map; + private final ObjectArrayList dictionary; + private final HashMap index; + private BitSet btst; + + public Node2AreaMap(int numareas, long nodeCount, long maxNodeId, boolean optimizeMem) { + System.out.println("Making Node2AreaMap"); + map = new SparseLong2ShortMapInline (nodeCount, maxNodeId, optimizeMem); + map.defaultReturnValue(unassigned); + dictionary = new ObjectArrayList(); + index = new HashMap(); + btst = new BitSet(); + + // init the dictionary + for (short i=0;i <= numareas; i++){ + BitSet b = new BitSet(); + b.set (i); + dictionary.add(b); + index.put( b, i); + } + } + + public void addTo(long key, BitSet out) { + int idx = map.get (key); + if (idx == unassigned) + return; + out.or(dictionary.get(idx)); + //System.out.println(key+":"+out.toString()); + } + + + public void put(long key, short val) { + //System.out.println("p"+key+","+val); + + int idx = map.get (key); + if (idx == unassigned) { + size++; + if (size %1000000 == 0) { + stats(); + } + // node is new -> val is equal index + map.put (key,val); + } + else { + // unlikely: node or way belongs to multiple areas + btst.clear(); + btst.or(dictionary.get(idx)); + btst.set(val); + + Short combiIndex = index.get(btst); + if (combiIndex == null){ + // very unlikely: + // this is a new combination of areas, create new entry + // in the dictionary + BitSet bnew = new BitSet(); + bnew.or(btst); + + combiIndex = (short) dictionary.size(); + + dictionary.add(bnew); + index.put(bnew, combiIndex); + //System.out.println("new combination " + combiIndex + " " + bnew.toString()); + } + map.put (key, (short) combiIndex); + } + + } + + public boolean contains (long key) { + return map.containsKey(key); + } + + + public int size() { + return size; + } + + public void stats() { + System.out.println("MAP occupancy: " + Utils.format(size) + ", number of area dictionary entries: " + dictionary.size()); + map.stats(); + } +} Index: memory_optimization/src/uk/me/parabola/splitter/NodeCollector.java =================================================================== --- memory_optimization/src/uk/me/parabola/splitter/NodeCollector.java (revision 185) +++ memory_optimization/src/uk/me/parabola/splitter/NodeCollector.java (working copy) @@ -42,19 +42,23 @@ // Since we are rounding areas to fit on a low zoom boundary we // can drop the bottom 8 bits of the lat and lon and then fit // the whole lot into a single int. - if (n.getId()> maxNodeId) { - maxNodeId = n.getId(); - } + int glat = n.getMapLat(); int glon = n.getMapLon(); int coord = ((glat << 8) & 0xffff0000) + ((glon >> 8) & 0xffff); + if (n.getId()> maxNodeId) + maxNodeId = n.getId(); coords.add(coord); details.addToBounds(glat, glon); } @Override - public void processWay(Way w) {} + public void processWay(Way w) { + if (w.getId()> maxNodeId) + maxNodeId = w.getId(); + + } @Override public void processRelation(Relation r) {} Index: memory_optimization/src/uk/me/parabola/splitter/OSMParser.java =================================================================== --- memory_optimization/src/uk/me/parabola/splitter/OSMParser.java (revision 185) +++ memory_optimization/src/uk/me/parabola/splitter/OSMParser.java (working copy) @@ -97,8 +97,13 @@ } else if (name.equals("way")) { if (!startNodeOnly) startWay(); - else if (!mixed) - return true; + else { + long wayid = getLongAttr("id"); + if (wayid > maxNodeId) + maxNodeId = wayid; + state = State.Way; + if (!mixed) return true; + } } else if (name.equals("relation")) { if (!startNodeOnly) startRelation(); Index: memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapFix.java =================================================================== --- memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapFix.java (revision 185) +++ memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapFix.java (working copy) @@ -1,264 +0,0 @@ -package uk.me.parabola.splitter; - -import it.unimi.dsi.bits.Fast; - -import java.util.Arrays; - -/** - * Stores long/short pairs, optimized for speed. - * Inspired by SparseInt2ShortMapInline, but uses fixed arrays instead of ArrayList. - * Key values are limited to a maximum of 2^37 - 1. - * - * @author GerdP - * - */ -public class SparseLong2ShortMapFix implements SparseLong2ShortMapFunction { - static final int POOL_SIZE = 8191; - private final long maxNodeId; - private final long nodeCount; - - /** What to return on unassigned indices */ - short unassigned = -1; - private long [] countChunkLen; - final long initsize; - - private int poolIndex = 0; - private int countPools = 0; - private short[][] pool; - - private long[] maskmap; - private short[][] chunkmap; - - long size; - - SparseLong2ShortMapFix(long nodeCount, long maxNodeId) { - this.initsize = maxNodeId/CHUNK_SIZE; - this.maxNodeId = maxNodeId; - this.nodeCount = nodeCount; - clear(); - } - - void arrayPush(short[] array, int index) { - for (int j = array.length - 1; j > index; j--) - array[j] = array[j - 1]; - } - - void arrayCopyFill(short[] from, short[] to) { - int j = 0; - for (; j < from.length; j++) - to[j] = from[j]; - for (; j < to.length; j++) - to[j] = unassigned; - } - - short[] chunkAdd(short[] array, int index, short val) { - if (array[array.length - 1] != unassigned) { - // give it back to my pool ? - if (array.length == 4 & poolIndex > 0) - pool[--poolIndex] = array; - //System.out.println("F:poolIndex:" + poolIndex); - --countChunkLen[array.length]; - short tmp[] = new short[array.length + 4]; - arrayCopyFill(array, tmp); - array = tmp; - ++countChunkLen[array.length]; - } - arrayPush(array, index); - array[index] = val; - return array; - } - - short[] chunkMake() { - short[] out; - if (pool[poolIndex] != null){ - out = pool[poolIndex]; - pool[poolIndex] = null; - if (poolIndex < POOL_SIZE) - ++poolIndex; - else { - //System.out.println("pool is full :-("); - makeNewPool(); - } - } - else - out = new short[4]; - - Arrays.fill(out, unassigned); - ++countChunkLen[out.length]; - return out; - } - - void chunkSet(short[] array, int index, short val) { - array[index] = val; - } - - short chunkGet(short[] array, int index) { - return array[index]; - } - - /** - * Count how many of the lowest X bits in mask are set - * - * @return - */ - int countUnder(long mask, int lowest) { - return Fast.count(mask & ((1L << lowest) - 1)); - } - - @Override - public boolean containsKey(long key) { - if (key >= MAX_KEY) { - throw new IllegalArgumentException( - "Cannot handle such key, is too high. key=" + key); - } - int chunkid = (int) (key / CHUNK_SIZE); - long chunkmask = maskmap[chunkid]; - - if (chunkmask == 0) - return false; - int chunkoffset = (int) (key % CHUNK_SIZE); - - long elementmask = 1L << chunkoffset; - return (chunkmask & elementmask) != 0; - } - - @Override - public short put(long key, short val) { - if (val == unassigned) { - throw new IllegalArgumentException( - "Cannot store the value that is reserved as being unassigned. val=" - + val); - } - if (key < 0) { - throw new IllegalArgumentException("Cannot store the negative key," - + key); - } else if (key >= MAX_KEY) { - throw new IllegalArgumentException( - "Cannot handle such key, is too high. key=" + key); - } - short chunk[]; - int chunkid = (int) (key / CHUNK_SIZE); - int chunkoffset = (int) (key % CHUNK_SIZE); - - long chunkmask = maskmap[chunkid]; - if (chunkmask == 0){ - //chunkmask = 0; - chunk = chunkMake(); - } - else - chunk = chunkmap[chunkid]; - - long elementmask = 1L << chunkoffset; - if ((chunkmask & elementmask) != 0) { - // Already in the array, find the offset and store. - short out = chunkGet(chunk, - countUnder(chunkmask, chunkoffset)); - chunkSet(chunk, countUnder(chunkmask, chunkoffset), val); - // System.out.println("Returning found key "+out+" from put "+ key + - // " " + val); - return out; - } else { - size++; - // Not in the array. Time to insert. - int offset = countUnder(chunkmask, chunkoffset); - chunk = chunkAdd(chunk, offset, val); - chunkmask |= elementmask; - maskmap[chunkid] = chunkmask; - chunkmap[chunkid] = chunk; - return unassigned; - } - } - - @Override - public short get(long key) { - if (key >= MAX_KEY) { - throw new IllegalArgumentException( - "Cannot handle such key, is too high. key=" + key); - } - - int chunkid = (int) (key / CHUNK_SIZE); - long chunkmask = maskmap[chunkid]; - if (chunkmask == 0) - return unassigned; - int chunkoffset = (int) key % CHUNK_SIZE; - long elementmask = 1L << chunkoffset; - if ((chunkmask & elementmask) == 0) { - return unassigned; - } else { - short [] chunk = chunkmap[chunkid]; - return chunkGet(chunk, countUnder(chunkmask, chunkoffset)); - } - } - - @Override - public void clear() { - chunkmap = new short [(int)(this.maxNodeId /CHUNK_SIZE)+1][]; - countChunkLen = new long[CHUNK_SIZE+1]; - maskmap = new long[(int)(this.maxNodeId /CHUNK_SIZE)+1]; - makeNewPool(); - size = 0; - } - - @Override - public short remove(long key) { - throw new UnsupportedOperationException("TODO: Implement"); - } - - @Override - public boolean containsKey(Object arg0) { - throw new UnsupportedOperationException("TODO: Implement"); - } - - @Override - public Short get(Object arg0) { - throw new UnsupportedOperationException("TODO: Implement"); - } - - @Override - public Short put(Long arg0, Short arg1) { - return put(arg0.intValue(), arg1.shortValue()); - } - - @Override - public Short remove(Object arg0) { - throw new UnsupportedOperationException("TODO: Implement"); - } - - @Override - public int size() { - return (int) size; - } - - @Override - public short defaultReturnValue() { - return unassigned; - } - - @Override - public void defaultReturnValue(short arg0) { - unassigned = arg0; - } - - private void makeNewPool(){ - pool = new short[POOL_SIZE+1][4]; - poolIndex = 0; - ++countPools; - } - - @Override - public void stats() { - long usedChunks = 0; - long pctusage = 0L; - for (int i=4; i<=CHUNK_SIZE; i+=4) { - usedChunks += countChunkLen[i]; - if (countChunkLen[i] > 0) { - // System.out.println("type-" + i + " chunks: " + Utils.format(countChunkLen[i])); - } - } - if (maskmap.length > 0) - pctusage = Math.round((double)usedChunks*100/maskmap.length) ; - - System.out.println("chunk map details: Used " + Utils.format(usedChunks) + " of " + Utils.format(maskmap.length) + " allocated map entries (~ " + pctusage + "%)"); - } -} - Index: memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapInline.java =================================================================== --- memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapInline.java (revision 185) +++ memory_optimization/src/uk/me/parabola/splitter/SparseLong2ShortMapInline.java (working copy) @@ -11,26 +11,36 @@ */ public class SparseLong2ShortMapInline implements SparseLong2ShortMapFunction{ static final int MASK_SIZE = 4; // number of chunk elements used to store mask - static final int POOL_SIZE = 8191; - + static final int POOL_SIZE = 8192; + enum Method {vector, hashmap}; + private final Method method; + private final long maxNodeId; + private final long nodeCount; + /** What to return on unassigned indices */ short unassigned = -1; private long [] countChunkLen; - final int initsize; - + int initsize; + // we use a simple pool to be able to reuse allocated small chunks private short[][] pool; private int poolIndex = 0; - + private Int2ObjectOpenHashMap chunkmap; + private short[][] chunkvector; int size; - SparseLong2ShortMapInline(long nodeCount, long MaxNodeId) { + SparseLong2ShortMapInline(long nodeCount, long maxNodeId, boolean optimizeMem) { // estimate the needed elements - this.initsize = (int) (nodeCount/(CHUNK_SIZE/8)); + this.maxNodeId = maxNodeId; + this.nodeCount = nodeCount; + if (optimizeMem) + method = Method.hashmap; + else + method = Method.vector; clear(); - + } void arrayPush(short[] array, int index) { @@ -68,9 +78,8 @@ out = pool[poolIndex]; storeMask(0L, out); pool[poolIndex] = null; - if (poolIndex < POOL_SIZE) - ++poolIndex; - else { + ++poolIndex; + if (poolIndex >= POOL_SIZE){ // chunks from current pool are now all in use by the chunkmap. makeNewPool(); } @@ -110,7 +119,7 @@ "Cannot handle such key, is too high. key=" + key); } int chunkid = (int) (key / CHUNK_SIZE); - short [] chunk = chunkmap.get(chunkid); + short [] chunk = getChunk(chunkid); if (chunk == null) return false; long chunkmask = extractMask(chunk); @@ -125,7 +134,7 @@ if (val == unassigned) { throw new IllegalArgumentException( "Cannot store the value that is reserved as being unassigned. val=" - + val); + + val); } if (key < 0) { throw new IllegalArgumentException("Cannot store the negative key," @@ -137,12 +146,12 @@ int chunkid = (int) (key / CHUNK_SIZE); int chunkoffset = (int) (key % CHUNK_SIZE); - short [] chunk = chunkmap.get(chunkid); + short [] chunk = getChunk(chunkid); if (chunk == null) chunk = chunkMake(); - + long chunkmask = extractMask(chunk); - + long elementmask = 1L << chunkoffset; if ((chunkmask & elementmask) != 0) { // Already in the array, find the offset and store. @@ -159,11 +168,11 @@ chunk = chunkAdd(chunk, offset, val); chunkmask |= elementmask; storeMask(chunkmask, chunk); - chunkmap.put(chunkid, chunk); + putChunk (chunkid, chunk); return unassigned; } } - + @Override public short get(long key) { if (key >= MAX_KEY) { @@ -172,7 +181,7 @@ } int chunkid = (int) (key / CHUNK_SIZE); - short [] chunk = chunkmap.get(chunkid); + short [] chunk = getChunk(chunkid); if (chunk == null) return unassigned; long chunkmask = extractMask(chunk); @@ -187,7 +196,16 @@ @Override public void clear() { - chunkmap = new Int2ObjectOpenHashMap(initsize); + if (method == Method.hashmap) { + initsize = (int) (nodeCount/(CHUNK_SIZE/8)) + 1; + System.out.println("Allocating hashmap for (estimated) " + Utils.format(initsize) + " chunks to store " + nodeCount + " ids."); + chunkmap = new Int2ObjectOpenHashMap(initsize); + } + else { + initsize = (int) (maxNodeId /CHUNK_SIZE) + 1; + System.out.println("Allocating vector for " + Utils.format(initsize) + " chunks to store " + nodeCount + " ids."); + chunkvector = new short [(int)(this.maxNodeId /CHUNK_SIZE)+1][]; + } countChunkLen = new long[CHUNK_SIZE + MASK_SIZE + 1 ]; makeNewPool(); size = 0; @@ -233,17 +251,29 @@ public void defaultReturnValue(short arg0) { unassigned = arg0; } - + + private short[] getChunk (int chunkid){ + if (method == Method.hashmap) + return chunkmap.get(chunkid); + else + return chunkvector[chunkid]; + } + + private void putChunk (int chunkid, short[] chunk) { + if (method == Method.hashmap) + chunkmap.put(chunkid, chunk); + else + chunkvector [chunkid] = chunk; + } + private void storeMask (long mask, short[] chunk) { - // store chunkmask in chunk - long tmp = mask; - if (tmp == 32768) - tmp = 32768; - for (int i = 0; i < MASK_SIZE; i++){ - chunk[i] = (short) (tmp & 0xffffL); - tmp = (tmp >> 16); + // store chunkmask in chunk + long tmp = mask; + for (int i = 0; i < MASK_SIZE; i++){ + chunk[i] = (short) (tmp & 0xffffL); + tmp = (tmp >> 16); + } } - } private long extractMask(short [] chunk){ long mask = 0; for (int i = 0; i < MASK_SIZE; i++) { @@ -255,10 +285,10 @@ } private void makeNewPool(){ - pool = new short[POOL_SIZE+1][8]; + pool = new short[POOL_SIZE][8]; poolIndex = 0; } - + @Override public void stats() { long usedChunks = 0; @@ -269,9 +299,14 @@ // System.out.println("type-" + i + " chunks: " + Utils.format(countChunkLen[i])); } } - if (initsize > 0) - pctusage = Math.round((double)usedChunks*100/initsize) ; - System.out.println("chunk map details: Used " + Utils.format(chunkmap.size()) + " (~ " + pctusage + "%) of the estimated " + initsize + " map entries." ); + pctusage = Math.round((float)usedChunks*100/initsize) ; + if (method == Method.hashmap) { + System.out.println("Chunk map details: used " + Utils.format(chunkmap.size()) + " (~ " + pctusage + "%) of the estimated " + Utils.format(initsize) + " entries." ); + } + else { + System.out.println("Chunk vector details: used " + Utils.format(usedChunks) + " of " + Utils.format(initsize) + " allocated entries (~ " + pctusage + "%)"); + } } -} +} + Index: memory_optimization/src/uk/me/parabola/splitter/SplitProcessor.java =================================================================== --- memory_optimization/src/uk/me/parabola/splitter/SplitProcessor.java (revision 185) +++ memory_optimization/src/uk/me/parabola/splitter/SplitProcessor.java (working copy) @@ -201,9 +201,9 @@ //System.out.println("Send to "+entry.getValue().size()); for (OSMWriter w : writersets[index]) { - int n = writerToID.get(w); boolean found = w.nodeBelongsToThisArea(currentNode); if (found) { + int n = writerToID.get(w); if (maxThreads > 1) { addToWorkingQueue(n, currentNode); } else { @@ -222,15 +222,13 @@ System.out.println("Writing ways " + new Date()); } if (!currentWayAreaSet.isEmpty()) { - // this way falls into 4 or less areas (the normal case). Store these areas in the ways map + // Store these areas in the ways map for (int n = currentWayAreaSet.nextSetBit(0); n >= 0; n = currentWayAreaSet.nextSetBit(n + 1)) { if (maxThreads > 1) { addToWorkingQueue(n, currentWay); } else { writers[n].write(currentWay); } - // add one to the area so we're in the range 1-255. This is because we treat 0 as the - // equivalent of a null ways.put(currentWay.getId(), (short) n); } }