Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1808#discussion_r167120778
--- Diff:
processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
---
@@ -606,122 +609,296 @@ private static void
createTaskListForNode(Map<String, List<List<Distributable>>>
}
/**
- * If any left over data blocks are present then assign those to nodes
in round robin way.
+ * If any left over data blocks are present then assign those to nodes
in round robin way. This
+ * will not obey the data locality.
*
* @param outputMap
- * @param uniqueBlocks
+ * @param leftOverBlocks
*/
- private static void assignLeftOverBlocks(Map<String,
List<Distributable>> outputMap,
- Set<Distributable> uniqueBlocks, int noOfBlocksPerNode, List<String>
activeNodes) {
+ private static void
assignLeftOverBlocks(ArrayList<NodeMultiBlockRelation> outputMap,
+ Set<Distributable> leftOverBlocks, long expectedSizePerNode,
List<String> activeNodes,
+ BlockAssignmentStrategy blockAssignmentStrategy) {
+ Map<String, Integer> node2Idx = new HashMap<>(outputMap.size());
+ for (int idx = 0; idx < outputMap.size(); idx++) {
+ node2Idx.put(outputMap.get(idx).getNode(), idx);
+ }
+ // iterate all the nodes and try to allocate blocks to the nodes
if (activeNodes != null) {
for (String activeNode : activeNodes) {
- List<Distributable> blockLst = outputMap.get(activeNode);
- if (null == blockLst) {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Second assignment iteration: assign for executor:
" + activeNode);
+ }
+
+ Integer idx;
+ List<Distributable> blockLst;
+ if (node2Idx.containsKey(activeNode)) {
+ idx = node2Idx.get(activeNode);
+ blockLst = outputMap.get(idx).getBlocks();
+ } else {
+ idx = node2Idx.size();
blockLst = new
ArrayList<Distributable>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
}
- populateBlocks(uniqueBlocks, noOfBlocksPerNode, blockLst);
- if (blockLst.size() > 0) {
- outputMap.put(activeNode, blockLst);
+ populateBlocks(leftOverBlocks, expectedSizePerNode, blockLst,
blockAssignmentStrategy);
+
+ if (!node2Idx.containsKey(activeNode) && blockLst.size() > 0) {
+ outputMap.add(idx, new NodeMultiBlockRelation(activeNode,
blockLst));
+ node2Idx.put(activeNode, idx);
}
}
} else {
- for (Map.Entry<String, List<Distributable>> entry :
outputMap.entrySet()) {
- List<Distributable> blockLst = entry.getValue();
- populateBlocks(uniqueBlocks, noOfBlocksPerNode, blockLst);
+ for (NodeMultiBlockRelation entry : outputMap) {
+ List<Distributable> blockLst = entry.getBlocks();
+ populateBlocks(leftOverBlocks, expectedSizePerNode, blockLst,
blockAssignmentStrategy);
}
-
}
- for (Map.Entry<String, List<Distributable>> entry :
outputMap.entrySet()) {
- Iterator<Distributable> blocks = uniqueBlocks.iterator();
- if (blocks.hasNext()) {
- Distributable block = blocks.next();
- List<Distributable> blockLst = entry.getValue();
- blockLst.add(block);
- blocks.remove();
- }
- }
+ // if there is still blocks left, allocate them in round robin manner
to each nodes
+ assignBlocksUseRoundRobin(outputMap, leftOverBlocks,
blockAssignmentStrategy);
}
/**
- * The method populate the blockLst to be allocate to a specific node.
- * @param uniqueBlocks
- * @param noOfBlocksPerNode
+ * assign blocks to nodes
+ * @param remainingBlocks
+ * @param expectedSizePerNode
+ * @param blockLst
+ * @param blockAssignmentStrategy
+ */
+ private static void populateBlocks(Set<Distributable> remainingBlocks,
+ long expectedSizePerNode, List<Distributable> blockLst,
+ BlockAssignmentStrategy blockAssignmentStrategy) {
+ switch (blockAssignmentStrategy) {
+ case BLOCK_NUM_FIRST:
+ populateBlocksByNum(remainingBlocks, expectedSizePerNode,
blockLst);
+ break;
+ case BLOCK_SIZE_FIRST:
+ populateBlocksBySize(remainingBlocks, expectedSizePerNode,
blockLst);
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Unsupported block assignment strategy: " +
blockAssignmentStrategy);
+ }
+ }
+ /**
+ * allocate blocks by block num
+ * @param remainingBlocks
+ * @param expectedSizePerNode
* @param blockLst
*/
- private static void populateBlocks(Set<Distributable> uniqueBlocks, int
noOfBlocksPerNode,
- List<Distributable> blockLst) {
- Iterator<Distributable> blocks = uniqueBlocks.iterator();
- //if the node is already having the per block nodes then avoid assign
the extra blocks
- if (blockLst.size() == noOfBlocksPerNode) {
+ private static void populateBlocksByNum(Set<Distributable>
remainingBlocks,
+ long expectedSizePerNode, List<Distributable> blockLst) {
+ Iterator<Distributable> blocks = remainingBlocks.iterator();
+ // if the node is already having the per block nodes then avoid assign
the extra blocks
+ if (blockLst.size() == expectedSizePerNode) {
return;
}
while (blocks.hasNext()) {
Distributable block = blocks.next();
blockLst.add(block);
blocks.remove();
- if (blockLst.size() >= noOfBlocksPerNode) {
+ if (blockLst.size() >= expectedSizePerNode) {
+ break;
+ }
+ }
+ }
+
+ /**
+ * allocate blocks by block size
--- End diff --
Change to:
Take N number of distributable blocks from remainingBlocks and add them to
output parameter blockList. The number of blocks added is determined by
accumulated block size is less than expectedSizePerNode
---