rymurr commented on a change in pull request #7347:
URL: https://github.com/apache/arrow/pull/7347#discussion_r446921213
##########
File path:
java/memory/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java
##########
@@ -17,33 +17,107 @@
package org.apache.arrow.memory.rounding;
-import java.lang.reflect.Field;
-
-import org.apache.arrow.memory.NettyAllocationManager;
import org.apache.arrow.memory.util.CommonUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.util.internal.SystemPropertyUtil;
/**
* The default rounding policy. That is, if the requested size is within the
chunk size,
* the rounded size will be the next power of two. Otherwise, the rounded size
* will be identical to the requested size.
*/
public class DefaultRoundingPolicy implements RoundingPolicy {
-
+ private static final Logger logger =
LoggerFactory.getLogger(DefaultRoundingPolicy.class);
public final long chunkSize;
/**
- * The singleton instance.
+ * The variables here and the static block calculates teh DEFAULT_CHUNK_SIZE.
+ *
+ * <p>
+ * It was copied from {@link io.netty.buffer.PooledByteBufAllocator}.
+ * </p>
*/
- public static final DefaultRoundingPolicy INSTANCE = new
DefaultRoundingPolicy();
+ private static final int MIN_PAGE_SIZE = 4096;
+ private static final int MAX_CHUNK_SIZE = (int) (((long) Integer.MAX_VALUE +
1) / 2);
+ private static final long DEFAULT_CHUNK_SIZE;
+ private static final int DEFAULT_PAGE_SIZE;
+ private static final int DEFAULT_MAX_ORDER;
- private DefaultRoundingPolicy() {
+
+ static {
+ int defaultPageSize =
SystemPropertyUtil.getInt("org.apache.memory.allocator.pageSize", 8192);
+ Throwable pageSizeFallbackCause = null;
try {
- Field field =
NettyAllocationManager.class.getDeclaredField("CHUNK_SIZE");
- field.setAccessible(true);
- chunkSize = (Long) field.get(null);
- } catch (Exception e) {
- throw new RuntimeException("Failed to get chunk size from allocation
manager");
+ validateAndCalculatePageShifts(defaultPageSize);
+ } catch (Throwable t) {
+ pageSizeFallbackCause = t;
+ defaultPageSize = 8192;
+ }
+ DEFAULT_PAGE_SIZE = defaultPageSize;
+
+ int defaultMaxOrder =
SystemPropertyUtil.getInt("org.apache.memory.allocator.maxOrder", 11);
+ Throwable maxOrderFallbackCause = null;
+ try {
+ validateAndCalculateChunkSize(DEFAULT_PAGE_SIZE, defaultMaxOrder);
+ } catch (Throwable t) {
+ maxOrderFallbackCause = t;
+ defaultMaxOrder = 11;
+ }
+ DEFAULT_MAX_ORDER = defaultMaxOrder;
+ DEFAULT_CHUNK_SIZE = validateAndCalculateChunkSize(DEFAULT_PAGE_SIZE,
DEFAULT_MAX_ORDER);
+ if (logger.isDebugEnabled()) {
+ if (pageSizeFallbackCause == null) {
+ logger.debug("-Dorg.apache.memory.allocator.pageSize: {}",
DEFAULT_PAGE_SIZE);
+ } else {
+ logger.debug("-Dorg.apache.memory.allocator.pageSize: {}",
DEFAULT_PAGE_SIZE, pageSizeFallbackCause);
+ }
+ if (maxOrderFallbackCause == null) {
+ logger.debug("-Dorg.apache.memory.allocator.maxOrder: {}",
DEFAULT_MAX_ORDER);
+ } else {
+ logger.debug("-Dorg.apache.memory.allocator.maxOrder: {}",
DEFAULT_MAX_ORDER, maxOrderFallbackCause);
+ }
Review comment:
done
##########
File path:
java/memory/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java
##########
@@ -17,33 +17,107 @@
package org.apache.arrow.memory.rounding;
-import java.lang.reflect.Field;
-
-import org.apache.arrow.memory.NettyAllocationManager;
import org.apache.arrow.memory.util.CommonUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.util.internal.SystemPropertyUtil;
/**
* The default rounding policy. That is, if the requested size is within the
chunk size,
* the rounded size will be the next power of two. Otherwise, the rounded size
* will be identical to the requested size.
*/
public class DefaultRoundingPolicy implements RoundingPolicy {
-
+ private static final Logger logger =
LoggerFactory.getLogger(DefaultRoundingPolicy.class);
public final long chunkSize;
/**
- * The singleton instance.
+ * The variables here and the static block calculates teh DEFAULT_CHUNK_SIZE.
+ *
+ * <p>
+ * It was copied from {@link io.netty.buffer.PooledByteBufAllocator}.
+ * </p>
*/
- public static final DefaultRoundingPolicy INSTANCE = new
DefaultRoundingPolicy();
+ private static final int MIN_PAGE_SIZE = 4096;
+ private static final int MAX_CHUNK_SIZE = (int) (((long) Integer.MAX_VALUE +
1) / 2);
+ private static final long DEFAULT_CHUNK_SIZE;
+ private static final int DEFAULT_PAGE_SIZE;
+ private static final int DEFAULT_MAX_ORDER;
- private DefaultRoundingPolicy() {
+
+ static {
+ int defaultPageSize =
SystemPropertyUtil.getInt("org.apache.memory.allocator.pageSize", 8192);
+ Throwable pageSizeFallbackCause = null;
try {
- Field field =
NettyAllocationManager.class.getDeclaredField("CHUNK_SIZE");
- field.setAccessible(true);
- chunkSize = (Long) field.get(null);
- } catch (Exception e) {
- throw new RuntimeException("Failed to get chunk size from allocation
manager");
+ validateAndCalculatePageShifts(defaultPageSize);
+ } catch (Throwable t) {
+ pageSizeFallbackCause = t;
+ defaultPageSize = 8192;
+ }
+ DEFAULT_PAGE_SIZE = defaultPageSize;
+
+ int defaultMaxOrder =
SystemPropertyUtil.getInt("org.apache.memory.allocator.maxOrder", 11);
+ Throwable maxOrderFallbackCause = null;
+ try {
+ validateAndCalculateChunkSize(DEFAULT_PAGE_SIZE, defaultMaxOrder);
+ } catch (Throwable t) {
+ maxOrderFallbackCause = t;
+ defaultMaxOrder = 11;
+ }
+ DEFAULT_MAX_ORDER = defaultMaxOrder;
+ DEFAULT_CHUNK_SIZE = validateAndCalculateChunkSize(DEFAULT_PAGE_SIZE,
DEFAULT_MAX_ORDER);
+ if (logger.isDebugEnabled()) {
+ if (pageSizeFallbackCause == null) {
+ logger.debug("-Dorg.apache.memory.allocator.pageSize: {}",
DEFAULT_PAGE_SIZE);
+ } else {
+ logger.debug("-Dorg.apache.memory.allocator.pageSize: {}",
DEFAULT_PAGE_SIZE, pageSizeFallbackCause);
Review comment:
done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
