This is an automated email from the ASF dual-hosted git repository. leerho pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/datasketches-memory17.git
commit a2aef41e14658d108bd8acbc44081d5f8773c6ce Author: Lee Rhodes <[email protected]> AuthorDate: Wed May 25 16:04:53 2022 -0700 Updates to main code so far. --- src/main/java/module-info.java | 1 - .../org/apache/datasketches/memory/BaseState.java | 49 +++++++- .../org/apache/datasketches/memory/Memory.java | 4 +- .../memory/internal/BaseStateImpl.java | 35 ++++-- .../memory/internal/BaseWritableMemoryImpl.java | 2 +- .../apache/datasketches/memory/internal/Util.java | 6 +- .../memory/test/AllocateDirectMapMemoryTest.java | 135 +++++++++++++++++++++ .../memory/test/AllocateDirectMemoryTest.java | 118 ++++++++++++++++++ .../datasketches/memory/test/package-info.java} | 10 +- src/test/resources/GettysburgAddress.txt | 7 ++ 10 files changed, 337 insertions(+), 30 deletions(-) diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index d62f61b..f5baf11 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -21,6 +21,5 @@ module org.apache.datasketches.memory { requires java.base; requires java.logging; requires transitive jdk.incubator.foreign; - //requires jdk.incubator.foreign; exports org.apache.datasketches.memory; } \ No newline at end of file diff --git a/src/main/java/org/apache/datasketches/memory/BaseState.java b/src/main/java/org/apache/datasketches/memory/BaseState.java index a78e76e..315672b 100644 --- a/src/main/java/org/apache/datasketches/memory/BaseState.java +++ b/src/main/java/org/apache/datasketches/memory/BaseState.java @@ -23,8 +23,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import jdk.incubator.foreign.MemorySegment; - -//import org.apache.datasketches.memory.internal.BaseStateImpl; +import jdk.incubator.foreign.ResourceScope; /** * Keeps key configuration state for Memory and Buffer plus some common static variables @@ -32,20 +31,62 @@ import jdk.incubator.foreign.MemorySegment; * * @author Lee Rhodes */ -public interface BaseState { +public interface BaseState extends AutoCloseable { /** * The java line separator character as a String. */ static final String LS = System.getProperty("line.separator"); + static final ByteOrder NATIVE_BYTE_ORDER = ByteOrder.nativeOrder(); + static final ByteOrder NON_NATIVE_BYTE_ORDER = + ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN + ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN; + /** * For off-heap segments, this closes the controlling ResourceScope. If the segment is * not off-heap, this does nothing. */ + @Override void close(); - //Byte Order Related + /** + * Forces any changes made to the contents of this mapped segment to be written to the storage device described + * by the mapped segment's file descriptor. Please refer to + * <a href="https://docs.oracle.com/en/java/javase/17/docs/api/jdk.incubator.foreign/jdk/incubator/foreign/MemorySegment.html#force()">force()</a> + */ + void force(); + + /** + * Loads the contents of this mapped segment into physical memory. Please refer to + * <a href="https://docs.oracle.com/en/java/javase/17/docs/api/jdk.incubator.foreign/jdk/incubator/foreign/MemorySegment.html#load()">load()</a> + */ + void load(); + + /** + * Unloads the contents of this mapped segment from physical memory. Please refer to + * <a href="https://docs.oracle.com/en/java/javase/17/docs/api/jdk.incubator.foreign/jdk/incubator/foreign/MemorySegment.html#unload()">unload()</a> + */ + void unload(); + + /** + * Tells whether or not the contents of this mapped segment is resident in physical memory. Please refer to + * <a href="https://docs.oracle.com/en/java/javase/17/docs/api/jdk.incubator.foreign/jdk/incubator/foreign/MemorySegment.html#isLoaded()">isLoaded()</a> + * @return true if it is likely that the contents of this segment is resident in physical memory. + */ + boolean isLoaded(); + + /** + * Returns the resource scope associated with this memory segment. + * @return the resource scope associated with this memory segment. + */ + ResourceScope scope(); + + /** + * Is the underlying resource scope alive? + * @return true, if the underlying resource scope is alive. + */ + boolean isAlive(); /** * Gets the current Type ByteOrder. diff --git a/src/main/java/org/apache/datasketches/memory/Memory.java b/src/main/java/org/apache/datasketches/memory/Memory.java index 6e1cb9c..ad003da 100644 --- a/src/main/java/org/apache/datasketches/memory/Memory.java +++ b/src/main/java/org/apache/datasketches/memory/Memory.java @@ -27,6 +27,7 @@ import java.nio.channels.WritableByteChannel; import java.util.Objects; import org.apache.datasketches.memory.internal.BaseWritableMemoryImpl; +import org.apache.datasketches.memory.internal.Util; import jdk.incubator.foreign.MemorySegment; @@ -92,6 +93,8 @@ public interface Memory extends BaseState { throws Exception { Objects.requireNonNull(file, "file must be non-null."); Objects.requireNonNull(byteOrder, "byteOrder must be non-null."); + Util.negativeCheck(fileOffsetBytes, "fileOffsetBytes"); + Util.negativeCheck(capacityBytes, "capacityBytes"); if (!file.canRead()) { throw new IllegalArgumentException("file must be readable."); } return BaseWritableMemoryImpl.wrapMap(file, fileOffsetBytes, capacityBytes, true, byteOrder); } @@ -262,7 +265,6 @@ public interface Memory extends BaseState { return BaseWritableMemoryImpl.wrapSegment(slice, ByteOrder.nativeOrder(), null); } - //PRIMITIVE getX() and getXArray() /** diff --git a/src/main/java/org/apache/datasketches/memory/internal/BaseStateImpl.java b/src/main/java/org/apache/datasketches/memory/internal/BaseStateImpl.java index 97dd075..701523f 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/BaseStateImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/BaseStateImpl.java @@ -28,6 +28,7 @@ import org.apache.datasketches.memory.BaseState; import org.apache.datasketches.memory.MemoryRequestServer; import jdk.incubator.foreign.MemorySegment; +import jdk.incubator.foreign.ResourceScope; /** * Keeps key configuration state for MemoryImpl and BufferImpl plus some common static variables @@ -49,9 +50,6 @@ public abstract class BaseStateImpl implements BaseState { public static final long FLOAT_SHIFT = 2; public static final long DOUBLE_SHIFT = 3; - public static final ByteOrder NATIVE_BYTE_ORDER = ByteOrder.nativeOrder(); - public static final ByteOrder NON_NATIVE_BYTE_ORDER; - //class type IDs. // 0000 0XXX public static final int READONLY = 1; @@ -83,9 +81,6 @@ public abstract class BaseStateImpl implements BaseState { final int[] p = parseJavaVersion(jdkVer); JDK = p[0] + "." + p[1]; JDK_MAJOR = (p[0] == 1) ? p[1] : p[0]; - - NON_NATIVE_BYTE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN - ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN; } public BaseStateImpl(final MemorySegment seg, final int typeId) { @@ -93,6 +88,24 @@ public abstract class BaseStateImpl implements BaseState { this.typeId = typeId; } + @Override + public void force() { seg.force(); } + + @Override + public void load() { seg.load(); } + + @Override + public void unload() { seg.unload(); } + + @Override + public boolean isLoaded() { return seg.isLoaded(); } + + @Override + public ResourceScope scope() { return seg.scope(); } + + @Override + public boolean isAlive() { return seg.scope().isAlive(); } + /** * Assert the requested offset and length against the allocated size. * The invariants equation is: {@code 0 <= reqOff <= reqLen <= reqOff + reqLen <= allocSize}. @@ -134,8 +147,6 @@ public abstract class BaseStateImpl implements BaseState { } } - //Byte Order Related - @Override public final ByteOrder getTypeByteOrder() { return isNonNativeType() ? NON_NATIVE_BYTE_ORDER : ByteOrder.nativeOrder(); @@ -386,19 +397,19 @@ public abstract class BaseStateImpl implements BaseState { public static int[] parseJavaVersion(final String jdkVer) { final int p0, p1; try { - String[] parts = jdkVer.trim().split("[^0-9\\.]");//grab only number groups and "." + String[] parts = jdkVer.trim().split("\\.");//grab only number groups and "." parts = parts[0].split("\\."); //split out the number groups p0 = Integer.parseInt(parts[0]); //the first number group p1 = (parts.length > 1) ? Integer.parseInt(parts[1]) : 0; //2nd number group, or 0 } catch (final NumberFormatException | ArrayIndexOutOfBoundsException e) { throw new IllegalArgumentException("Improper Java -version string: " + jdkVer + "\n" + e); } - checkJavaVersion(jdkVer, p0, p1); + checkJavaVersion(jdkVer, p0); return new int[] {p0, p1}; } - public static void checkJavaVersion(final String jdkVer, final int p0, final int p1) { - if ( (p0 < 1) || ((p0 == 1) && (p1 < 8)) || (p0 > 13) ) { + public static void checkJavaVersion(final String jdkVer, final int p0) { + if ( p0 != 17 ) { throw new IllegalArgumentException( "Unsupported JDK Major Version, must be 17; " + jdkVer); } diff --git a/src/main/java/org/apache/datasketches/memory/internal/BaseWritableMemoryImpl.java b/src/main/java/org/apache/datasketches/memory/internal/BaseWritableMemoryImpl.java index 7953d7c..69cbaed 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/BaseWritableMemoryImpl.java +++ b/src/main/java/org/apache/datasketches/memory/internal/BaseWritableMemoryImpl.java @@ -114,7 +114,7 @@ public abstract class BaseWritableMemoryImpl extends BaseStateImpl implements Wr * @return this class constructed via the leaf node. * @throws Exception */ - @SuppressWarnings("resource") + //@SuppressWarnings("resource") public static WritableMemory wrapMap(final File file, final long fileOffsetBytes, final long capacityBytes, final boolean localReadOnly, final ByteOrder byteOrder) throws Exception { diff --git a/src/main/java/org/apache/datasketches/memory/internal/Util.java b/src/main/java/org/apache/datasketches/memory/internal/Util.java index 1c1bed2..195ab77 100644 --- a/src/main/java/org/apache/datasketches/memory/internal/Util.java +++ b/src/main/java/org/apache/datasketches/memory/internal/Util.java @@ -278,19 +278,19 @@ public final class Util { public static final void zeroCheck(final long value, final String arg) { if (value <= 0) { - throw new IllegalArgumentException("The argument '" + arg + "' may not be negative or zero."); + throw new IllegalArgumentException("The argument '" + arg + "' must not be negative or zero."); } } public static final void negativeCheck(final long value, final String arg) { if (value < 0) { - throw new IllegalArgumentException("The argument '" + arg + "' may not be negative."); + throw new IllegalArgumentException("The argument '" + arg + "' must not be negative."); } } public static final void nullCheck(final Object obj, final String arg) { if (obj == null) { - throw new IllegalArgumentException("The argument '" + arg + "' may not be null."); + throw new IllegalArgumentException("The argument '" + arg + "' must not be null."); } } diff --git a/src/test/java/org/apache/datasketches/memory/test/AllocateDirectMapMemoryTest.java b/src/test/java/org/apache/datasketches/memory/test/AllocateDirectMapMemoryTest.java new file mode 100644 index 0000000..5af098d --- /dev/null +++ b/src/test/java/org/apache/datasketches/memory/test/AllocateDirectMapMemoryTest.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Note: Lincoln's Gettysburg Address is in the public domain. See LICENSE. + */ + +package org.apache.datasketches.memory.test; + +import static org.apache.datasketches.memory.internal.Util.*; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.io.File; +import java.nio.ByteOrder; + +import org.apache.datasketches.memory.Memory; +import org.testng.annotations.Test; + +import jdk.incubator.foreign.ResourceScope; + +public class AllocateDirectMapMemoryTest { + private static final String LS = System.getProperty("line.separator"); + +// @BeforeClass +// public void setReadOnly() { +// UtilTest.setGettysburgAddressFileToReadOnly(); +// } + + @Test + public void simpleMap() throws Exception { + File file = getResourceFile("GettysburgAddress.txt"); + file.setReadOnly(); + try (Memory mem = Memory.map(file)) { + mem.close(); + } + } + + @Test + public void testIllegalArguments() throws Exception { + File file = getResourceFile("GettysburgAddress.txt"); + try (Memory mem = Memory.map(file, -1, Integer.MAX_VALUE, ByteOrder.nativeOrder())) { + fail("Failed: testIllegalArgumentException: Position was negative."); + } catch (IllegalArgumentException e) { + //ok + } + + try (Memory mem = Memory.map(file, 0, -1, ByteOrder.nativeOrder())) { + fail("Failed: testIllegalArgumentException: Size was negative."); + } catch (IllegalArgumentException e) { + //ok + } + } + + @Test + public void testMapAndMultipleClose() throws Exception { + File file = getResourceFile("GettysburgAddress.txt"); + long memCapacity = file.length(); + try (Memory mem = Memory.map(file, 0, memCapacity, ByteOrder.nativeOrder())) { + assertEquals(memCapacity, mem.getCapacity()); + mem.close(); + mem.close(); //multiple closes are ok + mem.getByte(0); //throws + } catch (IllegalStateException e) { + //ok + } + } + + @Test + public void testLoad() throws Exception { + File file = getResourceFile("GettysburgAddress.txt"); + long memCapacity = file.length(); + try (Memory mem = Memory.map(file, 0, memCapacity, ByteOrder.nativeOrder())) { + mem.load(); + assertTrue(mem.isLoaded()); + mem.close(); + } + } + + @SuppressWarnings("resource") + @Test + public void testHandleHandoff() throws Exception { + File file = getResourceFile("GettysburgAddress.txt"); + long memCapacity = file.length(); + Memory mem = Memory.map(file, 0, memCapacity, ByteOrder.nativeOrder()); + ResourceScope.Handle handle = mem.scope().acquire(); + try { + mem.load(); + assertTrue(mem.isLoaded()); + } finally { + mem.scope().release(handle); + } + assertTrue(mem.isAlive()); + mem.close(); //handle must be released before close + assertFalse(mem.isAlive()); + } + + @Test + public void printlnTest() { + println("PRINTING: "+this.getClass().getName()); + } + + static void println(final Object o) { + if (o == null) { print(LS); } + else { print(o.toString() + LS); } + } + + /** + * @param o value to print + */ + static void print(final Object o) { + if (o != null) { + //System.out.print(o.toString()); //disable here + } + } + +} diff --git a/src/test/java/org/apache/datasketches/memory/test/AllocateDirectMemoryTest.java b/src/test/java/org/apache/datasketches/memory/test/AllocateDirectMemoryTest.java new file mode 100644 index 0000000..2f3a562 --- /dev/null +++ b/src/test/java/org/apache/datasketches/memory/test/AllocateDirectMemoryTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.memory.test; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +import org.apache.datasketches.memory.BaseState; +import org.apache.datasketches.memory.DefaultMemoryRequestServer; +import org.apache.datasketches.memory.MemoryRequestServer; +import org.apache.datasketches.memory.WritableMemory; +import org.testng.annotations.Test; + +import jdk.incubator.foreign.ResourceScope; + +public class AllocateDirectMemoryTest { + private final MemoryRequestServer memReqSvr = new DefaultMemoryRequestServer(); + + @SuppressWarnings("resource") + @Test + public void simpleAllocateDirect() throws Exception { + int longs = 32; + WritableMemory wMem2 = null; + try (WritableMemory wMem = WritableMemory.allocateDirect(longs << 3, memReqSvr)) { + for (int i = 0; i<longs; i++) { + wMem.putLong(i << 3, i); + assertEquals(wMem.getLong(i << 3), i); + } + //inside the TWR block the memory scope should be alive + assertTrue(wMem.isAlive()); + wMem2 = wMem; + } + //The TWR block has exited, so the memory should be invalid + assertFalse(wMem2.isAlive()); + wMem2.close(); + } + + @Test + public void checkMemoryRequestServer() throws Exception { + int longs1 = 32; + int bytes1 = longs1 << 3; + try (WritableMemory origWmem = WritableMemory.allocateDirect(bytes1, memReqSvr)) { + + for (int i = 0; i < longs1; i++) { //puts data in origWmem + origWmem.putLong(i << 3, i); + assertEquals(origWmem.getLong(i << 3), i); + } + println(origWmem.toHexString("Test", 0, 32 * 8)); + + int longs2 = 64; + int bytes2 = longs2 << 3; + MemoryRequestServer memReqSvr = origWmem.getMemoryRequestServer(); + if (memReqSvr == null) { + memReqSvr = new DefaultMemoryRequestServer(); + } + WritableMemory newWmem = memReqSvr.request(origWmem, bytes2); + assertFalse(newWmem.isDirect()); //on heap by default + for (int i = 0; i < longs2; i++) { + newWmem.putLong(i << 3, i); + assertEquals(newWmem.getLong(i << 3), i); + } + memReqSvr.requestClose(origWmem, newWmem); + //The default MRS doesn't actually release because it could be easily misused. + } // So we let the TWR release it here + } + + @Test + public void checkNonNativeDirect() throws Exception { + try (WritableMemory wmem = + WritableMemory.allocateDirect( + 128, + 8, + ResourceScope.newConfinedScope(), + BaseState.NON_NATIVE_BYTE_ORDER, + memReqSvr)) { + wmem.putChar(0, (char) 1); + assertEquals(wmem.getByte(1), (byte) 1); + } + } + + @Test + public void checkExplicitClose() throws Exception { + final long cap = 128; + try (WritableMemory wmem = WritableMemory.allocateDirect(cap, memReqSvr)) { + wmem.close(); //explicit close. Does the work of closing + } //end of scope + } + + @Test + public void printlnTest() { + println("PRINTING: "+this.getClass().getName()); + } + + /** + * @param s value to print + */ + static void println(String s) { + //System.out.println(s); //disable here + } +} diff --git a/src/main/java/module-info.java b/src/test/java/org/apache/datasketches/memory/test/package-info.java similarity index 77% copy from src/main/java/module-info.java copy to src/test/java/org/apache/datasketches/memory/test/package-info.java index d62f61b..be97bb8 100644 --- a/src/main/java/module-info.java +++ b/src/test/java/org/apache/datasketches/memory/test/package-info.java @@ -16,11 +16,5 @@ * specific language governing permissions and limitations * under the License. */ -@SuppressWarnings("javadoc") -module org.apache.datasketches.memory { - requires java.base; - requires java.logging; - requires transitive jdk.incubator.foreign; - //requires jdk.incubator.foreign; - exports org.apache.datasketches.memory; -} \ No newline at end of file + +package org.apache.datasketches.memory.test; diff --git a/src/test/resources/GettysburgAddress.txt b/src/test/resources/GettysburgAddress.txt new file mode 100644 index 0000000..3969d17 --- /dev/null +++ b/src/test/resources/GettysburgAddress.txt @@ -0,0 +1,7 @@ +Abraham Lincoln's Gettysburg Address: + + Four score and seven years ago our fathers brought forth on this continent, a new nation, conceived in Liberty, and dedicated to the proposition that all men are created equal. + + Now we are engaged in a great civil war, testing whether that nation, or any nation so conceived and so dedicated, can long endure. We are met on a great battle-field of that war. We have come to dedicate a portion of that field, as a final resting place for those who here gave their lives that that nation might live. It is altogether fitting and proper that we should do this. + + But, in a larger sense, we can not dedicate —- we can not consecrate —- we can not hallow —- this ground. The brave men, living and dead, who struggled here, have consecrated it, far above our poor power to add or detract. The world will little note, nor long remember what we say here, but it can never forget what they did here. It is for us the living, rather, to be dedicated here to the unfinished work which they who fought here have thus far so nobly advanced. It is rather for us [...] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
