[Android] Added support for wrapping a MemoryFile. Performance as low as using a memorymapped file

Author: csoroiu

main/android/uk/co/real_logic/sbe/codec/java/BitUtil.java

@@ -15,13 +15,18 @@
  */
 package uk.co.real_logic.sbe.codec.java;
 
+import java.lang.reflect.Constructor;
 import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Modifier;
 import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.security.AccessController;
 import java.security.PrivilegedActionException;
 import java.security.PrivilegedExceptionAction;
 
+import android.os.MemoryFile;
+
 /**
  * Miscellaneous useful functions for dealing with low level bits and bytes.
  */
@@ -34,6 +39,7 @@ final class BitUtil
  //at the same time, methods inside libcore.io.Memory where changed to use long addresses instead of ints
  //https://android.googlesource.com/platform/libcore/+/0121106d9dc1ba713b53822886355e4d9339e852%5E%21/
  // luni/src/main/java/java/nio/Buffer.java
+ //at the same time (api 18) ReadWriteDirectByteBuffer was merged into DirectByteBuffer
  //
  //In api 14 org/apache/harmony/luni/platform/OSMemory.java was renamed to libcore/io/Memory.java
  //https://android.googlesource.com/platform/libcore/+/f934c3d2c8dd9e6bc5299cef41adace2a671637d
@@ -47,6 +53,10 @@ final class BitUtil
  private static final MemoryAccess MEMORY_ACCESS;
  private static final boolean USE_LONG_ADDRESS;
 
+ private static final Field MEMORYFILE_ADDRESS_FIELD;
+ private static final boolean USE_LONG_MEMORYFILE_ADDRESS;
+ private static final Constructor<?> DIRECT_BYTE_BUFFER_CONSTRUCTOR;
+
  static
  {
  try
@@ -55,6 +65,32 @@ final class BitUtil
 
  USE_LONG_ADDRESS = EFFECTIVE_DIRECT_ADDRESS_FIELD.getType() == long.class;
  MEMORY_ACCESS = USE_LONG_ADDRESS ? new MemoryAccessLongAddress() : new MemoryAccessIntAddress();
+
+ MEMORYFILE_ADDRESS_FIELD = getField(MemoryFile.class, "mAddress");
+ USE_LONG_MEMORYFILE_ADDRESS = MEMORYFILE_ADDRESS_FIELD.getType() == long.class;
+
+ final PrivilegedExceptionAction<Constructor<?>> action = new PrivilegedExceptionAction<Constructor<?>>()
+ {
+ public Constructor<?> run() throws Exception
+ {
+ Class<?> clazz = Class.forName("java.nio.DirectByteBuffer");
+ Constructor<?> constructor;
+ if (Modifier.isAbstract(clazz.getModifiers()))
+ {
+ //use this starting with api level < 18
+ clazz = Class.forName("java.nio.ReadWriteDirectByteBuffer");
+ constructor = clazz.getDeclaredConstructor(int.class, int.class);
+ }
+ else
+ {
+ //use this starting with api level >= 18
+ constructor = clazz.getDeclaredConstructor(long.class, int.class);
+ }
+ constructor.setAccessible(true);
+ return constructor;
+ }
+ };
+ DIRECT_BYTE_BUFFER_CONSTRUCTOR = AccessController.doPrivileged(action);
  }
  catch (final Exception ex)
  {
@@ -83,6 +119,23 @@ static long getEffectiveDirectAddress(final ByteBuffer buffer)
  }
  }
 
+ static long getMemoryFileAddress(final MemoryFile memoryFile)
+ {
+ try
+ {
+ return USE_LONG_MEMORYFILE_ADDRESS ? MEMORYFILE_ADDRESS_FIELD
+ .getLong(memoryFile) : MEMORYFILE_ADDRESS_FIELD.getInt(memoryFile);
+ }
+ catch (final IllegalArgumentException e)
+ {
+ return 0;
+ }
+ catch (final IllegalAccessException e)
+ {
+ return 0;
+ }
+ }
+
  /**
  * Get the instance of {@link MemoryAccess}.
  *
@@ -115,4 +168,42 @@ public Field run() throws Exception
 
  return AccessController.doPrivileged(action);
  }
+
+ /**
+ * Builds a DirectByteBuffer out of an adddress and a size
+ *
+ * @param address
+ * @param size
+ * @return
+ */
+ static ByteBuffer newDirectByteBuffer(long address, int size)
+ {
+ try
+ {
+ if (USE_LONG_ADDRESS)
+ {
+ return (ByteBuffer) DIRECT_BYTE_BUFFER_CONSTRUCTOR.newInstance(address, size);
+ }
+ else
+ {
+ return (ByteBuffer) DIRECT_BYTE_BUFFER_CONSTRUCTOR.newInstance((int) address, size);
+ }
+ }
+ catch (final IllegalArgumentException e)
+ {
+ return null;
+ }
+ catch (final IllegalAccessException e)
+ {
+ return null;
+ }
+ catch (InstantiationException e)
+ {
+ return null;
+ }
+ catch (InvocationTargetException e)
+ {
+ return null;
+ }
+ }
 }

main/android/uk/co/real_logic/sbe/codec/java/DirectBuffer.java

@@ -19,6 +19,7 @@
 import java.nio.ByteOrder;
 import java.util.Locale;
 
+import android.os.MemoryFile;
 import libcore.io.Memory;
 
 /**
@@ -34,6 +35,9 @@ public final class DirectBuffer
  private boolean hasArray;
 
  private ByteBuffer byteBuffer;
+ //we keep this reference to avoid being cleaned by GC
+ @SuppressWarnings("unused")
+ private MemoryFile memoryFile;
  private long effectiveDirectAddress;
  private int offset;
  private int capacity;
@@ -59,6 +63,29 @@ public DirectBuffer(final ByteBuffer buffer)
  wrap(buffer);
  }
 
+ /**
+ * Attach a view to an off-heap memory region by address.
+ *
+ * <b>This constructor is not available under Android.</b>
+ *
+ * @param address where the memory begins off-heap
+ * @param capacity of the buffer from the given address
+ */
+ public DirectBuffer(final long address, final int capacity)
+ {
+ wrap(address, capacity);
+ }
+
+ /**
+ * Attach a view to a {@link MemoryFile} for providing direct access.
+ *
+ * @param memoryFile to which the view is attached.
+ */
+ public DirectBuffer(final MemoryFile memoryFile)
+ {
+ wrap(memoryFile);
+ }
+
  /**
  * Attach a view to a byte[] for providing direct access.
  *
@@ -72,6 +99,7 @@ public void wrap(final byte[] buffer)
  byteArray = buffer;
  hasArray = true;
  byteBuffer = null;
+ memoryFile = null;
  }
 
  /**
@@ -101,10 +129,41 @@ public void wrap(final ByteBuffer buffer)
  offset = 0;
  effectiveDirectAddress = BitUtil.getEffectiveDirectAddress(buffer);
  }
-
+ memoryFile = null;
  capacity = buffer.capacity();
  }
 
+ /**
+ * Attach a view to an off-heap memory region by address.
+ *
+ * <b>This method is not available under Android.</b>
+ *
+ * @param address where the memory begins off-heap
+ * @param capacity of the buffer from the given address
+ */
+ public void wrap(final long address, final int capacity)
+ {
+ effectiveDirectAddress = address;
+ this.capacity = capacity;
+ byteArray = null;
+ hasArray = false;
+ memoryFile = null;
+ //Memory.memmove needs either a bytebuffer or a bytearray
+ //it could only work with memory addresses, but it doesn't
+ byteBuffer = BitUtil.newDirectByteBuffer(effectiveDirectAddress, this.capacity);
+ }
+
+ /**
+ * Attach a view to a {@link MemoryFile} for providing direct access.
+ *
+ * @param memoryFile to which the view is attached.
+ */
+ public void wrap(final MemoryFile memoryFile)
+ {
+ wrap(BitUtil.getMemoryFileAddress(memoryFile), memoryFile.length());
+ this.memoryFile = memoryFile;
+ }
+
  /**
  * Return the underlying byte[] for this buffer or null if none is attached.
  *

test/android/uk/co/real_logic/sbe/codec/java/DirectBufferTest.java

@@ -23,6 +23,7 @@
 import org.junit.rules.ExpectedException;
 import org.junit.runner.RunWith;
 
+import android.os.MemoryFile;
 import android.test.suitebuilder.annotation.SmallTest;
 import android.util.Log;
 
@@ -79,13 +80,17 @@ public class DirectBufferTest
  @DataPoint
  public static final DirectBuffer MEMORY_MAPPED_BUFFER = new DirectBuffer(createMemoryMappedBuffer());
 
- // private static long memoryBlockAddress = 0;
+ private static ByteBuffer offHeapDummyBuffer = ByteBuffer.allocateDirect(BUFFER_CAPACITY);
+ private static long memoryBlockAddress = BitUtil.getEffectiveDirectAddress(offHeapDummyBuffer);
 
- // @DataPoint //not valid for android
- // public static final DirectBuffer OFF_HEAP_BUFFER =
- // new DirectBuffer(memoryBlockAddress, BUFFER_CAPACITY);
+ @DataPoint
+ public static final DirectBuffer OFF_HEAP_BUFFER = new DirectBuffer(memoryBlockAddress, BUFFER_CAPACITY);
 
- @Theory
+ @DataPoint //not valid for jdk
+ public static final DirectBuffer MEMORY_FILE_BUFFER =
+ new DirectBuffer(createMemoryFile());
+
+ @Theory
  public void shouldThrowExceptionForLimitAboveCapacity(final DirectBuffer buffer)
  {
  exceptionRule.expect(IndexOutOfBoundsException.class);
@@ -521,8 +526,33 @@ public void shouldPutDirectBytesToBufferTruncate(final DirectBuffer buffer)
  assertThat(Integer.valueOf(result), is(Integer.valueOf(15)));
  }
 
+ @Theory
+ public void shouldTestHelloMemoryFile() throws IOException
+ {
+ final byte[] testBytes = "Hello World!".getBytes();
+ MEMORY_FILE_BUFFER.putBytes(10, testBytes, 6, 6);
+ byte[] result = new byte[6];
+ memoryFile.readBytes(result, 10, 0, 6);
+ String s = new String(result);
+ assertThat(s, is("World!"));
+ }
+
  private static RandomAccessFile memoryMappedFile;
  private static MappedByteBuffer buffer;
+ private static MemoryFile memoryFile;
+
+ private static MemoryFile createMemoryFile()
+ {
+ try
+ {
+ memoryFile = new MemoryFile("gigica", BUFFER_CAPACITY);
+ }
+ catch (IOException e)
+ {
+ memoryFile = null;
+ }
+ return memoryFile;
+ }
 
  private static ByteBuffer createMemoryMappedBuffer()
  {
@@ -553,6 +583,7 @@ public static void cleanup()
  cleanMethod.invoke(buffer);
 
  memoryMappedFile.close();
+ memoryFile.close();
  }
  catch (Exception e)
  {