implement #147: memory mapped file backed allocator

compiler/src/test/java/org/capnproto/test/EncodingTest.java

@@ -28,7 +28,7 @@ public void testAllTypes() {
 
     @Test
     public void testAllTypesMultiSegment() {
-        MessageBuilder message = new MessageBuilder(5, BuilderArena.AllocationStrategy.FIXED_SIZE);
+        MessageBuilder message = new MessageBuilder(5, Allocator.AllocationStrategy.FIXED_SIZE);
         org.capnproto.test.Test.TestAllTypes.Builder allTypes = message.initRoot(org.capnproto.test.Test.TestAllTypes.factory);
         TestUtil.initTestMessage(allTypes);
 

runtime/pom.xml

@@ -31,8 +31,8 @@
     </developers>
     <properties>
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-      <maven.compiler.source>1.8</maven.compiler.source>
-      <maven.compiler.target>1.8</maven.compiler.target>
+      <maven.compiler.source>9</maven.compiler.source>
+      <maven.compiler.target>9</maven.compiler.target>
     </properties>
     <dependencies>
         <dependency>
@@ -59,9 +59,11 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-compiler-plugin</artifactId>
-          <version>3.6.2</version>
+          <version>3.11.0</version>
           <configuration>
             <compilerArgument>-Xlint:unchecked</compilerArgument>
+            <source>9</source>
+            <target>9</target>
           </configuration>
         </plugin>
       </plugins>
@@ -71,10 +73,10 @@
         <profile>
             <id>jdk9FF</id>
             <activation>
-                <jdk>(1.8,)</jdk>
+                <jdk>(9,)</jdk>
             </activation>
             <properties>
-                <maven.compiler.release>8</maven.compiler.release>
+                <maven.compiler.release>9</maven.compiler.release>
             </properties>
         </profile>
       <profile>

runtime/src/main/java/org/capnproto/Allocator.java

@@ -4,10 +4,19 @@
  * An object that allocates memory for a Cap'n Proto message as it is being built.
  */
 public interface Allocator {
+    public enum AllocationStrategy {
+        FIXED_SIZE,
+        GROW_HEURISTICALLY
+    }
     /**
      * Allocates a ByteBuffer to be used as a segment in a message. The returned
      * buffer must contain at least `minimumSize` bytes, all of which MUST be
      * set to zero.
      */
    public java.nio.ByteBuffer allocateSegment(int minimumSize);
+
+   /**
+    * set the size for the next buffer allocation in bytes
+    */
+   public void setNextAllocationSizeBytes(int nextSize);
 }

runtime/src/main/java/org/capnproto/BuilderArena.java

@@ -24,27 +24,38 @@
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.util.ArrayList;
+import org.capnproto.Allocator.AllocationStrategy;
 
 public final class BuilderArena implements Arena {
-    public enum AllocationStrategy {
-        FIXED_SIZE,
-        GROW_HEURISTICALLY
+    // allocator to use, default BYTE_BUFFER is faster but is limited by
+    // available process memory limit
+    public enum AllocatorType {
+        BYTE_BUFFER,
+        MEMORY_MAPPED
     }
 
     public static final int SUGGESTED_FIRST_SEGMENT_WORDS = 1024;
     public static final AllocationStrategy SUGGESTED_ALLOCATION_STRATEGY =
         AllocationStrategy.GROW_HEURISTICALLY;
 
+    public static final AllocatorType SUGGESTED_ALLOCATOR_TYPE =
+        AllocatorType.BYTE_BUFFER;
+
     public final ArrayList<SegmentBuilder> segments;
     private final Allocator allocator;
 
-    public BuilderArena(int firstSegmentSizeWords, AllocationStrategy allocationStrategy) {
+    // public BuilderArena(int firstSegmentSizeWords, AllocationStrategy allocationStrategy) {
+    //     this.segments = new ArrayList<SegmentBuilder>();
+    //     this.allocator = new DefaultAllocator(allocationStrategy);
+    //     this.allocator.setNextAllocationSizeBytes(
+    //         firstSegmentSizeWords * Constants.BYTES_PER_WORD);
+    // }
+
+    public BuilderArena(int firstSegmentSizeWords, AllocationStrategy allocationStrategy, AllocatorType allocatorType) {
         this.segments = new ArrayList<SegmentBuilder>();
-        {
-            DefaultAllocator allocator = new DefaultAllocator(allocationStrategy);
-            allocator.setNextAllocationSizeBytes(firstSegmentSizeWords * Constants.BYTES_PER_WORD);
-            this.allocator = allocator;
-        }
+        this.allocator = createAllocator(allocatorType, allocationStrategy);
+        this.allocator.setNextAllocationSizeBytes(
+            firstSegmentSizeWords * Constants.BYTES_PER_WORD);
     }
 
     public BuilderArena(Allocator allocator) {
@@ -77,15 +88,12 @@ public BuilderArena(Allocator allocator, ByteBuffer firstSegment) {
             segmentBuilder.id = ii;
             segmentBuilder.pos = segmentBuilder.capacity(); // buffer is pre-filled
             segments.add(segmentBuilder);
-
             // Find the largest segment for the allocation strategy.
             largestSegment = Math.max(largestSegment, segment.buffer.capacity());
         }
-        DefaultAllocator defaultAllocator = new DefaultAllocator(SUGGESTED_ALLOCATION_STRATEGY);
-
-        // Use largest segment as next size.
-        defaultAllocator.setNextAllocationSizeBytes(largestSegment);
-        this.allocator = defaultAllocator;
+        AllocatorType allocatorType = suggestAllocator(largestSegment);
+        this.allocator = createAllocator(allocatorType, SUGGESTED_ALLOCATION_STRATEGY);
+        this.allocator.setNextAllocationSizeBytes(largestSegment);
     }
 
     @Override
@@ -152,4 +160,31 @@ public final ByteBuffer[] getSegmentsForOutput() {
         }
         return result;
     }
+
+    private static Allocator createAllocator(AllocatorType allocatorType, AllocationStrategy allocationStrategy) {
+        switch (allocatorType) {
+            case BYTE_BUFFER:
+                DefaultAllocator allocator1 = new DefaultAllocator(allocationStrategy);
+                return allocator1;
+
+            case MEMORY_MAPPED:
+                MemoryMappedAllocator allocator2 = new MemoryMappedAllocator(
+                    "capnp_mmf", allocationStrategy);
+                return allocator2;
+            default:
+                throw new AssertionError(
+                    "Allocator must be BYTE_BUFFER or MEMORY_MAPPED");
+        }
+    }
+
+    public static AllocatorType suggestAllocator(int sizeToAllocate) {
+        Runtime runtime = Runtime.getRuntime();
+        long freeMemory = runtime.freeMemory();         // Free memory in bytes
+        long totalMemory = runtime.totalMemory();       // Total memory in JVM
+        long maxMemory = runtime.maxMemory();           // Max memory the JVM will attempt to use
+        if (freeMemory - sizeToAllocate < 0)
+            return AllocatorType.MEMORY_MAPPED;
+
+        return AllocatorType.BYTE_BUFFER;
+   };
 }

runtime/src/main/java/org/capnproto/DefaultAllocator.java

@@ -2,7 +2,6 @@
 
 import java.nio.ByteBuffer;
 
-import org.capnproto.BuilderArena.AllocationStrategy;
 
 public class DefaultAllocator implements Allocator {
 
@@ -45,6 +44,7 @@ public DefaultAllocator(AllocationStrategy allocationStrategy,
         this.allocationStyle = style;
     }
 
+    @Override
     public void setNextAllocationSizeBytes(int nextSize) {
         this.nextSize = nextSize;
     }

runtime/src/main/java/org/capnproto/MemoryMappedAllocator.java

@@ -0,0 +1,207 @@
+// Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
+// Licensed under the MIT License:
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package org.capnproto;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.lang.ref.Cleaner;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+
+public class MemoryMappedAllocator implements Allocator {
+    // cleaner for file cleanup when this is GCed
+    private static final Cleaner cleaner = Cleaner.create();
+    private final Cleaner.Cleanable cleanable;
+
+    // the length of the random prefix part of the random filename string
+    private final int PREFIX_LENGTH = 5;
+    // the used charset for creating random filenames
+    private static final String CHARSET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+
+    // (minimum) number of bytes in the next allocation
+    private int nextSize = BuilderArena.SUGGESTED_FIRST_SEGMENT_WORDS;
+    // the maximum allocateable size
+    public int maxSegmentBytes = Integer.MAX_VALUE - 2;
+
+    // the memory mapped file buffer name prefix
+    private final String rPrefix;
+
+    // the allocation strategy with which the allocation size grows
+    public AllocationStrategy allocationStrategy =
+        AllocationStrategy.GROW_HEURISTICALLY;
+
+    // hashmaps used for keeping track of files
+    private final Map<Integer, RandomAccessFile> randomAccFiles = Collections.synchronizedMap(new HashMap<>());
+    private final Map<Integer, FileChannel> channelMap = Collections.synchronizedMap(new HashMap<>());
+
+    public MemoryMappedAllocator(String baseFileName) {
+        // create random file name with baseFileNamePrefix:
+        // ${baseFileName}_XXXXX_000001
+        Random random = new Random();
+        StringBuilder sb = new StringBuilder(PREFIX_LENGTH);
+        for (int i = 0; i < PREFIX_LENGTH; i++) {
+            int index = random.nextInt(CHARSET.length());
+            sb.append(CHARSET.charAt(index));
+        }
+        rPrefix = baseFileName + "_" + sb.toString();
+        this.cleanable = cleaner.register(this, new State(this.randomAccFiles, rPrefix));
+    }
+
+
+    public MemoryMappedAllocator(String baseFileName, AllocationStrategy allocationStrategy) {
+        // create random file name with baseFileNamePrefix:
+        // ${baseFileName}_XXXXX_000001
+        Random random = new Random();
+        StringBuilder sb = new StringBuilder(PREFIX_LENGTH);
+        for (int i = 0; i < PREFIX_LENGTH; i++) {
+            int index = random.nextInt(CHARSET.length());
+            sb.append(CHARSET.charAt(index));
+        }
+        rPrefix = baseFileName + "_" + sb.toString();
+        this.cleanable = cleaner.register(this, new State(this.randomAccFiles, rPrefix));
+        this.allocationStrategy = allocationStrategy;
+    }
+
+
+    private static String nameForInt(String prefix, int key) {
+        String ret = prefix + "_" + String.format("%05d", key);
+        return ret;
+    }
+
+
+    private Integer generateFile() throws IOException {
+        int fCount;
+        synchronized (randomAccFiles) {
+            fCount = randomAccFiles.size();
+            String newFileName = nameForInt(rPrefix, fCount);
+            RandomAccessFile newFile = new RandomAccessFile(newFileName, "rw");
+            randomAccFiles.put(fCount, newFile);
+            File test = new File(newFileName);
+            test.deleteOnExit();
+        }
+        return fCount;
+    }
+
+
+    /**
+     * set the grow size of the memory mapped file
+     */
+    @Override
+    public void setNextAllocationSizeBytes(int nextSize) {
+        this.nextSize = nextSize;
+    }
+
+
+    private FileChannel createSegment(int segmentSize) throws IOException {
+        int fileKey = generateFile();
+        FileChannel channel = null;
+        synchronized (channelMap) {
+            if (!channelMap.containsKey(fileKey)) {
+                synchronized (randomAccFiles) {
+                    RandomAccessFile file = randomAccFiles.get(fileKey);
+                    file.setLength(segmentSize);
+                    channel = file.getChannel();
+                    channelMap.put(fileKey, channel);
+                }
+            }
+        }
+        return channel;
+    }
+
+
+    @Override
+    public java.nio.ByteBuffer allocateSegment(int minimumSize) {
+        int size = Math.max(minimumSize, this.nextSize);
+        MappedByteBuffer result = null;
+        try {
+            FileChannel channel = createSegment(size);
+            result = channel.map(FileChannel.MapMode.READ_WRITE, 0, size);
+        }
+        catch (IOException e)
+        {
+            System.err.println("IOException: allocateSegment failed with:" + e);
+        }
+
+
+        switch (this.allocationStrategy) {
+            case GROW_HEURISTICALLY:
+                if (size < this.maxSegmentBytes - this.nextSize) {
+                    this.nextSize += size;
+                } else {
+                    this.nextSize = maxSegmentBytes;
+                }
+                break;
+            case FIXED_SIZE:
+                break;
+        }
+
+
+
+        // if (size < this.maxSegmentBytes - this.nextSize) {
+        //     this.nextSize += size;
+        // } else {
+        //     this.nextSize = maxSegmentBytes;
+        // }
+        return result;
+    }
+
+
+    private static class State implements Runnable {
+        private final Map<Integer, RandomAccessFile> randomAccFiles;
+        private final String rPrefix;
+
+        State(Map<Integer, RandomAccessFile> files, String rPrefix) {
+            this.randomAccFiles = files;
+            this.rPrefix = rPrefix;
+        }
+
+        @Override
+        public void run() {
+            // Cleanup logic: delete all files
+            for (Map.Entry<Integer,RandomAccessFile> entry : randomAccFiles.entrySet()) {
+                try {
+                    entry.getValue().close();
+                }
+                catch (IOException e)
+                {
+                }
+                String name = nameForInt(rPrefix, entry.getKey());
+                File file = new File(name);
+                file.delete();
+            }
+        }
+    }
+
+    /**
+     * Explicit cleanup: WARNING: this invalidates all alloceted buffers,
+     * use close() after using the ByteBuffers.
+     */
+    public void close() {
+        cleanable.clean();
+    }
+}