First implementation of new SI API

cs/benchmark/FasterSpanByteYcsbBenchmark.cs

@@ -300,7 +300,7 @@ public unsafe void Run()
             long startTailAddress = store.Log.TailAddress;
             Console.WriteLine("Start tail address = " + startTailAddress);
 
-            if (!storeWasRecovered && this.backupMode.HasFlag(BackupMode.Backukp) && kPeriodicCheckpointMilliseconds <= 0)
+            if (!storeWasRecovered && this.backupMode.HasFlag(BackupMode.Backup) && kPeriodicCheckpointMilliseconds <= 0)
             {
                 Console.WriteLine("Checkpointing FasterKV for fast restart");
                 store.TakeFullCheckpoint(out _);

cs/benchmark/FasterYcsbBenchmark.cs

@@ -34,7 +34,7 @@ public enum Op : ulong
         const int kPeriodicCheckpointMilliseconds = 0;
 #else
         const bool kDumpDistribution = false;
-        const bool kUseSmallData = false;
+        const bool kUseSmallData = true; // false;
         const bool kUseSyntheticData = false;
         const bool kSmallMemoryLog = false;
         const bool kAffinitizedSession = true;
@@ -79,10 +79,11 @@ public enum Op : ulong
         readonly string distribution;
         readonly int readPercent;
         readonly Functions functions = new Functions();
+        SecondaryIndexType secondaryIndexType = SecondaryIndexType.None;
 
         volatile bool done = false;
 
-        public FASTER_YcsbBenchmark(int threadCount_, int numaStyle_, string distribution_, int readPercent_, int backupOptions_)
+        public FASTER_YcsbBenchmark(int threadCount_, int numaStyle_, string distribution_, int readPercent_, int backupOptions_, int indexType_)
         {
             // Pin loading thread if it is not used for checkpointing
             if (kPeriodicCheckpointMilliseconds <= 0)
@@ -113,10 +114,22 @@ public FASTER_YcsbBenchmark(int threadCount_, int numaStyle_, string distributio
 
             if (kSmallMemoryLog)
                 store = new FasterKV<Key, Value>
-                    (kMaxKey / 2, new LogSettings { LogDevice = device, PreallocateLog = true, PageSizeBits = 22, SegmentSizeBits = 26, MemorySizeBits = 26 }, new CheckpointSettings { CheckPointType = CheckpointType.FoldOver, CheckpointDir = path });
+                    (kMaxKey / 2, new LogSettings { LogDevice = device, PreallocateLog = true, PageSizeBits = 22, SegmentSizeBits = 26, MemorySizeBits = 26 },
+                    new CheckpointSettings { CheckPointType = CheckpointType.FoldOver, CheckpointDir = path },
+                    supportsMutableIndexes: secondaryIndexType != SecondaryIndexType.None);
             else
                 store = new FasterKV<Key, Value>
-                    (kMaxKey / 2, new LogSettings { LogDevice = device, PreallocateLog = true }, new CheckpointSettings { CheckPointType = CheckpointType.FoldOver, CheckpointDir = path });
+                    (kMaxKey / 2, new LogSettings { LogDevice = device, PreallocateLog = true },
+                    new CheckpointSettings { CheckPointType = CheckpointType.FoldOver, CheckpointDir = path },
+                    supportsMutableIndexes: secondaryIndexType != SecondaryIndexType.None);
+
+            if (secondaryIndexType != SecondaryIndexType.None)
+            {
+                if (secondaryIndexType.HasFlag(SecondaryIndexType.Key))
+                    store.SecondaryIndexBroker.AddIndex(new NullKeyIndex<Key>());
+                if (secondaryIndexType.HasFlag(SecondaryIndexType.Value))
+                    store.SecondaryIndexBroker.AddIndex(new NullValueIndex<Value>());
+            }
         }
 
         private void RunYcsb(int thread_idx)
@@ -146,7 +159,7 @@ private void RunYcsb(int thread_idx)
             int count = 0;
 #endif
 
-            var session = store.For(functions).NewSession<Functions>(null, kAffinitizedSession);
+            var session = store.For(functions).NewSession<Functions>(null, threadAffinitized: secondaryIndexType == SecondaryIndexType.None);
 
             while (!done)
             {
@@ -295,7 +308,7 @@ public unsafe void Run()
             long startTailAddress = store.Log.TailAddress;
             Console.WriteLine("Start tail address = " + startTailAddress);
 
-            if (!storeWasRecovered && this.backupMode.HasFlag(BackupMode.Backukp) && kPeriodicCheckpointMilliseconds <= 0)
+            if (!storeWasRecovered && this.backupMode.HasFlag(BackupMode.Backup) && kPeriodicCheckpointMilliseconds <= 0)
             {
                 Console.WriteLine("Checkpointing FasterKV for fast restart");
                 store.TakeFullCheckpoint(out _);
@@ -383,7 +396,7 @@ private void SetupYcsb(int thread_idx)
             else
                 Native32.AffinitizeThreadShardedNuma((uint)thread_idx, 2); // assuming two NUMA sockets
 
-            var session = store.For(functions).NewSession<Functions>(null, kAffinitizedSession);
+            var session = store.For(functions).NewSession<Functions>(null, threadAffinitized: secondaryIndexType == SecondaryIndexType.None);
 
 #if DASHBOARD
             var tstart = Stopwatch.GetTimestamp();

cs/benchmark/Functions.cs

@@ -80,5 +80,11 @@ public void CopyUpdater(ref Key key, ref Input input, ref Value oldValue, ref Va
         {
             newValue.value = input.value + oldValue.value;
         }
+
+        public bool SupportsLocks => false;
+
+        public void Lock(ref RecordInfo recordInfo, ref Key key, ref Value value, OperationType opType, ref long context) { }
+
+        public bool Unlock(ref RecordInfo recordInfo, ref Key key, ref Value value, OperationType opType, long context) => true;
     }
 }

cs/benchmark/Program.cs

@@ -28,6 +28,14 @@ class Options
                         "\n    3 = Both (Recover FasterKV if the Checkpoint is available, else populate FasterKV from data and Checkpoint it so it can be Restored in a subsequent run)")]
         public int Backup { get; set; }
 
+        [Option('i', "index", Required = false, Default = 0,
+             HelpText = "Secondary index type(s); these implement a no-op index to test the overhead on FasterKV operations:" +
+                        "\n    0 = None (default)" +
+                        "\n    1 = Key-based index" +
+                        "\n    2 = Value-based index" +
+                        "\n    3 = Both index types")]
+        public int SecondaryIndexType { get; set; }
+
         [Option('r', "read_percent", Required = false, Default = 50,
          HelpText = "Percentage of reads (-1 for 100% read-modify-write")]
         public int ReadPercent { get; set; }
@@ -44,7 +52,13 @@ enum BenchmarkType : int
 
     [Flags] enum BackupMode : int
     {
-        None, Restore, Backukp, Both
+        None, Restore, Backup, Both
+    };
+
+    [Flags]
+    enum SecondaryIndexType : int
+    {
+        None, Key, Value, Both
     };
 
     public class Program
@@ -62,7 +76,7 @@ public static void Main(string[] args)
 
             if (b == BenchmarkType.Ycsb)
             {
-                var test = new FASTER_YcsbBenchmark(options.ThreadCount, options.NumaStyle, options.Distribution, options.ReadPercent, options.Backup);
+                var test = new FASTER_YcsbBenchmark(options.ThreadCount, options.NumaStyle, options.Distribution, options.ReadPercent, options.Backup, options.SecondaryIndexType);
                 test.Run();
             }
             else if (b == BenchmarkType.SpanByte)

cs/benchmark/SecondaryIndexes.cs

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+using FASTER.core;
+
+namespace FASTER.benchmark
+{
+    class NullKeyIndex<Key> : ISecondaryKeyIndex<Key>
+    {
+        public string Name => "KeyIndex";
+
+        public bool IsMutable => true;
+
+        public void Delete(ref Key key) { }
+
+        public void Insert(ref Key key) { }
+
+        public void Upsert(ref Key key, bool isMutable) { }
+    }
+
+    class NullValueIndex<Value> : ISecondaryValueIndex<Value>
+    {
+        public string Name => "ValueIndex";
+
+        public bool IsMutable => true;
+
+        public void Delete(long recordId) { }
+
+        public void Insert(ref Value value, long recordId) { }
+
+        public void Upsert(ref Value value, long recordId, bool isMutable) { }
+    }
+}

cs/samples/ReadAddress/Types.cs

@@ -43,14 +43,14 @@ public class Context
     public class Functions : AdvancedSimpleFunctions<Key, Value, Context>
     {
         // Return false to force a chain of values.
-        public override bool ConcurrentWriter(ref Key key, ref Value src, ref Value dst, long address) => false;
+        public override bool ConcurrentWriter(ref Key key, ref Value src, ref Value dst, ref RecordInfo recordInfo, long address) => false;
 
-        public override bool InPlaceUpdater(ref Key key, ref Value input, ref Value value, long address) => false;
+        public override bool InPlaceUpdater(ref Key key, ref Value input, ref Value value, ref RecordInfo recordInfo, long address) => false;
 
         // Track the recordInfo for its PreviousAddress.
         public override void ReadCompletionCallback(ref Key key, ref Value input, ref Value output, Context ctx, Status status, RecordInfo recordInfo)
         {
-            if (!(ctx is null))
+            if (ctx is { })
             {
                 ctx.recordInfo = recordInfo;
                 ctx.status = status;

cs/src/core/Allocator/GenericAllocator.cs

@@ -48,20 +48,17 @@ public GenericAllocator(LogSettings settings, SerializerSettings<Key, Value> ser
             }
 
             SerializerSettings = serializerSettings;
+            SerializerSettings = serializerSettings ?? new SerializerSettings<Key, Value>();
 
             if ((!keyBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.keySerializer == null)))
             {
                 Debug.WriteLine("Key is not blittable, but no serializer specified via SerializerSettings. Using (slow) DataContractSerializer as default.");
-                if (SerializerSettings == null)
-                    SerializerSettings = new SerializerSettings<Key, Value>();
                 SerializerSettings.keySerializer = ObjectSerializer.Get<Key>();
             }
 
             if ((!valueBlittable) && (settings.LogDevice as NullDevice == null) && ((SerializerSettings == null) || (SerializerSettings.valueSerializer == null)))
             {
                 Debug.WriteLine("Value is not blittable, but no serializer specified via SerializerSettings. Using (slow) DataContractSerializer as default.");
-                if (SerializerSettings == null)
-                    SerializerSettings = new SerializerSettings<Key, Value>();
                 SerializerSettings.valueSerializer = ObjectSerializer.Get<Value>();
             }