diff --git a/RocksDbSharp/CompactionFilter.cs b/RocksDbSharp/CompactionFilter.cs new file mode 100644 index 0000000..99513fe --- /dev/null +++ b/RocksDbSharp/CompactionFilter.cs @@ -0,0 +1,23 @@ +using System; + +namespace RocksDbSharp +{ + public class CompactionFilter + { + public IntPtr Handle; + private readonly NameDelegate getNameDelegate; + private readonly FilterDelegate filterDelegate; + private readonly DestructorDelegate destroyDelegate; + + public CompactionFilter(NameDelegate nameDelegate, + FilterDelegate filterDelegate, + DestructorDelegate destroyDelegate, + IntPtr state) + { + this.getNameDelegate = nameDelegate; + this.filterDelegate = filterDelegate; + this.destroyDelegate = destroyDelegate; + Handle = Native.Instance.rocksdb_compactionfilter_create(state, destroyDelegate, filterDelegate, getNameDelegate); + } + } +} diff --git a/RocksDbSharp/FlushOptions.cs b/RocksDbSharp/FlushOptions.cs new file mode 100644 index 0000000..c83d835 --- /dev/null +++ b/RocksDbSharp/FlushOptions.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace RocksDbSharp +{ + public class FlushOptions: OptionsHandle + { + public FlushOptions() + { + Native.Instance.rocksdb_flushoptions_create(); + } + + public FlushOptions SetWaitForFlush(bool waitForFlush) + { + Native.Instance.rocksdb_flushoptions_set_wait(Handle, waitForFlush); + return this; + } + } +} diff --git a/RocksDbSharp/LiveFilesMetadata.cs b/RocksDbSharp/LiveFilesMetadata.cs new file mode 100644 index 0000000..d4504d4 --- /dev/null +++ b/RocksDbSharp/LiveFilesMetadata.cs @@ -0,0 +1,23 @@ +namespace RocksDbSharp +{ + public class LiveFileMetadata + { + public FileMetadata FileMetadata; + public FileDataMetadata FileDataMetadata; + } + + public class FileMetadata + { + public string FileName; + public int FileLevel; + public ulong FileSize; + } + + public class FileDataMetadata + { + public string SmallestKeyInFile; + public string LargestKeyInFile; + public ulong NumEntriesInFile; + public ulong NumDeletionsInFile; + } +} diff --git a/RocksDbSharp/RocksDb.cs b/RocksDbSharp/RocksDb.cs index 770898b..96af4cd 100644 --- a/RocksDbSharp/RocksDb.cs +++ b/RocksDbSharp/RocksDb.cs @@ -45,6 +45,11 @@ public static RocksDb Open(OptionsHandle options, string path) return new RocksDb(db, optionsReferences: null, cfOptionsRefs: null); } + public static void RepairDB(OptionsHandle options, string path) + { + Native.Instance.rocksdb_repair_db(options.Handle, path); + } + public static RocksDb OpenReadOnly(OptionsHandle options, string path, bool errorIfLogFileExists) { IntPtr db = Native.Instance.rocksdb_open_for_read_only(options.Handle, path, errorIfLogFileExists); @@ -168,7 +173,7 @@ public long Get(byte[] key, long keyLength, byte[] buffer, long offset, long len } } - public KeyValuePair[] MultiGet(byte[][] keys, ColumnFamilyHandle[] cf = null, ReadOptions readOptions = null) + public KeyValuePair[] MultiGet(byte[][] keys, ColumnFamilyHandle[] cf = null, ReadOptions readOptions = null) { return Native.Instance.rocksdb_multi_get(Handle, (readOptions ?? DefaultReadOptions).Handle, keys); } @@ -261,7 +266,7 @@ public void DropColumnFamily(string name) Native.Instance.rocksdb_drop_column_family(Handle, cf.Handle); columnFamilies.Remove(name); } - + public ColumnFamilyHandle GetDefaultColumnFamily() { return GetColumnFamily(ColumnFamilies.DefaultName); @@ -306,5 +311,120 @@ public void CompactRange(string start, string limit, ColumnFamilyHandle cf = nul encoding = Encoding.UTF8; CompactRange(start == null ? null : encoding.GetBytes(start), limit == null ? null : encoding.GetBytes(limit), cf); } + + public void Flush(FlushOptions flushOptions) + { + Native.Instance.rocksdb_flush(Handle, flushOptions.Handle); + } + + + /// + /// Returns metadata about the file and data in the file. + /// + /// setting it to true only populates FileName, + /// Filesize and filelevel; By default it is false + /// LiveFilesMetadata or null in case of failure + public List GetLiveFilesMetadata(bool populateFileMetadataOnly=false) + { + IntPtr buffer = Native.Instance.rocksdb_livefiles(Handle); + if (buffer == IntPtr.Zero) + { + return null; + } + + try + { + List filesMetadata = new List(); + + int fileCount = Native.Instance.rocksdb_livefiles_count(buffer); + for (int index = 0; index < fileCount; index++) + { + LiveFileMetadata liveFileMetadata = new LiveFileMetadata(); + + FileMetadata metadata = new FileMetadata(); + IntPtr fileMetadata = Native.Instance.rocksdb_livefiles_name(buffer, index); + string fileName = Marshal.PtrToStringAnsi(fileMetadata); + + int level = Native.Instance.rocksdb_livefiles_level(buffer, index); + + UIntPtr fS = Native.Instance.rocksdb_livefiles_size(buffer, index); + ulong fileSize = fS.ToUInt64(); + + metadata.FileName = fileName; + metadata.FileLevel = level; + metadata.FileSize = fileSize; + + liveFileMetadata.FileMetadata = metadata; + + if (!populateFileMetadataOnly) + { + FileDataMetadata fileDataMetadata = new FileDataMetadata(); + var smallestKeyPtr = Native.Instance.rocksdb_livefiles_smallestkey(buffer, + index, + out var smallestKeySize); + string smallestKey = Marshal.PtrToStringAnsi(smallestKeyPtr); + + var largestKeyPtr = Native.Instance.rocksdb_livefiles_largestkey(buffer, + index, + out var largestKeySize); + string largestKey = Marshal.PtrToStringAnsi(largestKeyPtr); + + ulong entries = Native.Instance.rocksdb_livefiles_entries(buffer, index); + ulong deletions = Native.Instance.rocksdb_livefiles_deletions(buffer, index); + + fileDataMetadata.SmallestKeyInFile = smallestKey; + fileDataMetadata.LargestKeyInFile = largestKey; + fileDataMetadata.NumEntriesInFile = entries; + fileDataMetadata.NumDeletionsInFile = deletions; + + liveFileMetadata.FileDataMetadata = fileDataMetadata; + } + + filesMetadata.Add(liveFileMetadata); + } + + return filesMetadata; + } + finally + { + Native.Instance.rocksdb_livefiles_destroy(buffer); + buffer = IntPtr.Zero; + } + } + + /// + /// Lean API to just get Live file names. + /// Refer to GetLiveFilesMetadata() for the complete metadata + /// + /// + public List GetLiveFileNames() + { + IntPtr buffer = Native.Instance.rocksdb_livefiles(Handle); + if (buffer == IntPtr.Zero) + { + return new List(); + } + + try + { + List liveFiles = new List(); + + int fileCount = Native.Instance.rocksdb_livefiles_count(buffer); + + for (int index = 0; index < fileCount; index++) + { + IntPtr fileMetadata = Native.Instance.rocksdb_livefiles_name(buffer, index); + string fileName = Marshal.PtrToStringAnsi(fileMetadata); + liveFiles.Add(fileName); + } + + return liveFiles; + } + finally + { + Native.Instance.rocksdb_livefiles_destroy(buffer); + buffer = IntPtr.Zero; + } + } } } diff --git a/RocksDbSharp/RocksDbSharp.csproj b/RocksDbSharp/RocksDbSharp.csproj index 9f03eff..11f6cb4 100644 --- a/RocksDbSharp/RocksDbSharp.csproj +++ b/RocksDbSharp/RocksDbSharp.csproj @@ -1,5 +1,5 @@  - + RocksDbSharp netstandard1.6;net40;net45 @@ -31,7 +31,7 @@ TRACE - + @@ -42,6 +42,6 @@ - + \ No newline at end of file diff --git a/tests/RocksDbSharpTest/AdditionalTests.cs b/tests/RocksDbSharpTest/AdditionalTests.cs new file mode 100644 index 0000000..79db352 --- /dev/null +++ b/tests/RocksDbSharpTest/AdditionalTests.cs @@ -0,0 +1,156 @@ +using System; +using Xunit; +using System.IO; +using RocksDbSharp; +using System.Linq; + +namespace RocksDbSharpTest +{ + public class AdditionalTests + { + public const string ReplacedText = "REPLACEMENT_TEXT"; + + [Fact] + public void TestFlush() + { + var dbName = "TestFlushDB"; + + DeleteDb(dbName); + + var options = new DbOptions().SetCreateIfMissing(true); + + using (var db = RocksDb.Open(options, dbName)) + { + db.Put("key", "value"); + } + + var sstFiles = Directory.EnumerateFiles(dbName).Where(s => s.EndsWith(".sst", StringComparison.OrdinalIgnoreCase)); + + Assert.True(!sstFiles.Any()); + + DeleteDb(dbName); + + FlushOptions flushOptions = new FlushOptions().SetWaitForFlush(true); + + using (var db = RocksDb.Open(options, dbName)) + { + db.Put("key", "value"); + db.Flush(flushOptions); + } + + sstFiles = Directory.EnumerateFiles(dbName) + .Where(s => s.EndsWith(".sst", StringComparison.OrdinalIgnoreCase)); + Assert.True(sstFiles.Any()); + } + + [Fact] + public void TestRepairDB() + { + var dbName = "TestFlushDB"; + DeleteDb(dbName); + var options = new DbOptions().SetCreateIfMissing(true); + var flushOptions = new FlushOptions().SetWaitForFlush(true); + + using (var db = RocksDb.Open(options, dbName)) + { + db.Put("key0", "value0"); + db.Flush(flushOptions); + } + + var firstSSTfile = Directory.EnumerateFiles(dbName) + .Where(s => s.EndsWith(".sst", StringComparison.OrdinalIgnoreCase)) + .FirstOrDefault(); + + var checkKey = "key1"; + var checkValue = "value0"; + using (var db = RocksDb.Open(options, dbName)) + { + db.Put(checkKey, checkValue); + db.Flush(flushOptions); + } + + File.Delete($"{firstSSTfile}"); + RocksDb.RepairDB(options, dbName); + + using (var db = RocksDb.Open(options, dbName)) + { + Assert.Equal(db.Get("key0"), null); + Assert.Equal(db.Get(checkKey), checkValue); + } + } + + [Fact] + public void TestLiveFiles() + { + var dbName = "TestLiveFiles"; + DeleteDb(dbName); + var options = new DbOptions().SetCreateIfMissing(true); + var flushOptions = new FlushOptions().SetWaitForFlush(true); + + { + using (var db = RocksDb.Open(options, dbName)) + { + var files = db.GetLiveFilesMetadata(); + + Assert.True(files.Count == 0); + } + } + + { + using (var db = RocksDb.Open(options, dbName)) + { + db.Put("key0", "value0"); + db.Put("key1", "value0"); + db.Flush(flushOptions); + + db.Put("key7", "value0"); + db.Put("key8", "value0"); + + db.Flush(flushOptions); + + var files = db.GetLiveFilesMetadata(); + var fileNames = files.Select(file => file.FileMetadata.FileName); + var fileList = Directory.EnumerateFiles(dbName); + + Assert.True(fileList.All(file => fileList.Contains(file))); + Assert.Equal(db.Get("key0"), "value0"); + } + } + } + + [Fact] + public void TestLiveFileNames() + { + var dbName = "TestLiveFiles"; + DeleteDb(dbName); + var options = new DbOptions().SetCreateIfMissing(true); + var flushOptions = new FlushOptions().SetWaitForFlush(true); + + using (var db = RocksDb.Open(options, dbName)) + { + db.Put("key0", "value0"); + db.Put("key1", "value0"); + db.Flush(flushOptions); + + db.Put("key7", "value0"); + db.Put("key8", "value0"); + + db.Flush(flushOptions); + + var files = db.GetLiveFileNames(); + var fileList = Directory.EnumerateFiles(dbName); + + Assert.True(fileList.All(file => fileList.Contains(file))); + Assert.Equal(db.Get("key0"), "value0"); + } + } + + public static void DeleteDb(string dbName) + { + if (Directory.Exists(dbName)) + { + Directory.Delete(dbName, true); + } + } + } +} diff --git a/tests/RocksDbSharpTest/CompactionFilterTest.cs b/tests/RocksDbSharpTest/CompactionFilterTest.cs new file mode 100644 index 0000000..ec0b4a2 --- /dev/null +++ b/tests/RocksDbSharpTest/CompactionFilterTest.cs @@ -0,0 +1,143 @@ +using RocksDbSharp; +using System; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading.Tasks; +using Xunit; + +namespace RocksDbSharpTest +{ + public class CompactionFilterTest + { + [Fact] + public void TestAllDelegates() + { + var dbName = "TestCompactionFilterDb"; + FlushOptions flushOptions = new FlushOptions().SetWaitForFlush(true); + DbOptions options = new DbOptions().SetCreateIfMissing(true); + var filter = new CompactionFilter(DefaultNameDelegate, + TestFilterDelegate, + DefaultDestructorDelegate, + IntPtr.Zero); + options.SetCompactionFilter(filter.Handle); + options.SetDisableAutoCompactions(1); + + SetupAndPopulate(dbName, flushOptions, options); + } + + private static void SetupAndPopulate(string dbName, FlushOptions flushOptions, DbOptions options) + { + DeleteDb(dbName); + using (var db = RocksDb.Open(options, dbName)) + { + byte[] key = Encoding.UTF8.GetBytes("keep_a"); + byte[] value = Encoding.UTF8.GetBytes("1"); + + db.Put(key, value); + + key = Encoding.UTF8.GetBytes("keep_b"); + value = Encoding.UTF8.GetBytes("2"); + + db.Put(key, value); + + key = Encoding.UTF8.GetBytes("Remove_c"); + value = Encoding.UTF8.GetBytes("3"); + + db.Put(key, value); + + key = Encoding.UTF8.GetBytes("Replace_d"); + value = Encoding.UTF8.GetBytes("4"); + + db.Put(key, value); + + db.Flush(flushOptions); + db.CompactRange(null, null, null); + + key = Encoding.UTF8.GetBytes("keep_a"); + var result = db.Get(key); + Assert.NotNull(result); + + key = Encoding.UTF8.GetBytes("Remove_c"); + result = db.Get(key); + Assert.Null(result); + + key = Encoding.UTF8.GetBytes("Replace_d"); + result = db.Get(key); + Assert.NotNull(result); + Assert.Equal(result, Encoding.UTF8.GetBytes("8")); + } + } + + public char TestFilterDelegate(IntPtr p0, int level, IntPtr key, UIntPtr key_length, IntPtr existing_value, UIntPtr value_length, IntPtr new_value, IntPtr new_value_length, IntPtr value_changed) + { + int keyLength = (int)key_length.ToUInt64(); + byte[] keyValue = new byte[keyLength]; + Marshal.Copy(key, keyValue, 0, keyLength); + + int valueLength = (int)value_length.ToUInt64(); + byte[] existingValue = new byte[valueLength]; + Marshal.Copy(existing_value, existingValue, 0, valueLength); + + var keyStr = Encoding.UTF8.GetString(keyValue); + if (keyStr.StartsWith("Remove")) + { + return (char)1; + } + + if (keyStr.StartsWith("Replace")) + { + var valueStr = Encoding.UTF8.GetString(existingValue); + int val = Convert.ToInt32(valueStr); + byte[] newValue = Encoding.UTF8.GetBytes($"{val * 2}"); + + if (newValue != null && newValue.Length != 0) + { + IntPtr buffer = Marshal.AllocHGlobal(newValue.Length); + try + { + Marshal.Copy(newValue, 0, buffer, newValue.Length); + Marshal.WriteIntPtr(new_value, buffer); + Marshal.WriteByte(value_changed, 1); + + var intSize = IntPtr.Size; + switch (intSize) + { + case 4: + Marshal.WriteInt32(new_value_length, newValue.Length); + break; + case 8: + Marshal.WriteInt64(new_value_length, newValue.Length); + break; + } + } + finally + { + Marshal.FreeHGlobal(buffer); + } + } + } + + return (char)0; + } + + public unsafe void DefaultDestructorDelegate(IntPtr state) + { + Marshal.FreeHGlobal(state); + } + + public unsafe IntPtr DefaultNameDelegate(IntPtr state) + { + return Marshal.StringToHGlobalAnsi("DefaultCompactionFilter"); + } + + public static void DeleteDb(string dbName) + { + if (Directory.Exists(dbName)) + { + Directory.Delete(dbName, true); + } + } + } +}