From 80dff54867a3dfd4ec32fa1d7bb09324f319e79f Mon Sep 17 00:00:00 2001 From: Ahmed Yasin Koculu Date: Sat, 17 Jun 2023 00:21:15 +0200 Subject: [PATCH] Add delete value confgiuration validation to prevent unexpected not working delete calls. --- src/Playground/Benchmark/OldTests.cs | 8 +-- src/Playground/Benchmark/ZoneTreeTestBase.cs | 3 +- src/ZoneTree.UnitTests/AtomicUpdateTests.cs | 14 ++++-- .../BottomSegmentMergeTests.cs | 2 + .../ExceptionlessTransactionTests.cs | 4 +- .../FixedSizeKeyAndValueTests.cs | 6 ++- src/ZoneTree.UnitTests/IteratorTests.cs | 8 +++ .../OptimisticTransactionTests.cs | 7 ++- src/ZoneTree.UnitTests/StringTreeTests.cs | 4 +- src/ZoneTree/Directory.Build.props | 4 +- .../DeleteValueConfigurationValidation.cs | 23 +++++++++ src/ZoneTree/Options/ZoneTreeOptions.cs | 49 +++++++++++++++++-- src/ZoneTree/ZoneTreeFactory.cs | 21 ++++++-- 13 files changed, 129 insertions(+), 24 deletions(-) create mode 100644 src/ZoneTree/Options/DeleteValueConfigurationValidation.cs diff --git a/src/Playground/Benchmark/OldTests.cs b/src/Playground/Benchmark/OldTests.cs index c229d48..4240dec 100644 --- a/src/Playground/Benchmark/OldTests.cs +++ b/src/Playground/Benchmark/OldTests.cs @@ -18,7 +18,7 @@ public sealed class OldTests { var recCount = count / 1000000.0 + "M"; Console.WriteLine("\r\n--------------------------"); - new StatsCollector().LogWithColor($"\r\n{mode} Insert {recCount}\r\n", ConsoleColor.Cyan); + new StatsCollector().LogWithColor($"\r\n{mode} Insert {recCount}\r\n", ConsoleColor.Cyan); string dataPath = GetDataPath(mode, count); if (TestConfig.RecreateDatabases && Directory.Exists(dataPath)) Directory.Delete(dataPath, true); @@ -51,7 +51,7 @@ public sealed class OldTests new StatsCollector().LogWithColor( "Completed in:", - stopWatch.ElapsedMilliseconds, + stopWatch.ElapsedMilliseconds, ConsoleColor.Green); stopWatch.Restart(); @@ -179,7 +179,8 @@ public sealed class OldTests stopWatch.ElapsedMilliseconds, ConsoleColor.DarkYellow); - for (var i = 0; i < amount; ++i) { + for (var i = 0; i < amount; ++i) + { zoneTree.Upsert(key, key + key); ++key; } @@ -271,6 +272,7 @@ public sealed class OldTests private static IZoneTree OpenOrCreateZoneTree(WriteAheadLogMode mode, string dataPath) { return new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetMutableSegmentMaxItemCount(TestConfig.MutableSegmentMaxItemCount) .SetDiskSegmentCompression(TestConfig.EnableDiskSegmentCompression) .SetDiskSegmentCompressionBlockSize(TestConfig.DiskCompressionBlockSize) diff --git a/src/Playground/Benchmark/ZoneTreeTestBase.cs b/src/Playground/Benchmark/ZoneTreeTestBase.cs index f8cf248..5e191a5 100644 --- a/src/Playground/Benchmark/ZoneTreeTestBase.cs +++ b/src/Playground/Benchmark/ZoneTreeTestBase.cs @@ -32,6 +32,7 @@ public class ZoneTreeTestBase protected ZoneTreeFactory GetFactory() { return new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetMutableSegmentMaxItemCount(TestConfig.MutableSegmentMaxItemCount) .SetDiskSegmentMaxItemCount(TestConfig.DiskSegmentMaxItemCount) .SetDiskSegmentCompression(TestConfig.EnableDiskSegmentCompression) @@ -87,7 +88,7 @@ public class ZoneTreeTestBase long totalBytes = 0; foreach (var file in files) { - var finfo = new FileInfo(file); + var finfo = new FileInfo(file); totalBytes += finfo.Length; } stats.AddAdditionalStats("Disk Usage", totalBytes.Bytes().Humanize()); diff --git a/src/ZoneTree.UnitTests/AtomicUpdateTests.cs b/src/ZoneTree.UnitTests/AtomicUpdateTests.cs index 94dd116..8985f92 100644 --- a/src/ZoneTree.UnitTests/AtomicUpdateTests.cs +++ b/src/ZoneTree.UnitTests/AtomicUpdateTests.cs @@ -16,6 +16,7 @@ public sealed class AtomicUpdateTests Directory.Delete(dataPath, true); var counterKey = -3999; using var data = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetComparer(new Int32ComparerDescending()) .SetMutableSegmentMaxItemCount(500) .SetDataDirectory(dataPath) @@ -42,7 +43,7 @@ public sealed class AtomicUpdateTests len = random.Next(1501); for (var i = 0; i < len; ++i) { - data.TryAtomicAddOrUpdate(counterKey, 0, + data.TryAtomicAddOrUpdate(counterKey, 0, bool (ref int y) => { ++y; @@ -86,6 +87,7 @@ public sealed class AtomicUpdateTests Directory.Delete(dataPath, true); using var data = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetComparer(new Int32ComparerDescending()) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) @@ -109,8 +111,9 @@ public sealed class AtomicUpdateTests len = random.Next(1501); for (var i = 0; i < len; ++i) { - data.TryAtomicAddOrUpdate(3999, 0, - bool (ref int y) => { + data.TryAtomicAddOrUpdate(3999, 0, + bool (ref int y) => + { ++y; return true; }); @@ -150,6 +153,7 @@ public sealed class AtomicUpdateTests Directory.Delete(dataPath, true); var counterKey = -3999; using var data = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetComparer(new Int32ComparerDescending()) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) @@ -174,7 +178,8 @@ public sealed class AtomicUpdateTests for (var i = 0; i < len; ++i) { data.TryAtomicAddOrUpdate(counterKey, 0, - bool (ref int y) => { + bool (ref int y) => + { ++y; return true; }); @@ -213,6 +218,7 @@ public sealed class AtomicUpdateTests if (Directory.Exists(dataPath)) Directory.Delete(dataPath, true); using var data = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetComparer(new Int32ComparerDescending()) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) diff --git a/src/ZoneTree.UnitTests/BottomSegmentMergeTests.cs b/src/ZoneTree.UnitTests/BottomSegmentMergeTests.cs index 813359a..fe4714f 100644 --- a/src/ZoneTree.UnitTests/BottomSegmentMergeTests.cs +++ b/src/ZoneTree.UnitTests/BottomSegmentMergeTests.cs @@ -13,6 +13,7 @@ public sealed class BottomSegmentMergeTests Directory.Delete(dataPath, true); var zoneTree = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetDiskSegmentMaxItemCount(10) .SetDataDirectory(dataPath) .ConfigureDiskSegmentOptions( @@ -56,6 +57,7 @@ public sealed class BottomSegmentMergeTests zoneTree.Dispose(); zoneTree = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetDiskSegmentMaxItemCount(10) .SetDataDirectory(dataPath) .OpenOrCreate(); diff --git a/src/ZoneTree.UnitTests/ExceptionlessTransactionTests.cs b/src/ZoneTree.UnitTests/ExceptionlessTransactionTests.cs index af1888a..5968aae 100644 --- a/src/ZoneTree.UnitTests/ExceptionlessTransactionTests.cs +++ b/src/ZoneTree.UnitTests/ExceptionlessTransactionTests.cs @@ -17,6 +17,7 @@ public sealed class ExceptionlessTransactionTests Directory.Delete(dataPath, true); using var zoneTree = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) .OpenOrCreateTransactional(); @@ -63,6 +64,7 @@ public sealed class ExceptionlessTransactionTests Directory.Delete(dataPath, true); using var zoneTree = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) .ConfigureWriteAheadLogOptions(x => @@ -103,7 +105,7 @@ public sealed class ExceptionlessTransactionTests if (transaction.TotalPendingTransactionsRetried > 0) Console.WriteLine("pending:" + transaction.TotalPendingTransactionsRetried); }); - + zoneTree.Maintenance.DestroyTree(); } } \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/FixedSizeKeyAndValueTests.cs b/src/ZoneTree.UnitTests/FixedSizeKeyAndValueTests.cs index f5b2e91..3c529d4 100644 --- a/src/ZoneTree.UnitTests/FixedSizeKeyAndValueTests.cs +++ b/src/ZoneTree.UnitTests/FixedSizeKeyAndValueTests.cs @@ -12,6 +12,7 @@ public sealed class FixedSizeKeyAndValueTests if (Directory.Exists(dataPath)) Directory.Delete(dataPath, true); using var data = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetMutableSegmentMaxItemCount(5) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) @@ -41,6 +42,7 @@ public sealed class FixedSizeKeyAndValueTests Directory.Delete(dataPath, true); using var data = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetMutableSegmentMaxItemCount(5) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) @@ -73,6 +75,7 @@ public sealed class FixedSizeKeyAndValueTests Directory.Delete(dataPath, true); using var data = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetMutableSegmentMaxItemCount(5) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) @@ -107,6 +110,7 @@ public sealed class FixedSizeKeyAndValueTests Directory.Delete(dataPath, true); using var data = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetMutableSegmentMaxItemCount(5) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) @@ -139,7 +143,7 @@ public sealed class FixedSizeKeyAndValueTests .SetMutableSegmentMaxItemCount(5) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) - .ConfigureWriteAheadLogOptions(x => + .ConfigureWriteAheadLogOptions(x => x.WriteAheadLogMode = WriteAheadLogMode.Sync) .SetIsValueDeletedDelegate((in int x) => x == -1) .SetMarkValueDeletedDelegate((ref int x) => x = -1) diff --git a/src/ZoneTree.UnitTests/IteratorTests.cs b/src/ZoneTree.UnitTests/IteratorTests.cs index 3306ca9..691911c 100644 --- a/src/ZoneTree.UnitTests/IteratorTests.cs +++ b/src/ZoneTree.UnitTests/IteratorTests.cs @@ -245,6 +245,7 @@ public sealed class IteratorTests var iteratorCount = 1000; using var zoneTree = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetMutableSegmentMaxItemCount(insertCount * 2) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) @@ -293,6 +294,7 @@ public sealed class IteratorTests var iteratorCount = 1000; using var zoneTree = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetMutableSegmentMaxItemCount(insertCount * 2) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) @@ -355,6 +357,7 @@ public sealed class IteratorTests var iteratorCount = 1000; using var zoneTree = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetMutableSegmentMaxItemCount(insertCount * 2) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) @@ -397,6 +400,7 @@ public sealed class IteratorTests Directory.Delete(dataPath, true); using var zoneTree = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) .OpenOrCreate(); @@ -438,6 +442,7 @@ public sealed class IteratorTests Directory.Delete(dataPath, true); using var zoneTree = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) .OpenOrCreate(); @@ -486,6 +491,7 @@ public sealed class IteratorTests Directory.Delete(dataPath, true); using var zoneTree = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) .SetComparer(new StringCurrentCultureComparerAscending()) @@ -560,6 +566,7 @@ public sealed class IteratorTests Directory.Delete(dataPath, true); var zoneTree = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) .SetComparer(new StringCurrentCultureComparerAscending()) @@ -582,6 +589,7 @@ public sealed class IteratorTests zoneTree.Dispose(); zoneTree = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) .SetComparer(new StringCurrentCultureComparerAscending()) diff --git a/src/ZoneTree.UnitTests/OptimisticTransactionTests.cs b/src/ZoneTree.UnitTests/OptimisticTransactionTests.cs index 18f9984..1e0bb77 100644 --- a/src/ZoneTree.UnitTests/OptimisticTransactionTests.cs +++ b/src/ZoneTree.UnitTests/OptimisticTransactionTests.cs @@ -17,6 +17,7 @@ public sealed class OptimisticTransactionTests Directory.Delete(dataPath, true); using var zoneTree = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) .OpenOrCreateTransactional(); @@ -64,6 +65,7 @@ public sealed class OptimisticTransactionTests int n = 10000; using var zoneTree = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) .ConfigureWriteAheadLogOptions(x => x.WriteAheadLogMode = walMode) @@ -91,6 +93,7 @@ public sealed class OptimisticTransactionTests int n = 10000; using var zoneTree = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) .ConfigureWriteAheadLogOptions(x => x.WriteAheadLogMode = walMode) @@ -165,7 +168,7 @@ public sealed class OptimisticTransactionTests var dataPath = "data/TransactionLogCompactionTest" + compactionThreshold; if (Directory.Exists(dataPath)) Directory.Delete(dataPath, true); - + using var zoneTree = new ZoneTreeFactory() .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) @@ -269,7 +272,7 @@ public sealed class OptimisticTransactionTests Assert.IsTrue(zoneTree.TryGetNoThrow(tx1, 5, out var v5).IsAborted); Assert.IsTrue(zoneTree.TryGetNoThrow(tx2, 5, out v5).Succeeded); Assert.That(v5, Is.EqualTo(12)); - + // tx2 depends on tx1 bcs of read of key 9. tx1 aborted and so tx2. Assert.That(zoneTree.PrepareAndCommitNoThrow(tx2).IsAborted, Is.True); diff --git a/src/ZoneTree.UnitTests/StringTreeTests.cs b/src/ZoneTree.UnitTests/StringTreeTests.cs index 7012020..45b715b 100644 --- a/src/ZoneTree.UnitTests/StringTreeTests.cs +++ b/src/ZoneTree.UnitTests/StringTreeTests.cs @@ -15,6 +15,7 @@ public sealed class StringTreeTests Directory.Delete(dataPath, true); using var zoneTree = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetDataDirectory(dataPath) .SetWriteAheadLogDirectory(dataPath) .OpenOrCreate(); @@ -27,7 +28,7 @@ public sealed class StringTreeTests { "Zbc", "DDD", "Abc", "Cbc", "Dbc", null }; - for(var i = 0; i < keys.Length; i++) + for (var i = 0; i < keys.Length; i++) { zoneTree.Upsert(keys[i], values[i]); } @@ -65,6 +66,7 @@ public sealed class StringTreeTests for (var i = 0; i < 2; ++i) { using var db = new ZoneTreeFactory() + .DisableDeleteValueConfigurationValidation(false) .SetDataDirectory(dataPath) .OpenOrCreate(); db.Upsert("0", 123); diff --git a/src/ZoneTree/Directory.Build.props b/src/ZoneTree/Directory.Build.props index 85dde03..f2b68dd 100644 --- a/src/ZoneTree/Directory.Build.props +++ b/src/ZoneTree/Directory.Build.props @@ -5,8 +5,8 @@ Ahmed Yasin Koculu ZoneTree ZoneTree - 1.6.3.0 - 1.6.3.0 + 1.6.4.0 + 1.6.4.0 Ahmed Yasin Koculu ZoneTree ZoneTree is a persistent, high-performance, transactional, ACID-compliant ordered key-value database for NET. It can operate in memory or on local/cloud storage. diff --git a/src/ZoneTree/Options/DeleteValueConfigurationValidation.cs b/src/ZoneTree/Options/DeleteValueConfigurationValidation.cs new file mode 100644 index 0000000..33e55b4 --- /dev/null +++ b/src/ZoneTree/Options/DeleteValueConfigurationValidation.cs @@ -0,0 +1,23 @@ +namespace Tenray.ZoneTree.Options; + +/// +/// Defines the validation behavior +/// of not providing the delete value delegates. +/// +public enum DeleteValueConfigurationValidation +{ + /// + /// Throws an error if the value deletion delegates are not configured. + /// + Required, + + /// + /// Logs a warning if the value deletion delegates are not configured. + /// + Warning, + + /// + /// Allows creating ZoneTree without delete record support. + /// + NotRequired, +} diff --git a/src/ZoneTree/Options/ZoneTreeOptions.cs b/src/ZoneTree/Options/ZoneTreeOptions.cs index 546f0c7..c341266 100644 --- a/src/ZoneTree/Options/ZoneTreeOptions.cs +++ b/src/ZoneTree/Options/ZoneTreeOptions.cs @@ -109,33 +109,66 @@ public sealed class ZoneTreeOptions { if (KeySerializer == null) { - exception = new MissingOptionException("KeySerializer"); + exception = new MissingOptionException(nameof(KeySerializer)); return false; } if (ValueSerializer == null) { - exception = new MissingOptionException("ValueSerializer"); + exception = new MissingOptionException(nameof(ValueSerializer)); return false; } if (Comparer == null) { - exception = new MissingOptionException("Comparer"); + exception = new MissingOptionException(nameof(Comparer)); return false; } if (RandomAccessDeviceManager == null) { - exception = new MissingOptionException("RandomAccessDeviceManager"); + exception = new MissingOptionException(nameof(RandomAccessDeviceManager)); return false; } if (WriteAheadLogProvider == null) { - exception = new MissingOptionException("WriteAheadLogProvider"); + exception = new MissingOptionException(nameof(WriteAheadLogProvider)); return false; } + switch (DeleteValueConfigurationValidation) + { + case DeleteValueConfigurationValidation.Required: + { + if (IsValueDeleted == DefaultIsValueDeleted) + { + exception = new MissingOptionException(nameof(IsValueDeleted)); + return false; + } + + if (MarkValueDeleted == DefaultMarkValueDeleted) + { + exception = new MissingOptionException(nameof(MarkValueDeleted)); + return false; + } + } + break; + case DeleteValueConfigurationValidation.Warning: + { + if (IsValueDeleted == DefaultIsValueDeleted) + { + Logger.LogWarning(new MissingOptionException(nameof(IsValueDeleted))); + } + + if (MarkValueDeleted == DefaultMarkValueDeleted) + { + Logger.LogWarning(new MissingOptionException(nameof(MarkValueDeleted))); + } + } + break; + default: break; + } + exception = ValidateCompressionLevel( "disk segment", DiskSegmentOptions.CompressionMethod, @@ -202,4 +235,10 @@ public sealed class ZoneTreeOptions /// Configures the random access device manager. /// public IRandomAccessDeviceManager RandomAccessDeviceManager { get; set; } + + /// + /// Defines the validation behavior + /// of not providing the delete value delegates. + /// + public DeleteValueConfigurationValidation DeleteValueConfigurationValidation { get; set; } } diff --git a/src/ZoneTree/ZoneTreeFactory.cs b/src/ZoneTree/ZoneTreeFactory.cs index 12b0dbb..f722415 100644 --- a/src/ZoneTree/ZoneTreeFactory.cs +++ b/src/ZoneTree/ZoneTreeFactory.cs @@ -48,7 +48,7 @@ public sealed class ZoneTreeFactory FileStreamProvider = fileStreamProvider; GetWriteAheadLogProvider = (options) => - WalDirectory == null ? + WalDirectory == null ? new WriteAheadLogProvider(options.Logger, fileStreamProvider) : new WriteAheadLogProvider(options.Logger, fileStreamProvider, WalDirectory); } @@ -165,6 +165,19 @@ public sealed class ZoneTreeFactory return this; } + /// + /// Disables the delete value configuration validation. + /// + /// If true, validation logs a warning when value deletion is not configured. + /// ZoneTree Factory + public ZoneTreeFactory DisableDeleteValueConfigurationValidation(bool keepWarning = true) + { + Options.DeleteValueConfigurationValidation = keepWarning ? + DeleteValueConfigurationValidation.Warning : + DeleteValueConfigurationValidation.NotRequired; + return this; + } + /// /// Sets the maximum cached block count. /// @@ -424,7 +437,7 @@ public sealed class ZoneTreeFactory }; if (typeof(TKey) == typeof(string)) - Options.KeySerializer = + Options.KeySerializer = new Utf8StringSerializer() as ISerializer; else if (typeof(TKey) == typeof(byte[])) @@ -521,7 +534,7 @@ public sealed class ZoneTreeFactory /// ZoneTree Factory public ITransactionalZoneTree OpenOrCreateTransactional() { - var zoneTree = OpenOrCreate(); + var zoneTree = OpenOrCreate(); InitTransactionLog(); return new OptimisticZoneTree(Options, TransactionLog, zoneTree); } @@ -533,7 +546,7 @@ public sealed class ZoneTreeFactory /// Thrown when the database exists in the location. public ITransactionalZoneTree CreateTransactional() { - var zoneTree = Create(); + var zoneTree = Create(); InitTransactionLog(); return new OptimisticZoneTree(Options, TransactionLog, zoneTree); }