Merge pull request #37 from koculu/validate-delete-value-configuration

Validate delete value configuration.
This commit is contained in:
Ahmed Yasin Koculu
2023-06-17 01:31:49 +02:00
committed by GitHub
14 changed files with 217 additions and 24 deletions
+5 -3
View File
@@ -18,7 +18,7 @@ public sealed class OldTests
{
var recCount = count / 1000000.0 + "M";
Console.WriteLine("\r\n--------------------------");
new StatsCollector().LogWithColor($"\r\n{mode} Insert <int,int> {recCount}\r\n", ConsoleColor.Cyan);
new StatsCollector().LogWithColor($"\r\n{mode} Insert <int,int> {recCount}\r\n", ConsoleColor.Cyan);
string dataPath = GetDataPath(mode, count);
if (TestConfig.RecreateDatabases && Directory.Exists(dataPath))
Directory.Delete(dataPath, true);
@@ -51,7 +51,7 @@ public sealed class OldTests
new StatsCollector().LogWithColor(
"Completed in:",
stopWatch.ElapsedMilliseconds,
stopWatch.ElapsedMilliseconds,
ConsoleColor.Green);
stopWatch.Restart();
@@ -179,7 +179,8 @@ public sealed class OldTests
stopWatch.ElapsedMilliseconds,
ConsoleColor.DarkYellow);
for (var i = 0; i < amount; ++i) {
for (var i = 0; i < amount; ++i)
{
zoneTree.Upsert(key, key + key);
++key;
}
@@ -271,6 +272,7 @@ public sealed class OldTests
private static IZoneTree<int, int> OpenOrCreateZoneTree(WriteAheadLogMode mode, string dataPath)
{
return new ZoneTreeFactory<int, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetMutableSegmentMaxItemCount(TestConfig.MutableSegmentMaxItemCount)
.SetDiskSegmentCompression(TestConfig.EnableDiskSegmentCompression)
.SetDiskSegmentCompressionBlockSize(TestConfig.DiskCompressionBlockSize)
+2 -1
View File
@@ -32,6 +32,7 @@ public class ZoneTreeTestBase<TKey, TValue>
protected ZoneTreeFactory<TKey, TValue> GetFactory()
{
return new ZoneTreeFactory<TKey, TValue>()
.DisableDeleteValueConfigurationValidation(false)
.SetMutableSegmentMaxItemCount(TestConfig.MutableSegmentMaxItemCount)
.SetDiskSegmentMaxItemCount(TestConfig.DiskSegmentMaxItemCount)
.SetDiskSegmentCompression(TestConfig.EnableDiskSegmentCompression)
@@ -87,7 +88,7 @@ public class ZoneTreeTestBase<TKey, TValue>
long totalBytes = 0;
foreach (var file in files)
{
var finfo = new FileInfo(file);
var finfo = new FileInfo(file);
totalBytes += finfo.Length;
}
stats.AddAdditionalStats("Disk Usage", totalBytes.Bytes().Humanize());
+10 -4
View File
@@ -16,6 +16,7 @@ public sealed class AtomicUpdateTests
Directory.Delete(dataPath, true);
var counterKey = -3999;
using var data = new ZoneTreeFactory<int, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetComparer(new Int32ComparerDescending())
.SetMutableSegmentMaxItemCount(500)
.SetDataDirectory(dataPath)
@@ -42,7 +43,7 @@ public sealed class AtomicUpdateTests
len = random.Next(1501);
for (var i = 0; i < len; ++i)
{
data.TryAtomicAddOrUpdate(counterKey, 0,
data.TryAtomicAddOrUpdate(counterKey, 0,
bool (ref int y) =>
{
++y;
@@ -86,6 +87,7 @@ public sealed class AtomicUpdateTests
Directory.Delete(dataPath, true);
using var data = new ZoneTreeFactory<int, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetComparer(new Int32ComparerDescending())
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
@@ -109,8 +111,9 @@ public sealed class AtomicUpdateTests
len = random.Next(1501);
for (var i = 0; i < len; ++i)
{
data.TryAtomicAddOrUpdate(3999, 0,
bool (ref int y) => {
data.TryAtomicAddOrUpdate(3999, 0,
bool (ref int y) =>
{
++y;
return true;
});
@@ -150,6 +153,7 @@ public sealed class AtomicUpdateTests
Directory.Delete(dataPath, true);
var counterKey = -3999;
using var data = new ZoneTreeFactory<int, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetComparer(new Int32ComparerDescending())
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
@@ -174,7 +178,8 @@ public sealed class AtomicUpdateTests
for (var i = 0; i < len; ++i)
{
data.TryAtomicAddOrUpdate(counterKey, 0,
bool (ref int y) => {
bool (ref int y) =>
{
++y;
return true;
});
@@ -213,6 +218,7 @@ public sealed class AtomicUpdateTests
if (Directory.Exists(dataPath))
Directory.Delete(dataPath, true);
using var data = new ZoneTreeFactory<int, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetComparer(new Int32ComparerDescending())
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
@@ -13,6 +13,7 @@ public sealed class BottomSegmentMergeTests
Directory.Delete(dataPath, true);
var zoneTree = new ZoneTreeFactory<int, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetDiskSegmentMaxItemCount(10)
.SetDataDirectory(dataPath)
.ConfigureDiskSegmentOptions(
@@ -56,6 +57,7 @@ public sealed class BottomSegmentMergeTests
zoneTree.Dispose();
zoneTree = new ZoneTreeFactory<int, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetDiskSegmentMaxItemCount(10)
.SetDataDirectory(dataPath)
.OpenOrCreate();
@@ -17,6 +17,7 @@ public sealed class ExceptionlessTransactionTests
Directory.Delete(dataPath, true);
using var zoneTree = new ZoneTreeFactory<int, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
.OpenOrCreateTransactional();
@@ -63,6 +64,7 @@ public sealed class ExceptionlessTransactionTests
Directory.Delete(dataPath, true);
using var zoneTree = new ZoneTreeFactory<int, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
.ConfigureWriteAheadLogOptions(x =>
@@ -103,7 +105,7 @@ public sealed class ExceptionlessTransactionTests
if (transaction.TotalPendingTransactionsRetried > 0)
Console.WriteLine("pending:" + transaction.TotalPendingTransactionsRetried);
});
zoneTree.Maintenance.DestroyTree();
}
}
@@ -12,6 +12,7 @@ public sealed class FixedSizeKeyAndValueTests
if (Directory.Exists(dataPath))
Directory.Delete(dataPath, true);
using var data = new ZoneTreeFactory<int, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetMutableSegmentMaxItemCount(5)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
@@ -64,6 +65,45 @@ public sealed class FixedSizeKeyAndValueTests
data.Maintenance.DestroyTree();
}
[Test]
public void IntStringDeleteTest()
{
var dataPath = "data/IntStringDeleteTest";
if (Directory.Exists(dataPath))
Directory.Delete(dataPath, true);
using var data = new ZoneTreeFactory<int, string>()
.SetDataDirectory(dataPath)
.OpenOrCreate();
data.TryAtomicAdd(1, "1");
data.TryAtomicAdd(2, "2");
data.TryAtomicAdd(3, "3");
data.TryDelete(2);
Assert.That(data.ContainsKey(1), Is.True);
Assert.That(data.ContainsKey(2), Is.False);
Assert.That(data.ContainsKey(3), Is.True);
}
[Test]
public void IntNullableIntDeleteTest()
{
var dataPath = "data/IntStringDeleteTest";
if (Directory.Exists(dataPath))
Directory.Delete(dataPath, true);
using var data = new ZoneTreeFactory<int, int?>()
.SetDataDirectory(dataPath)
.SetValueSerializer(new NullableInt32Serializer())
.OpenOrCreate();
data.TryAtomicAdd(1, 1);
data.TryAtomicAdd(2, 2);
data.TryAtomicAdd(3, 3);
data.TryDelete(2);
Assert.That(data.ContainsKey(1), Is.True);
Assert.That(data.ContainsKey(2), Is.False);
Assert.That(data.ContainsKey(3), Is.True);
}
[TestCase(true)]
[TestCase(false)]
public void StringIntTreeTest(bool useSparseArray)
@@ -73,6 +113,7 @@ public sealed class FixedSizeKeyAndValueTests
Directory.Delete(dataPath, true);
using var data = new ZoneTreeFactory<string, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetMutableSegmentMaxItemCount(5)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
@@ -139,7 +180,7 @@ public sealed class FixedSizeKeyAndValueTests
.SetMutableSegmentMaxItemCount(5)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
.ConfigureWriteAheadLogOptions(x =>
.ConfigureWriteAheadLogOptions(x =>
x.WriteAheadLogMode = WriteAheadLogMode.Sync)
.SetIsValueDeletedDelegate((in int x) => x == -1)
.SetMarkValueDeletedDelegate((ref int x) => x = -1)
+8
View File
@@ -245,6 +245,7 @@ public sealed class IteratorTests
var iteratorCount = 1000;
using var zoneTree = new ZoneTreeFactory<int, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetMutableSegmentMaxItemCount(insertCount * 2)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
@@ -293,6 +294,7 @@ public sealed class IteratorTests
var iteratorCount = 1000;
using var zoneTree = new ZoneTreeFactory<int, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetMutableSegmentMaxItemCount(insertCount * 2)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
@@ -355,6 +357,7 @@ public sealed class IteratorTests
var iteratorCount = 1000;
using var zoneTree = new ZoneTreeFactory<int, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetMutableSegmentMaxItemCount(insertCount * 2)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
@@ -397,6 +400,7 @@ public sealed class IteratorTests
Directory.Delete(dataPath, true);
using var zoneTree = new ZoneTreeFactory<string, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
.OpenOrCreate();
@@ -438,6 +442,7 @@ public sealed class IteratorTests
Directory.Delete(dataPath, true);
using var zoneTree = new ZoneTreeFactory<string, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
.OpenOrCreate();
@@ -486,6 +491,7 @@ public sealed class IteratorTests
Directory.Delete(dataPath, true);
using var zoneTree = new ZoneTreeFactory<string, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
.SetComparer(new StringCurrentCultureComparerAscending())
@@ -560,6 +566,7 @@ public sealed class IteratorTests
Directory.Delete(dataPath, true);
var zoneTree = new ZoneTreeFactory<string, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
.SetComparer(new StringCurrentCultureComparerAscending())
@@ -582,6 +589,7 @@ public sealed class IteratorTests
zoneTree.Dispose();
zoneTree = new ZoneTreeFactory<string, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
.SetComparer(new StringCurrentCultureComparerAscending())
@@ -17,6 +17,7 @@ public sealed class OptimisticTransactionTests
Directory.Delete(dataPath, true);
using var zoneTree = new ZoneTreeFactory<int, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
.OpenOrCreateTransactional();
@@ -64,6 +65,7 @@ public sealed class OptimisticTransactionTests
int n = 10000;
using var zoneTree = new ZoneTreeFactory<int, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
.ConfigureWriteAheadLogOptions(x => x.WriteAheadLogMode = walMode)
@@ -91,6 +93,7 @@ public sealed class OptimisticTransactionTests
int n = 10000;
using var zoneTree = new ZoneTreeFactory<int, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
.ConfigureWriteAheadLogOptions(x => x.WriteAheadLogMode = walMode)
@@ -165,7 +168,7 @@ public sealed class OptimisticTransactionTests
var dataPath = "data/TransactionLogCompactionTest" + compactionThreshold;
if (Directory.Exists(dataPath))
Directory.Delete(dataPath, true);
using var zoneTree = new ZoneTreeFactory<int, int>()
.SetDataDirectory(dataPath)
.SetWriteAheadLogDirectory(dataPath)
@@ -269,7 +272,7 @@ public sealed class OptimisticTransactionTests
Assert.IsTrue(zoneTree.TryGetNoThrow(tx1, 5, out var v5).IsAborted);
Assert.IsTrue(zoneTree.TryGetNoThrow(tx2, 5, out v5).Succeeded);
Assert.That(v5, Is.EqualTo(12));
// tx2 depends on tx1 bcs of read of key 9. tx1 aborted and so tx2.
Assert.That(zoneTree.PrepareAndCommitNoThrow(tx2).IsAborted, Is.True);
+2 -1
View File
@@ -27,7 +27,7 @@ public sealed class StringTreeTests
{
"Zbc", "DDD", "Abc", "Cbc", "Dbc", null
};
for(var i = 0; i < keys.Length; i++)
for (var i = 0; i < keys.Length; i++)
{
zoneTree.Upsert(keys[i], values[i]);
}
@@ -65,6 +65,7 @@ public sealed class StringTreeTests
for (var i = 0; i < 2; ++i)
{
using var db = new ZoneTreeFactory<string, int>()
.DisableDeleteValueConfigurationValidation(false)
.SetDataDirectory(dataPath)
.OpenOrCreate();
db.Upsert("0", 123);
+2 -2
View File
@@ -5,8 +5,8 @@
<Authors>Ahmed Yasin Koculu</Authors>
<PackageId>ZoneTree</PackageId>
<Title>ZoneTree</Title>
<ProductVersion>1.6.3.0</ProductVersion>
<Version>1.6.3.0</Version>
<ProductVersion>1.6.4.0</ProductVersion>
<Version>1.6.4.0</Version>
<Authors>Ahmed Yasin Koculu</Authors>
<AssemblyTitle>ZoneTree</AssemblyTitle>
<Description>ZoneTree is a persistent, high-performance, transactional, ACID-compliant ordered key-value database for NET. It can operate in memory or on local/cloud storage.</Description>
@@ -0,0 +1,23 @@
namespace Tenray.ZoneTree.Options;
/// <summary>
/// Defines the validation behavior
/// of not providing the delete value delegates.
/// </summary>
public enum DeleteValueConfigurationValidation
{
/// <summary>
/// Throws an error if the value deletion delegates are not configured.
/// </summary>
Required,
/// <summary>
/// Logs a warning if the value deletion delegates are not configured.
/// </summary>
Warning,
/// <summary>
/// Allows creating ZoneTree without delete record support.
/// </summary>
NotRequired,
}
+77 -5
View File
@@ -34,6 +34,10 @@ public sealed class ZoneTreeOptions<TKey, TValue>
static void DefaultMarkValueDeleted(ref TValue value) { value = default; }
static bool ReferenceTypeIsValueDeleted(in TValue value) => object.ReferenceEquals(value, default(TValue));
static void ReferenceTypeMarkValueDeleted(ref TValue value) { value = default; }
/// <summary>
/// Mutable segment maximumum key-value pair count.
/// When the maximum count is reached
@@ -109,33 +113,66 @@ public sealed class ZoneTreeOptions<TKey, TValue>
{
if (KeySerializer == null)
{
exception = new MissingOptionException("KeySerializer");
exception = new MissingOptionException(nameof(KeySerializer));
return false;
}
if (ValueSerializer == null)
{
exception = new MissingOptionException("ValueSerializer");
exception = new MissingOptionException(nameof(ValueSerializer));
return false;
}
if (Comparer == null)
{
exception = new MissingOptionException("Comparer");
exception = new MissingOptionException(nameof(Comparer));
return false;
}
if (RandomAccessDeviceManager == null)
{
exception = new MissingOptionException("RandomAccessDeviceManager");
exception = new MissingOptionException(nameof(RandomAccessDeviceManager));
return false;
}
if (WriteAheadLogProvider == null)
{
exception = new MissingOptionException("WriteAheadLogProvider");
exception = new MissingOptionException(nameof(WriteAheadLogProvider));
return false;
}
switch (DeleteValueConfigurationValidation)
{
case DeleteValueConfigurationValidation.Required:
{
if (IsValueDeleted == DefaultIsValueDeleted)
{
exception = new MissingOptionException(nameof(IsValueDeleted));
return false;
}
if (MarkValueDeleted == DefaultMarkValueDeleted)
{
exception = new MissingOptionException(nameof(MarkValueDeleted));
return false;
}
}
break;
case DeleteValueConfigurationValidation.Warning:
{
if (IsValueDeleted == DefaultIsValueDeleted)
{
Logger.LogWarning(new MissingOptionException(nameof(IsValueDeleted)));
}
if (MarkValueDeleted == DefaultMarkValueDeleted)
{
Logger.LogWarning(new MissingOptionException(nameof(MarkValueDeleted)));
}
}
break;
default: break;
}
exception = ValidateCompressionLevel(
"disk segment",
DiskSegmentOptions.CompressionMethod,
@@ -202,4 +239,39 @@ public sealed class ZoneTreeOptions<TKey, TValue>
/// Configures the random access device manager.
/// </summary>
public IRandomAccessDeviceManager RandomAccessDeviceManager { get; set; }
/// <summary>
/// Defines the validation behavior
/// of not providing the delete value delegates.
/// </summary>
public DeleteValueConfigurationValidation DeleteValueConfigurationValidation { get; set; }
/// <summary>
/// Creates default delete delegates for nullable types.
/// </summary>
public void CreateDefaultDeleteDelegates()
{
if (!IsAssignableToNull(typeof(TValue)))
return;
if (MarkValueDeleted == DefaultMarkValueDeleted)
{
MarkValueDeleted = ReferenceTypeMarkValueDeleted;
}
if (IsValueDeleted == DefaultIsValueDeleted)
{
IsValueDeleted = ReferenceTypeIsValueDeleted;
}
}
static bool IsAssignableToNull(Type type)
{
if (!type.IsValueType || Nullable.GetUnderlyingType(type) != null)
{
return true;
}
return false;
}
}
@@ -0,0 +1,18 @@
namespace Tenray.ZoneTree.Serializers;
public sealed class NullableInt32Serializer : ISerializer<int?>
{
public int? Deserialize(byte[] bytes)
{
if (bytes.Length == 0)
return null;
return BitConverter.ToInt32(bytes);
}
public byte[] Serialize(in int? entry)
{
if (entry.HasValue)
return BitConverter.GetBytes(entry.Value);
return Array.Empty<byte>();
}
}
+18 -4
View File
@@ -48,7 +48,7 @@ public sealed class ZoneTreeFactory<TKey, TValue>
FileStreamProvider = fileStreamProvider;
GetWriteAheadLogProvider = (options) =>
WalDirectory == null ?
WalDirectory == null ?
new WriteAheadLogProvider(options.Logger, fileStreamProvider) :
new WriteAheadLogProvider(options.Logger, fileStreamProvider, WalDirectory);
}
@@ -165,6 +165,19 @@ public sealed class ZoneTreeFactory<TKey, TValue>
return this;
}
/// <summary>
/// Disables the delete value configuration validation.
/// </summary>
/// <param name="keepWarning">If true, validation logs a warning when value deletion is not configured.</param>
/// <returns>ZoneTree Factory</returns>
public ZoneTreeFactory<TKey, TValue> DisableDeleteValueConfigurationValidation(bool keepWarning = true)
{
Options.DeleteValueConfigurationValidation = keepWarning ?
DeleteValueConfigurationValidation.Warning :
DeleteValueConfigurationValidation.NotRequired;
return this;
}
/// <summary>
/// Sets the maximum cached block count.
/// </summary>
@@ -372,6 +385,7 @@ public sealed class ZoneTreeFactory<TKey, TValue>
Options.Logger,
FileStreamProvider);
}
Options.CreateDefaultDeleteDelegates();
FillComparer();
FillKeySerializer();
FillValueSerializer();
@@ -424,7 +438,7 @@ public sealed class ZoneTreeFactory<TKey, TValue>
};
if (typeof(TKey) == typeof(string))
Options.KeySerializer =
Options.KeySerializer =
new Utf8StringSerializer() as ISerializer<TKey>;
else if (typeof(TKey) == typeof(byte[]))
@@ -521,7 +535,7 @@ public sealed class ZoneTreeFactory<TKey, TValue>
/// <returns>ZoneTree Factory</returns>
public ITransactionalZoneTree<TKey, TValue> OpenOrCreateTransactional()
{
var zoneTree = OpenOrCreate();
var zoneTree = OpenOrCreate();
InitTransactionLog();
return new OptimisticZoneTree<TKey, TValue>(Options, TransactionLog, zoneTree);
}
@@ -533,7 +547,7 @@ public sealed class ZoneTreeFactory<TKey, TValue>
/// <exception cref="DatabaseAlreadyExistsException">Thrown when the database exists in the location.</exception>
public ITransactionalZoneTree<TKey, TValue> CreateTransactional()
{
var zoneTree = Create();
var zoneTree = Create();
InitTransactionLog();
return new OptimisticZoneTree<TKey, TValue>(Options, TransactionLog, zoneTree);
}