diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/0.json b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/0.json new file mode 100644 index 0000000..2d41bc8 --- /dev/null +++ b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/0.json @@ -0,0 +1,47 @@ +{ + "Version": "1.9.9.0", + "ComparerType": "ZoneTree.Comparers.Int32ComparerAscending", + "KeyHasherType": "ZoneTree.Hashers.DefaultKeyHasher\u003CSystem.Int32\u003E", + "KeyType": "System.Int32", + "ValueType": "System.Int32", + "KeySerializerType": "ZoneTree.Serializers.Int32Serializer", + "ValueSerializerType": "ZoneTree.Serializers.Int32Serializer", + "MutableSegmentMaxItemCount": 1000000, + "MutableSegmentBloomFilterBitsPerItem": 8, + "DiskSegmentMaxItemCount": 20000000, + "WriteAheadLogOptions": { + "WriteAheadLogMode": 2, + "CustomOptions": null, + "CompressionBlockSize": 128, + "CompressionMethod": 4, + "CompressionLevel": 0, + "SyncCompressedModeOptions": { + "EnableTailWriterJob": false, + "TailWriterJobInterval": 500 + }, + "AsyncCompressedModeOptions": { + "EmptyQueuePollInterval": 100 + }, + "EnableIncrementalBackup": false + }, + "DiskSegmentOptions": { + "DiskSegmentMode": 1, + "CompressionBlockSize": 4194304, + "CompressionMethod": 1, + "CompressionLevel": 0, + "MaximumRecordCount": 3000000, + "MinimumRecordCount": 1500000, + "KeyCacheSize": 1024, + "ValueCacheSize": 1024, + "MaterializedEntryCacheSize": 4096, + "SearchHintPrefetchSize": 16, + "KeyCacheRecordLifeTimeInMillisecond": 10000, + "ValueCacheRecordLifeTimeInMillisecond": 10000, + "DefaultSparseArrayStepSize": 1024 + }, + "MutableSegment": 1, + "ReadOnlySegments": [], + "DiskSegment": 0, + "BottomSegments": [], + "MaximumOpIndex": 0 +} \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/0.meta.wal b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/0.meta.wal new file mode 100644 index 0000000..422ae4a Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/0.meta.wal differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/1.wal.2 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/1.wal.2 new file mode 100644 index 0000000..45a8ca0 --- /dev/null +++ b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/1.wal.2 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/1.wal.2.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/1.wal.2.tail new file mode 100644 index 0000000..c3052ac Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/1.wal.2.tail differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/2.wal.2 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/2.wal.2 new file mode 100644 index 0000000..45a8ca0 --- /dev/null +++ b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/2.wal.2 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/2.wal.2.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/2.wal.2.tail new file mode 100644 index 0000000..113a5ce Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/2.wal.2.tail differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txd/0.wal.2 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txd/0.wal.2 new file mode 100644 index 0000000..45a8ca0 --- /dev/null +++ b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txd/0.wal.2 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txd/0.wal.2.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txd/0.wal.2.tail new file mode 100644 index 0000000..1b1cb4d Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txd/0.wal.2.tail differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txh/0.wal.2 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txh/0.wal.2 new file mode 100644 index 0000000..4424ff2 Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txh/0.wal.2 differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txh/0.wal.2.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txh/0.wal.2.tail new file mode 100644 index 0000000..0730bf3 Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txh/0.wal.2.tail differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txm/0.wal.2 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txm/0.wal.2 new file mode 100644 index 0000000..c20e10c Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txm/0.wal.2 differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txm/0.wal.2.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txm/0.wal.2.tail new file mode 100644 index 0000000..7a94dc2 Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txm/0.wal.2.tail differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txs/0.wal.2 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txs/0.wal.2 new file mode 100644 index 0000000..9e9d7f9 Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txs/0.wal.2 differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txs/0.wal.2.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txs/0.wal.2.tail new file mode 100644 index 0000000..1f7e14c Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txs/0.wal.2.tail differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/0.json b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/0.json new file mode 100644 index 0000000..fd12d07 --- /dev/null +++ b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/0.json @@ -0,0 +1,47 @@ +{ + "Version": "1.9.9.0", + "ComparerType": "ZoneTree.Comparers.Int32ComparerAscending", + "KeyHasherType": "ZoneTree.Hashers.DefaultKeyHasher\u003CSystem.Int32\u003E", + "KeyType": "System.Int32", + "ValueType": "System.Int32", + "KeySerializerType": "ZoneTree.Serializers.Int32Serializer", + "ValueSerializerType": "ZoneTree.Serializers.Int32Serializer", + "MutableSegmentMaxItemCount": 1000000, + "MutableSegmentBloomFilterBitsPerItem": 8, + "DiskSegmentMaxItemCount": 20000000, + "WriteAheadLogOptions": { + "WriteAheadLogMode": 0, + "CustomOptions": null, + "CompressionBlockSize": 128, + "CompressionMethod": 4, + "CompressionLevel": 0, + "SyncCompressedModeOptions": { + "EnableTailWriterJob": false, + "TailWriterJobInterval": 500 + }, + "AsyncCompressedModeOptions": { + "EmptyQueuePollInterval": 100 + }, + "EnableIncrementalBackup": false + }, + "DiskSegmentOptions": { + "DiskSegmentMode": 1, + "CompressionBlockSize": 4194304, + "CompressionMethod": 1, + "CompressionLevel": 0, + "MaximumRecordCount": 3000000, + "MinimumRecordCount": 1500000, + "KeyCacheSize": 1024, + "ValueCacheSize": 1024, + "MaterializedEntryCacheSize": 4096, + "SearchHintPrefetchSize": 16, + "KeyCacheRecordLifeTimeInMillisecond": 10000, + "ValueCacheRecordLifeTimeInMillisecond": 10000, + "DefaultSparseArrayStepSize": 1024 + }, + "MutableSegment": 1, + "ReadOnlySegments": [], + "DiskSegment": 0, + "BottomSegments": [], + "MaximumOpIndex": 0 +} \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/0.meta.wal b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/0.meta.wal new file mode 100644 index 0000000..422ae4a Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/0.meta.wal differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/seg/1.wal.0 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/seg/1.wal.0 new file mode 100644 index 0000000..5aacda2 Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/seg/1.wal.0 differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/seg/2.wal.0 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/seg/2.wal.0 new file mode 100644 index 0000000..67496a5 Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/seg/2.wal.0 differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txd/0.wal.0 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txd/0.wal.0 new file mode 100644 index 0000000..e69de29 diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txh/0.wal.0 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txh/0.wal.0 new file mode 100644 index 0000000..572192a Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txh/0.wal.0 differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txm/0.wal.0 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txm/0.wal.0 new file mode 100644 index 0000000..09f69d1 Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txm/0.wal.0 differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txs/0.wal.0 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txs/0.wal.0 new file mode 100644 index 0000000..5dc0847 Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txs/0.wal.0 differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/0.json b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/0.json new file mode 100644 index 0000000..19428d6 --- /dev/null +++ b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/0.json @@ -0,0 +1,47 @@ +{ + "Version": "1.9.9.0", + "ComparerType": "ZoneTree.Comparers.Int32ComparerAscending", + "KeyHasherType": "ZoneTree.Hashers.DefaultKeyHasher\u003CSystem.Int32\u003E", + "KeyType": "System.Int32", + "ValueType": "System.Int32", + "KeySerializerType": "ZoneTree.Serializers.Int32Serializer", + "ValueSerializerType": "ZoneTree.Serializers.Int32Serializer", + "MutableSegmentMaxItemCount": 1000000, + "MutableSegmentBloomFilterBitsPerItem": 8, + "DiskSegmentMaxItemCount": 20000000, + "WriteAheadLogOptions": { + "WriteAheadLogMode": 1, + "CustomOptions": null, + "CompressionBlockSize": 128, + "CompressionMethod": 4, + "CompressionLevel": 0, + "SyncCompressedModeOptions": { + "EnableTailWriterJob": false, + "TailWriterJobInterval": 500 + }, + "AsyncCompressedModeOptions": { + "EmptyQueuePollInterval": 100 + }, + "EnableIncrementalBackup": false + }, + "DiskSegmentOptions": { + "DiskSegmentMode": 1, + "CompressionBlockSize": 4194304, + "CompressionMethod": 1, + "CompressionLevel": 0, + "MaximumRecordCount": 3000000, + "MinimumRecordCount": 1500000, + "KeyCacheSize": 1024, + "ValueCacheSize": 1024, + "MaterializedEntryCacheSize": 4096, + "SearchHintPrefetchSize": 16, + "KeyCacheRecordLifeTimeInMillisecond": 10000, + "ValueCacheRecordLifeTimeInMillisecond": 10000, + "DefaultSparseArrayStepSize": 1024 + }, + "MutableSegment": 1, + "ReadOnlySegments": [], + "DiskSegment": 0, + "BottomSegments": [], + "MaximumOpIndex": 0 +} \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/0.meta.wal b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/0.meta.wal new file mode 100644 index 0000000..422ae4a Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/0.meta.wal differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/1.wal.1 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/1.wal.1 new file mode 100644 index 0000000..45a8ca0 --- /dev/null +++ b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/1.wal.1 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/1.wal.1.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/1.wal.1.tail new file mode 100644 index 0000000..c3052ac Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/1.wal.1.tail differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/2.wal.1 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/2.wal.1 new file mode 100644 index 0000000..45a8ca0 --- /dev/null +++ b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/2.wal.1 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/2.wal.1.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/2.wal.1.tail new file mode 100644 index 0000000..113a5ce Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/2.wal.1.tail differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txd/0.wal.1 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txd/0.wal.1 new file mode 100644 index 0000000..45a8ca0 --- /dev/null +++ b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txd/0.wal.1 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txd/0.wal.1.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txd/0.wal.1.tail new file mode 100644 index 0000000..1b1cb4d Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txd/0.wal.1.tail differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txh/0.wal.1 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txh/0.wal.1 new file mode 100644 index 0000000..4424ff2 Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txh/0.wal.1 differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txh/0.wal.1.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txh/0.wal.1.tail new file mode 100644 index 0000000..0730bf3 Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txh/0.wal.1.tail differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txm/0.wal.1 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txm/0.wal.1 new file mode 100644 index 0000000..0b5400c Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txm/0.wal.1 differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txm/0.wal.1.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txm/0.wal.1.tail new file mode 100644 index 0000000..fcc0436 Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txm/0.wal.1.tail differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txs/0.wal.1 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txs/0.wal.1 new file mode 100644 index 0000000..9e9d7f9 Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txs/0.wal.1 differ diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txs/0.wal.1.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txs/0.wal.1.tail new file mode 100644 index 0000000..1f7e14c Binary files /dev/null and b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txs/0.wal.1.tail differ diff --git a/src/ZoneTree.UnitTests/LegacyWalMigrationTests.cs b/src/ZoneTree.UnitTests/LegacyWalMigrationTests.cs new file mode 100644 index 0000000..4af650b --- /dev/null +++ b/src/ZoneTree.UnitTests/LegacyWalMigrationTests.cs @@ -0,0 +1,275 @@ +using ZoneTree.AbstractFileStream; +using ZoneTree.Logger; +using ZoneTree.Options; +using ZoneTree.Serializers; +using ZoneTree.WAL; + +namespace ZoneTree.UnitTests; + +public sealed class LegacyWalMigrationTests +{ + // Real filesystem fixtures generated with main commit 8347664f5e5d1c98a530e2c15eb63ab42118e9e4 + // (ZoneTree 1.9.9): transaction 1 commits {1:10, 2:20}, then rotates the mutable segment; + // transaction 2 commits {3:30}; transaction 3 leaves {2:999, 4:40} uncommitted. + // Compression uses 128-byte blocks and CompressionMethod.None to keep fixtures tiny. + string DataDirectory; + + [SetUp] + public void SetUp() + { + DataDirectory = Path.Combine(TestContext.CurrentContext.WorkDirectory, + nameof(LegacyWalMigrationTests), Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(DataDirectory); + } + + [TearDown] + public void TearDown() + { + Directory.Delete(DataDirectory, true); + } + + [TestCase(WriteAheadLogMode.Sync, false)] + [TestCase(WriteAheadLogMode.Sync, true)] + [TestCase(WriteAheadLogMode.SyncCompressed, false)] + [TestCase(WriteAheadLogMode.SyncCompressed, true)] + [TestCase(WriteAheadLogMode.AsyncCompressed, false)] + [TestCase(WriteAheadLogMode.AsyncCompressed, true)] + public void LegacyDatabaseMigratesOnceAndRecoversTransactions( + WriteAheadLogMode mode, bool configureTransactionsBeforeOpen) + { + CopyFixture(mode); + var originalFiles = LegacyWalFiles(); + var oldWal = WalPath("seg", 2, "wal", mode); + var oldEntries = ReadEntries(oldWal, mode); + Assert.That(oldEntries.Any(x => x.Checksum != 0), Is.True); + var provider = new CountingWalProvider(DataDirectory); + var factory = CreateFactory(mode).SetWriteAheadLogProvider(_ => provider); + if (configureTransactionsBeforeOpen) + { + factory.ConfigureTransactionLog(log => log.CompactionThreshold = int.MaxValue); + factory.ConfigureTransactionLog(log => log.CompactionThreshold = int.MaxValue); + } + + using (var tree = factory.OpenTransactional()) + { + Assert.That(provider.MigrationVersions, Is.EqualTo(new[] { new Version("1.9.9.0") })); + Assert.That(tree.Maintenance.ZoneTree.Maintenance.MutableSegment.SegmentId, Is.EqualTo(2)); + Assert.That(tree.Maintenance.ZoneTree.Maintenance.ReadOnlySegments.Count, Is.EqualTo(1)); + Assert.That(tree.Maintenance.UncommittedTransactionIds, Is.EqualTo(new[] { 3L })); + AssertCommitted(tree, 1, 10); + AssertCommitted(tree, 2, 20); + AssertCommitted(tree, 3, 30); + Assert.That(tree.ReadCommittedContainsKey(4), Is.False); + foreach (var file in originalFiles) + { + Assert.That(File.Exists(file), Is.False); + Assert.That(File.Exists(file.Replace(".wal.", ".wal_crc.", StringComparison.Ordinal)), Is.True); + } + Assert.That(Directory.GetFiles(DataDirectory, "*.wal_xxh3.*", SearchOption.AllDirectories), Is.Empty); + + tree.PrepareAndCommit(3); + tree.UpsertAutoCommit(5, 50); + } + + var legacyWal = WalPath("seg", 2, "wal_crc", mode); + var appendedEntries = ReadEntries(legacyWal, mode); + Assert.That(appendedEntries.Count, Is.GreaterThan(oldEntries.Count)); + Assert.That(appendedEntries.Take(oldEntries.Count).Select(x => x.Checksum), + Is.EqualTo(oldEntries.Select(x => x.Checksum))); + Assert.That(appendedEntries.Skip(oldEntries.Count).Select(x => x.Checksum), Is.All.Zero); + + using (var tree = CreateFactory(mode).OpenTransactional()) + { + Assert.That(tree.Maintenance.UncommittedTransactionIds, Is.Empty); + AssertCommitted(tree, 2, 999); + AssertCommitted(tree, 4, 40); + AssertCommitted(tree, 5, 50); + tree.Maintenance.ZoneTree.Maintenance.MoveMutableSegmentForward(); + tree.UpsertAutoCommit(6, 60); + } + + var newWal = WalPath("seg", 3, "wal_xxh3", mode); + var newEntries = ReadEntries(newWal, mode); + Assert.That(newEntries, Is.Not.Empty); + Assert.That(newEntries.All(x => x.ValidateChecksum()), Is.True); + using var reopened = CreateFactory(mode).OpenTransactional(); + AssertCommitted(reopened, 1, 10); + AssertCommitted(reopened, 6, 60); + } + + [Test] + public void ProviderSuppliedThroughOptionsMigratesBeforeTransactionConfiguration() + { + CopyFixture(WriteAheadLogMode.Sync); + var provider = new CountingWalProvider(DataDirectory); + var factory = CreateFactory(WriteAheadLogMode.Sync) + .Configure(options => options.WriteAheadLogProvider = provider) + .ConfigureTransactionLog(log => log.CompactionThreshold = int.MaxValue); + using var tree = factory.OpenTransactional(); + Assert.That(provider.MigrationVersions, Is.EqualTo(new[] { new Version("1.9.9.0") })); + Assert.That(tree.Maintenance.UncommittedTransactionIds, Is.EqualTo(new[] { 3L })); + AssertCommitted(tree, 2, 20); + } + + [TestCase(WriteAheadLogMode.Sync)] + [TestCase(WriteAheadLogMode.SyncCompressed)] + [TestCase(WriteAheadLogMode.AsyncCompressed)] + public void PartiallyRenamedDatabaseCompletesMigration(WriteAheadLogMode mode) + { + CopyFixture(mode); + var original = WalPath("seg", 2, "wal", mode); + File.Move(original, WalPath("seg", 2, "wal_crc", mode)); + // Compressed tails deliberately retain their old names, as after an interrupted rename. + using var tree = CreateFactory(mode).OpenTransactional(); + AssertCommitted(tree, 2, 20); + Assert.That(tree.Maintenance.UncommittedTransactionIds, Is.EqualTo(new[] { 3L })); + Assert.That(LegacyWalFiles(), Is.Empty); + } + + [TestCase(WriteAheadLogMode.Sync)] + [TestCase(WriteAheadLogMode.SyncCompressed)] + [TestCase(WriteAheadLogMode.AsyncCompressed)] + public void LegacyChecksumIsIgnoredButNewChecksumIsValidated(WriteAheadLogMode mode) + { + CopyFixture(mode); + CorruptChecksum(WalPath("seg", 2, "wal", mode), mode); + using (var tree = CreateFactory(mode).Open()) + { + Assert.That(tree.TryGet(2, out var value), Is.True); + Assert.That(value, Is.EqualTo(999)); + tree.Maintenance.MoveMutableSegmentForward(); + tree.Upsert(6, 60); + } + + var newWal = WalPath("seg", 3, "wal_xxh3", mode); + CorruptChecksum(newWal, mode); + var provider = new WriteAheadLogProvider(new ConsoleLogger(), new LocalFileStreamProvider(), DataDirectory); + using var wal = provider.GetOrCreateWAL(3, "seg", WalOptions(mode), new Int32Serializer(), new Int32Serializer()); + var result = wal.ReadLogEntries(true, true, false); + Assert.That(result.Success, Is.False); + Assert.That(result.Exceptions.Values.Any(x => x.Message.Contains("Checksum failed", StringComparison.Ordinal)), Is.True); + } + + [TestCase(WriteAheadLogMode.Sync)] + [TestCase(WriteAheadLogMode.SyncCompressed)] + [TestCase(WriteAheadLogMode.AsyncCompressed)] + public void ReplacingLegacyWalKeepsZeroChecksumsAndCanBeReopened(WriteAheadLogMode mode) + { + CopyFixture(mode); + using (CreateFactory(mode).Open()) { } + var provider = new WriteAheadLogProvider(new ConsoleLogger(), new LocalFileStreamProvider(), DataDirectory); + using (var wal = provider.GetOrCreateWAL(2, "seg", WalOptions(mode), new Int32Serializer(), new Int32Serializer())) + { + wal.ReplaceWriteAheadLog(new[] { 2, 3, 4 }, new[] { 222, 30, 40 }, true); + } + var entries = ReadEntries(WalPath("seg", 2, "wal_crc", mode), mode); + Assert.That(entries.Count, Is.EqualTo(3)); + Assert.That(entries.Select(x => x.Checksum), Is.All.Zero); + using var tree = CreateFactory(mode).Open(); + Assert.That(tree.TryGet(2, out var value), Is.True); + Assert.That(value, Is.EqualTo(222)); + } + + ZoneTreeFactory CreateFactory(WriteAheadLogMode mode) + { + return new ZoneTreeFactory() + .SetDataDirectory(DataDirectory) + .Configure(options => options.AllowUnsafeOptionValues = true) + .ConfigureWriteAheadLogOptions(options => + { + options.WriteAheadLogMode = mode; + options.CompressionBlockSize = 128; + options.CompressionMethod = CompressionMethod.None; + options.SyncCompressedModeOptions.EnableTailWriterJob = false; + }); + } + + static WriteAheadLogOptions WalOptions(WriteAheadLogMode mode) + { + var options = new WriteAheadLogOptions + { + WriteAheadLogMode = mode, + CompressionBlockSize = 128, + CompressionMethod = CompressionMethod.None + }; + options.SyncCompressedModeOptions.EnableTailWriterJob = false; + return options; + } + + void CopyFixture(WriteAheadLogMode mode) + { + var source = Path.Combine(TestContext.CurrentContext.TestDirectory, "Fixtures", "LegacyCrcWals", mode.ToString()); + Assert.That(Directory.Exists(source), Is.True, "Legacy fixtures must be copied to the test output."); + foreach (var file in Directory.GetFiles(source, "*", SearchOption.AllDirectories)) + { + var destination = Path.Combine(DataDirectory, Path.GetRelativePath(source, file)); + Directory.CreateDirectory(Path.GetDirectoryName(destination)); + File.Copy(file, destination); + } + } + + string WalPath(string category, int segmentId, string format, WriteAheadLogMode mode) + => Path.Combine(DataDirectory, category, $"{segmentId}.{format}.{(int)mode}"); + + string[] LegacyWalFiles() + => Directory.GetFiles(DataDirectory, "*", SearchOption.AllDirectories) + .Where(x => Path.GetFileName(x).Contains(".wal.", StringComparison.Ordinal)).ToArray(); + + static void AssertCommitted(ITransactionalZoneTree tree, int key, int expected) + { + Assert.That(tree.ReadCommittedTryGet(key, out var value), Is.True); + Assert.That(value, Is.EqualTo(expected)); + } + + static Stream OpenRecordStream(string path, WriteAheadLogMode mode) + => mode == WriteAheadLogMode.Sync + ? new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None) + : new CompressedFileStream(new ConsoleLogger(), new LocalFileStreamProvider(), path, + 128, false, 0, CompressionMethod.None, 0); + + static List ReadEntries(string path, WriteAheadLogMode mode) + { + using var stream = OpenRecordStream(path, mode); + stream.Seek(0, SeekOrigin.Begin); + using var reader = new BinaryReader(stream); + var entries = new List(); + while (stream.Position < stream.Length) + { + LogEntry entry = default; + LogEntry.ReadLogEntry(reader, ref entry); + entries.Add(entry); + } + return entries; + } + + static void CorruptChecksum(string path, WriteAheadLogMode mode) + { + using var stream = OpenRecordStream(path, mode); + stream.Seek(0, SeekOrigin.Begin); + using var reader = new BinaryReader(stream, System.Text.Encoding.UTF8, true); + var bytes = reader.ReadBytes((int)stream.Length); + bytes[^1] ^= 0x80; + stream.SetLength(0); + stream.Seek(0, SeekOrigin.Begin); + stream.Write(bytes, 0, bytes.Length); + } + + sealed class CountingWalProvider(string directory) : IWriteAheadLogProvider + { + readonly WriteAheadLogProvider Inner = new(new ConsoleLogger(), new LocalFileStreamProvider(), directory); + public List MigrationVersions { get; } = new(); + public void MigrateWals(Version databaseVersion) + { + MigrationVersions.Add(databaseVersion); + Inner.MigrateWals(databaseVersion); + } + public void InitCategory(string category) => Inner.InitCategory(category); + public IWriteAheadLog GetOrCreateWAL(long segmentId, string category, + WriteAheadLogOptions options, ISerializer keySerializer, ISerializer valueSerializer) + => Inner.GetOrCreateWAL(segmentId, category, options, keySerializer, valueSerializer); + public IWriteAheadLog GetWAL(long segmentId, string category) + => Inner.GetWAL(segmentId, category); + public bool RemoveWAL(long segmentId, string category) => Inner.RemoveWAL(segmentId, category); + public void DropStore() => Inner.DropStore(); + } +} diff --git a/src/ZoneTree.UnitTests/ZoneTree.UnitTests.csproj b/src/ZoneTree.UnitTests/ZoneTree.UnitTests.csproj index 0c1a191..c86ba78 100644 --- a/src/ZoneTree.UnitTests/ZoneTree.UnitTests.csproj +++ b/src/ZoneTree.UnitTests/ZoneTree.UnitTests.csproj @@ -30,4 +30,8 @@ + + + +