From 26ea76b4a7e3a2c3bf230e4e5cd4dd01d8d282cb Mon Sep 17 00:00:00 2001 From: Ahmed Yasin Koculu Date: Fri, 9 Oct 2026 22:38:43 +0200 Subject: [PATCH] Add migration tests for all three WAL modes. --- .../LegacyCrcWals/AsyncCompressed/0.json | 47 +++ .../LegacyCrcWals/AsyncCompressed/0.meta.wal | Bin 0 -> 72 bytes .../LegacyCrcWals/AsyncCompressed/seg/1.wal.2 | 1 + .../AsyncCompressed/seg/1.wal.2.tail | Bin 0 -> 64 bytes .../LegacyCrcWals/AsyncCompressed/seg/2.wal.2 | 1 + .../AsyncCompressed/seg/2.wal.2.tail | Bin 0 -> 92 bytes .../LegacyCrcWals/AsyncCompressed/txd/0.wal.2 | 1 + .../AsyncCompressed/txd/0.wal.2.tail | Bin 0 -> 8 bytes .../LegacyCrcWals/AsyncCompressed/txh/0.wal.2 | Bin 0 -> 281 bytes .../AsyncCompressed/txh/0.wal.2.tail | Bin 0 -> 12 bytes .../LegacyCrcWals/AsyncCompressed/txm/0.wal.2 | Bin 0 -> 281 bytes .../AsyncCompressed/txm/0.wal.2.tail | Bin 0 -> 12 bytes .../LegacyCrcWals/AsyncCompressed/txs/0.wal.2 | Bin 0 -> 141 bytes .../AsyncCompressed/txs/0.wal.2.tail | Bin 0 -> 80 bytes .../Fixtures/LegacyCrcWals/Sync/0.json | 47 +++ .../Fixtures/LegacyCrcWals/Sync/0.meta.wal | Bin 0 -> 72 bytes .../Fixtures/LegacyCrcWals/Sync/seg/1.wal.0 | Bin 0 -> 56 bytes .../Fixtures/LegacyCrcWals/Sync/seg/2.wal.0 | Bin 0 -> 84 bytes .../Fixtures/LegacyCrcWals/Sync/txd/0.wal.0 | 0 .../Fixtures/LegacyCrcWals/Sync/txh/0.wal.0 | Bin 0 -> 260 bytes .../Fixtures/LegacyCrcWals/Sync/txm/0.wal.0 | Bin 0 -> 260 bytes .../Fixtures/LegacyCrcWals/Sync/txs/0.wal.0 | Bin 0 -> 200 bytes .../LegacyCrcWals/SyncCompressed/0.json | 47 +++ .../LegacyCrcWals/SyncCompressed/0.meta.wal | Bin 0 -> 72 bytes .../LegacyCrcWals/SyncCompressed/seg/1.wal.1 | 1 + .../SyncCompressed/seg/1.wal.1.tail | Bin 0 -> 64 bytes .../LegacyCrcWals/SyncCompressed/seg/2.wal.1 | 1 + .../SyncCompressed/seg/2.wal.1.tail | Bin 0 -> 92 bytes .../LegacyCrcWals/SyncCompressed/txd/0.wal.1 | 1 + .../SyncCompressed/txd/0.wal.1.tail | Bin 0 -> 8 bytes .../LegacyCrcWals/SyncCompressed/txh/0.wal.1 | Bin 0 -> 281 bytes .../SyncCompressed/txh/0.wal.1.tail | Bin 0 -> 12 bytes .../LegacyCrcWals/SyncCompressed/txm/0.wal.1 | Bin 0 -> 281 bytes .../SyncCompressed/txm/0.wal.1.tail | Bin 0 -> 12 bytes .../LegacyCrcWals/SyncCompressed/txs/0.wal.1 | Bin 0 -> 141 bytes .../SyncCompressed/txs/0.wal.1.tail | Bin 0 -> 80 bytes .../LegacyWalMigrationTests.cs | 275 ++++++++++++++++++ .../ZoneTree.UnitTests.csproj | 4 + 38 files changed, 426 insertions(+) create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/0.json create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/0.meta.wal create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/1.wal.2 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/1.wal.2.tail create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/2.wal.2 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/2.wal.2.tail create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txd/0.wal.2 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txd/0.wal.2.tail create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txh/0.wal.2 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txh/0.wal.2.tail create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txm/0.wal.2 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txm/0.wal.2.tail create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txs/0.wal.2 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txs/0.wal.2.tail create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/0.json create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/0.meta.wal create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/seg/1.wal.0 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/seg/2.wal.0 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txd/0.wal.0 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txh/0.wal.0 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txm/0.wal.0 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txs/0.wal.0 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/0.json create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/0.meta.wal create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/1.wal.1 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/1.wal.1.tail create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/2.wal.1 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/2.wal.1.tail create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txd/0.wal.1 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txd/0.wal.1.tail create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txh/0.wal.1 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txh/0.wal.1.tail create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txm/0.wal.1 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txm/0.wal.1.tail create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txs/0.wal.1 create mode 100644 src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txs/0.wal.1.tail create mode 100644 src/ZoneTree.UnitTests/LegacyWalMigrationTests.cs diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/0.json b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/0.json new file mode 100644 index 0000000..2d41bc8 --- /dev/null +++ b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/0.json @@ -0,0 +1,47 @@ +{ + "Version": "1.9.9.0", + "ComparerType": "ZoneTree.Comparers.Int32ComparerAscending", + "KeyHasherType": "ZoneTree.Hashers.DefaultKeyHasher\u003CSystem.Int32\u003E", + "KeyType": "System.Int32", + "ValueType": "System.Int32", + "KeySerializerType": "ZoneTree.Serializers.Int32Serializer", + "ValueSerializerType": "ZoneTree.Serializers.Int32Serializer", + "MutableSegmentMaxItemCount": 1000000, + "MutableSegmentBloomFilterBitsPerItem": 8, + "DiskSegmentMaxItemCount": 20000000, + "WriteAheadLogOptions": { + "WriteAheadLogMode": 2, + "CustomOptions": null, + "CompressionBlockSize": 128, + "CompressionMethod": 4, + "CompressionLevel": 0, + "SyncCompressedModeOptions": { + "EnableTailWriterJob": false, + "TailWriterJobInterval": 500 + }, + "AsyncCompressedModeOptions": { + "EmptyQueuePollInterval": 100 + }, + "EnableIncrementalBackup": false + }, + "DiskSegmentOptions": { + "DiskSegmentMode": 1, + "CompressionBlockSize": 4194304, + "CompressionMethod": 1, + "CompressionLevel": 0, + "MaximumRecordCount": 3000000, + "MinimumRecordCount": 1500000, + "KeyCacheSize": 1024, + "ValueCacheSize": 1024, + "MaterializedEntryCacheSize": 4096, + "SearchHintPrefetchSize": 16, + "KeyCacheRecordLifeTimeInMillisecond": 10000, + "ValueCacheRecordLifeTimeInMillisecond": 10000, + "DefaultSparseArrayStepSize": 1024 + }, + "MutableSegment": 1, + "ReadOnlySegments": [], + "DiskSegment": 0, + "BottomSegments": [], + "MaximumOpIndex": 0 +} \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/0.meta.wal b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/0.meta.wal new file mode 100644 index 0000000000000000000000000000000000000000..422ae4a906116923c9f3827c3b3c9e0b01bbe102 GIT binary patch literal 72 Xcmd;JfB+^ajZQJ53Bc6BXqYqr1AqV! literal 0 HcmV?d00001 diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/1.wal.2 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/1.wal.2 new file mode 100644 index 0000000..45a8ca0 --- /dev/null +++ b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/1.wal.2 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/1.wal.2.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/1.wal.2.tail new file mode 100644 index 0000000000000000000000000000000000000000..c3052ace75653fdc30a0ffb21358781f2057647e GIT binary patch literal 64 rcmZQzU|_HSVn!ea0Tw6*@wtHbDf8r=Oh6h0U~(Y72oSrK_7wpDSxE$3 literal 0 HcmV?d00001 diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/2.wal.2 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/2.wal.2 new file mode 100644 index 0000000..45a8ca0 --- /dev/null +++ b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/2.wal.2 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/2.wal.2.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/seg/2.wal.2.tail new file mode 100644 index 0000000000000000000000000000000000000000..113a5ce2568ce5cd4f8b603438338a93b153aba5 GIT binary patch literal 92 zcmZQzU|OrkQxmjuGbNd F1^^qF2KE2| literal 0 HcmV?d00001 diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txd/0.wal.2 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txd/0.wal.2 new file mode 100644 index 0000000..45a8ca0 --- /dev/null +++ b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txd/0.wal.2 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txd/0.wal.2.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txd/0.wal.2.tail new file mode 100644 index 0000000000000000000000000000000000000000..1b1cb4d44c57c2d7a5122870fa6ac3e62ff7e94e GIT binary patch literal 8 KcmZQzfB*mh2mk>9 literal 0 HcmV?d00001 diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txh/0.wal.2 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txh/0.wal.2 new file mode 100644 index 0000000000000000000000000000000000000000..4424ff20ee69a435d132b2f38d14e34be282e7d0 GIT binary patch literal 281 zcmZQ!0D=Z6W(0A7fCGpnfEdPy(I7bxV1eS~6%S@HfkiO&z{H`_a6LTxmX|W4>4B*Q z+0y{UFf~x6aQ(Lhw5EdW2LYHnVQN9^V`YWG3XVBiw+VKEbwjjji#7p6y~AENi< z7qxn3G(9jgKzbUW7^;LJS^61@o!1|FonnDX!t6xW50ZlENtJ=<-F@!dHdZt}FgHMj K8A_mPVSE4$k0yZt literal 0 HcmV?d00001 diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txm/0.wal.2.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txm/0.wal.2.tail new file mode 100644 index 0000000000000000000000000000000000000000..7a94dc2dca2263aee7e11b5bdd1e094ba3934cff GIT binary patch literal 12 RcmZQ#U|?VY;!wF3e*gnX0Zjk^ literal 0 HcmV?d00001 diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txs/0.wal.2 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/AsyncCompressed/txs/0.wal.2 new file mode 100644 index 0000000000000000000000000000000000000000..9e9d7f95fcdf95e1bca521e62bb2792369e9d362 GIT binary patch literal 141 zcmZQ!0D=Z6W(0A7fCY#JfEdOHiNV}{YP(F-?av4~F3=mFU@n9AcR1jScOdMSg&%WiQ%xHRGYGLB&dTtA7 uO$FHp0x)}EYC(EMfEeaRnA^OQM1)yE8h`*@56moxEW|G$v5gT6PXYj^hzef- literal 0 HcmV?d00001 diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txm/0.wal.0 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txm/0.wal.0 new file mode 100644 index 0000000000000000000000000000000000000000..09f69d183f3f9b1ee112a3a8d1bafbaee3912f07 GIT binary patch literal 260 zcmZQ%fB+64B>}{YP(F;>_3_$WXSMqvX)t);>+R13WuxnX>4oV@V7&p>`#F2ZU1l^r yFf*XS40|r$M6u^g7KacE4m~h?V0ub#Li9G}m24Dh`w+E^g#%HT#n*{*RWGFcR literal 0 HcmV?d00001 diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txs/0.wal.0 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/Sync/txs/0.wal.0 new file mode 100644 index 0000000000000000000000000000000000000000..5dc0847eb88e50962ca55255e4b389646b4c0808 GIT binary patch literal 200 zcmZQ%fB+UCB>=>XP(Fx)vGc!7^JapI!_>j}$m((w*xi`X)WP^5VVEA4*Joye%mD#x b=E3x=a=uc-3Q-1E2h#@5?~HD(JX literal 0 HcmV?d00001 diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/0.json b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/0.json new file mode 100644 index 0000000..19428d6 --- /dev/null +++ b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/0.json @@ -0,0 +1,47 @@ +{ + "Version": "1.9.9.0", + "ComparerType": "ZoneTree.Comparers.Int32ComparerAscending", + "KeyHasherType": "ZoneTree.Hashers.DefaultKeyHasher\u003CSystem.Int32\u003E", + "KeyType": "System.Int32", + "ValueType": "System.Int32", + "KeySerializerType": "ZoneTree.Serializers.Int32Serializer", + "ValueSerializerType": "ZoneTree.Serializers.Int32Serializer", + "MutableSegmentMaxItemCount": 1000000, + "MutableSegmentBloomFilterBitsPerItem": 8, + "DiskSegmentMaxItemCount": 20000000, + "WriteAheadLogOptions": { + "WriteAheadLogMode": 1, + "CustomOptions": null, + "CompressionBlockSize": 128, + "CompressionMethod": 4, + "CompressionLevel": 0, + "SyncCompressedModeOptions": { + "EnableTailWriterJob": false, + "TailWriterJobInterval": 500 + }, + "AsyncCompressedModeOptions": { + "EmptyQueuePollInterval": 100 + }, + "EnableIncrementalBackup": false + }, + "DiskSegmentOptions": { + "DiskSegmentMode": 1, + "CompressionBlockSize": 4194304, + "CompressionMethod": 1, + "CompressionLevel": 0, + "MaximumRecordCount": 3000000, + "MinimumRecordCount": 1500000, + "KeyCacheSize": 1024, + "ValueCacheSize": 1024, + "MaterializedEntryCacheSize": 4096, + "SearchHintPrefetchSize": 16, + "KeyCacheRecordLifeTimeInMillisecond": 10000, + "ValueCacheRecordLifeTimeInMillisecond": 10000, + "DefaultSparseArrayStepSize": 1024 + }, + "MutableSegment": 1, + "ReadOnlySegments": [], + "DiskSegment": 0, + "BottomSegments": [], + "MaximumOpIndex": 0 +} \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/0.meta.wal b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/0.meta.wal new file mode 100644 index 0000000000000000000000000000000000000000..422ae4a906116923c9f3827c3b3c9e0b01bbe102 GIT binary patch literal 72 Xcmd;JfB+^ajZQJ53Bc6BXqYqr1AqV! literal 0 HcmV?d00001 diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/1.wal.1 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/1.wal.1 new file mode 100644 index 0000000..45a8ca0 --- /dev/null +++ b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/1.wal.1 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/1.wal.1.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/1.wal.1.tail new file mode 100644 index 0000000000000000000000000000000000000000..c3052ace75653fdc30a0ffb21358781f2057647e GIT binary patch literal 64 rcmZQzU|_HSVn!ea0Tw6*@wtHbDf8r=Oh6h0U~(Y72oSrK_7wpDSxE$3 literal 0 HcmV?d00001 diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/2.wal.1 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/2.wal.1 new file mode 100644 index 0000000..45a8ca0 --- /dev/null +++ b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/2.wal.1 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/2.wal.1.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/seg/2.wal.1.tail new file mode 100644 index 0000000000000000000000000000000000000000..113a5ce2568ce5cd4f8b603438338a93b153aba5 GIT binary patch literal 92 zcmZQzU|OrkQxmjuGbNd F1^^qF2KE2| literal 0 HcmV?d00001 diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txd/0.wal.1 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txd/0.wal.1 new file mode 100644 index 0000000..45a8ca0 --- /dev/null +++ b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txd/0.wal.1 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txd/0.wal.1.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txd/0.wal.1.tail new file mode 100644 index 0000000000000000000000000000000000000000..1b1cb4d44c57c2d7a5122870fa6ac3e62ff7e94e GIT binary patch literal 8 KcmZQzfB*mh2mk>9 literal 0 HcmV?d00001 diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txh/0.wal.1 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txh/0.wal.1 new file mode 100644 index 0000000000000000000000000000000000000000..4424ff20ee69a435d132b2f38d14e34be282e7d0 GIT binary patch literal 281 zcmZQ!0D=Z6W(0A7fCGpnfEdPy(I7bxV1eS~6%S@HfkiO&z{H`_a6LTxmX|W4>4B*Q z+0y{UFf~x6aQ(Lhw5EdW2LYHnVQN9l#$)$W6&!5~3m-3cZr8(j}fFHDcO$X&4B z$e{h{%xHRGW`OiGKrvJaL-&DuD0VVGDxSdtm4w-etREx=)05wHAFP+}bITf5G(9jk MK!q84plV@!02(1C>Hq)$ literal 0 HcmV?d00001 diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txm/0.wal.1.tail b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txm/0.wal.1.tail new file mode 100644 index 0000000000000000000000000000000000000000..fcc0436c335629b9f6e16c35f354c7a102f35f27 GIT binary patch literal 12 RcmZQ#U|?VY;-w#rOaKN$0s{a5 literal 0 HcmV?d00001 diff --git a/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txs/0.wal.1 b/src/ZoneTree.UnitTests/Fixtures/LegacyCrcWals/SyncCompressed/txs/0.wal.1 new file mode 100644 index 0000000000000000000000000000000000000000..9e9d7f95fcdf95e1bca521e62bb2792369e9d362 GIT binary patch literal 141 zcmZQ!0D=Z6W(0A7fCY#JfEdOHiNV x.Checksum != 0), Is.True); + var provider = new CountingWalProvider(DataDirectory); + var factory = CreateFactory(mode).SetWriteAheadLogProvider(_ => provider); + if (configureTransactionsBeforeOpen) + { + factory.ConfigureTransactionLog(log => log.CompactionThreshold = int.MaxValue); + factory.ConfigureTransactionLog(log => log.CompactionThreshold = int.MaxValue); + } + + using (var tree = factory.OpenTransactional()) + { + Assert.That(provider.MigrationVersions, Is.EqualTo(new[] { new Version("1.9.9.0") })); + Assert.That(tree.Maintenance.ZoneTree.Maintenance.MutableSegment.SegmentId, Is.EqualTo(2)); + Assert.That(tree.Maintenance.ZoneTree.Maintenance.ReadOnlySegments.Count, Is.EqualTo(1)); + Assert.That(tree.Maintenance.UncommittedTransactionIds, Is.EqualTo(new[] { 3L })); + AssertCommitted(tree, 1, 10); + AssertCommitted(tree, 2, 20); + AssertCommitted(tree, 3, 30); + Assert.That(tree.ReadCommittedContainsKey(4), Is.False); + foreach (var file in originalFiles) + { + Assert.That(File.Exists(file), Is.False); + Assert.That(File.Exists(file.Replace(".wal.", ".wal_crc.", StringComparison.Ordinal)), Is.True); + } + Assert.That(Directory.GetFiles(DataDirectory, "*.wal_xxh3.*", SearchOption.AllDirectories), Is.Empty); + + tree.PrepareAndCommit(3); + tree.UpsertAutoCommit(5, 50); + } + + var legacyWal = WalPath("seg", 2, "wal_crc", mode); + var appendedEntries = ReadEntries(legacyWal, mode); + Assert.That(appendedEntries.Count, Is.GreaterThan(oldEntries.Count)); + Assert.That(appendedEntries.Take(oldEntries.Count).Select(x => x.Checksum), + Is.EqualTo(oldEntries.Select(x => x.Checksum))); + Assert.That(appendedEntries.Skip(oldEntries.Count).Select(x => x.Checksum), Is.All.Zero); + + using (var tree = CreateFactory(mode).OpenTransactional()) + { + Assert.That(tree.Maintenance.UncommittedTransactionIds, Is.Empty); + AssertCommitted(tree, 2, 999); + AssertCommitted(tree, 4, 40); + AssertCommitted(tree, 5, 50); + tree.Maintenance.ZoneTree.Maintenance.MoveMutableSegmentForward(); + tree.UpsertAutoCommit(6, 60); + } + + var newWal = WalPath("seg", 3, "wal_xxh3", mode); + var newEntries = ReadEntries(newWal, mode); + Assert.That(newEntries, Is.Not.Empty); + Assert.That(newEntries.All(x => x.ValidateChecksum()), Is.True); + using var reopened = CreateFactory(mode).OpenTransactional(); + AssertCommitted(reopened, 1, 10); + AssertCommitted(reopened, 6, 60); + } + + [Test] + public void ProviderSuppliedThroughOptionsMigratesBeforeTransactionConfiguration() + { + CopyFixture(WriteAheadLogMode.Sync); + var provider = new CountingWalProvider(DataDirectory); + var factory = CreateFactory(WriteAheadLogMode.Sync) + .Configure(options => options.WriteAheadLogProvider = provider) + .ConfigureTransactionLog(log => log.CompactionThreshold = int.MaxValue); + using var tree = factory.OpenTransactional(); + Assert.That(provider.MigrationVersions, Is.EqualTo(new[] { new Version("1.9.9.0") })); + Assert.That(tree.Maintenance.UncommittedTransactionIds, Is.EqualTo(new[] { 3L })); + AssertCommitted(tree, 2, 20); + } + + [TestCase(WriteAheadLogMode.Sync)] + [TestCase(WriteAheadLogMode.SyncCompressed)] + [TestCase(WriteAheadLogMode.AsyncCompressed)] + public void PartiallyRenamedDatabaseCompletesMigration(WriteAheadLogMode mode) + { + CopyFixture(mode); + var original = WalPath("seg", 2, "wal", mode); + File.Move(original, WalPath("seg", 2, "wal_crc", mode)); + // Compressed tails deliberately retain their old names, as after an interrupted rename. + using var tree = CreateFactory(mode).OpenTransactional(); + AssertCommitted(tree, 2, 20); + Assert.That(tree.Maintenance.UncommittedTransactionIds, Is.EqualTo(new[] { 3L })); + Assert.That(LegacyWalFiles(), Is.Empty); + } + + [TestCase(WriteAheadLogMode.Sync)] + [TestCase(WriteAheadLogMode.SyncCompressed)] + [TestCase(WriteAheadLogMode.AsyncCompressed)] + public void LegacyChecksumIsIgnoredButNewChecksumIsValidated(WriteAheadLogMode mode) + { + CopyFixture(mode); + CorruptChecksum(WalPath("seg", 2, "wal", mode), mode); + using (var tree = CreateFactory(mode).Open()) + { + Assert.That(tree.TryGet(2, out var value), Is.True); + Assert.That(value, Is.EqualTo(999)); + tree.Maintenance.MoveMutableSegmentForward(); + tree.Upsert(6, 60); + } + + var newWal = WalPath("seg", 3, "wal_xxh3", mode); + CorruptChecksum(newWal, mode); + var provider = new WriteAheadLogProvider(new ConsoleLogger(), new LocalFileStreamProvider(), DataDirectory); + using var wal = provider.GetOrCreateWAL(3, "seg", WalOptions(mode), new Int32Serializer(), new Int32Serializer()); + var result = wal.ReadLogEntries(true, true, false); + Assert.That(result.Success, Is.False); + Assert.That(result.Exceptions.Values.Any(x => x.Message.Contains("Checksum failed", StringComparison.Ordinal)), Is.True); + } + + [TestCase(WriteAheadLogMode.Sync)] + [TestCase(WriteAheadLogMode.SyncCompressed)] + [TestCase(WriteAheadLogMode.AsyncCompressed)] + public void ReplacingLegacyWalKeepsZeroChecksumsAndCanBeReopened(WriteAheadLogMode mode) + { + CopyFixture(mode); + using (CreateFactory(mode).Open()) { } + var provider = new WriteAheadLogProvider(new ConsoleLogger(), new LocalFileStreamProvider(), DataDirectory); + using (var wal = provider.GetOrCreateWAL(2, "seg", WalOptions(mode), new Int32Serializer(), new Int32Serializer())) + { + wal.ReplaceWriteAheadLog(new[] { 2, 3, 4 }, new[] { 222, 30, 40 }, true); + } + var entries = ReadEntries(WalPath("seg", 2, "wal_crc", mode), mode); + Assert.That(entries.Count, Is.EqualTo(3)); + Assert.That(entries.Select(x => x.Checksum), Is.All.Zero); + using var tree = CreateFactory(mode).Open(); + Assert.That(tree.TryGet(2, out var value), Is.True); + Assert.That(value, Is.EqualTo(222)); + } + + ZoneTreeFactory CreateFactory(WriteAheadLogMode mode) + { + return new ZoneTreeFactory() + .SetDataDirectory(DataDirectory) + .Configure(options => options.AllowUnsafeOptionValues = true) + .ConfigureWriteAheadLogOptions(options => + { + options.WriteAheadLogMode = mode; + options.CompressionBlockSize = 128; + options.CompressionMethod = CompressionMethod.None; + options.SyncCompressedModeOptions.EnableTailWriterJob = false; + }); + } + + static WriteAheadLogOptions WalOptions(WriteAheadLogMode mode) + { + var options = new WriteAheadLogOptions + { + WriteAheadLogMode = mode, + CompressionBlockSize = 128, + CompressionMethod = CompressionMethod.None + }; + options.SyncCompressedModeOptions.EnableTailWriterJob = false; + return options; + } + + void CopyFixture(WriteAheadLogMode mode) + { + var source = Path.Combine(TestContext.CurrentContext.TestDirectory, "Fixtures", "LegacyCrcWals", mode.ToString()); + Assert.That(Directory.Exists(source), Is.True, "Legacy fixtures must be copied to the test output."); + foreach (var file in Directory.GetFiles(source, "*", SearchOption.AllDirectories)) + { + var destination = Path.Combine(DataDirectory, Path.GetRelativePath(source, file)); + Directory.CreateDirectory(Path.GetDirectoryName(destination)); + File.Copy(file, destination); + } + } + + string WalPath(string category, int segmentId, string format, WriteAheadLogMode mode) + => Path.Combine(DataDirectory, category, $"{segmentId}.{format}.{(int)mode}"); + + string[] LegacyWalFiles() + => Directory.GetFiles(DataDirectory, "*", SearchOption.AllDirectories) + .Where(x => Path.GetFileName(x).Contains(".wal.", StringComparison.Ordinal)).ToArray(); + + static void AssertCommitted(ITransactionalZoneTree tree, int key, int expected) + { + Assert.That(tree.ReadCommittedTryGet(key, out var value), Is.True); + Assert.That(value, Is.EqualTo(expected)); + } + + static Stream OpenRecordStream(string path, WriteAheadLogMode mode) + => mode == WriteAheadLogMode.Sync + ? new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None) + : new CompressedFileStream(new ConsoleLogger(), new LocalFileStreamProvider(), path, + 128, false, 0, CompressionMethod.None, 0); + + static List ReadEntries(string path, WriteAheadLogMode mode) + { + using var stream = OpenRecordStream(path, mode); + stream.Seek(0, SeekOrigin.Begin); + using var reader = new BinaryReader(stream); + var entries = new List(); + while (stream.Position < stream.Length) + { + LogEntry entry = default; + LogEntry.ReadLogEntry(reader, ref entry); + entries.Add(entry); + } + return entries; + } + + static void CorruptChecksum(string path, WriteAheadLogMode mode) + { + using var stream = OpenRecordStream(path, mode); + stream.Seek(0, SeekOrigin.Begin); + using var reader = new BinaryReader(stream, System.Text.Encoding.UTF8, true); + var bytes = reader.ReadBytes((int)stream.Length); + bytes[^1] ^= 0x80; + stream.SetLength(0); + stream.Seek(0, SeekOrigin.Begin); + stream.Write(bytes, 0, bytes.Length); + } + + sealed class CountingWalProvider(string directory) : IWriteAheadLogProvider + { + readonly WriteAheadLogProvider Inner = new(new ConsoleLogger(), new LocalFileStreamProvider(), directory); + public List MigrationVersions { get; } = new(); + public void MigrateWals(Version databaseVersion) + { + MigrationVersions.Add(databaseVersion); + Inner.MigrateWals(databaseVersion); + } + public void InitCategory(string category) => Inner.InitCategory(category); + public IWriteAheadLog GetOrCreateWAL(long segmentId, string category, + WriteAheadLogOptions options, ISerializer keySerializer, ISerializer valueSerializer) + => Inner.GetOrCreateWAL(segmentId, category, options, keySerializer, valueSerializer); + public IWriteAheadLog GetWAL(long segmentId, string category) + => Inner.GetWAL(segmentId, category); + public bool RemoveWAL(long segmentId, string category) => Inner.RemoveWAL(segmentId, category); + public void DropStore() => Inner.DropStore(); + } +} diff --git a/src/ZoneTree.UnitTests/ZoneTree.UnitTests.csproj b/src/ZoneTree.UnitTests/ZoneTree.UnitTests.csproj index 0c1a191..c86ba78 100644 --- a/src/ZoneTree.UnitTests/ZoneTree.UnitTests.csproj +++ b/src/ZoneTree.UnitTests/ZoneTree.UnitTests.csproj @@ -30,4 +30,8 @@ + + + +