From 4c55b8c4f87db2bede93af534a7481727bf92fa5 Mon Sep 17 00:00:00 2001 From: Ahmed Yasin Koculu Date: Sun, 25 Oct 2026 09:13:50 +0100 Subject: [PATCH] Improve getting started and usage guides. --- zonetree-dev/README.md | 3 + zonetree-dev/content/docs/_nav.json | 18 +- .../partitioning-and-replication.md | 6 +- .../content/docs/concepts/key-ordering.md | 2 +- .../content/docs/concepts/lsm-tree.mdx | 6 +- zonetree-dev/content/docs/getting-started.md | 73 ----- zonetree-dev/content/docs/getting-started.mdx | 165 ++++++++++ .../docs/operations/troubleshooting.md | 2 +- .../content/docs/usage/atomic-operations.md | 169 ---------- .../content/docs/usage/atomic-operations.mdx | 154 +++++++++ zonetree-dev/content/docs/usage/index.mdx | 102 ++++++ .../docs/usage/iteration-and-range-scans.md | 158 --------- .../docs/usage/iteration-and-range-scans.mdx | 174 ++++++++++ .../content/docs/usage/maintenance.md | 279 ---------------- .../content/docs/usage/maintenance.mdx | 308 ++++++++++++++++++ .../content/docs/usage/opening-a-tree.md | 79 ----- .../content/docs/usage/opening-a-tree.mdx | 132 ++++++++ .../content/docs/usage/reads-and-writes.md | 81 ----- .../content/docs/usage/reads-and-writes.mdx | 144 ++++++++ .../content/docs/usage/transactions.md | 130 -------- .../content/docs/usage/transactions.mdx | 188 +++++++++++ zonetree-dev/content/ui/guide-styles.ts | 62 ++++ zonetree-dev/content/ui/styles.ts | 2 + zonetree-dev/package.json | 2 +- 24 files changed, 1453 insertions(+), 986 deletions(-) delete mode 100644 zonetree-dev/content/docs/getting-started.md create mode 100644 zonetree-dev/content/docs/getting-started.mdx delete mode 100644 zonetree-dev/content/docs/usage/atomic-operations.md create mode 100644 zonetree-dev/content/docs/usage/atomic-operations.mdx create mode 100644 zonetree-dev/content/docs/usage/index.mdx delete mode 100644 zonetree-dev/content/docs/usage/iteration-and-range-scans.md create mode 100644 zonetree-dev/content/docs/usage/iteration-and-range-scans.mdx delete mode 100644 zonetree-dev/content/docs/usage/maintenance.md create mode 100644 zonetree-dev/content/docs/usage/maintenance.mdx delete mode 100644 zonetree-dev/content/docs/usage/opening-a-tree.md create mode 100644 zonetree-dev/content/docs/usage/opening-a-tree.mdx delete mode 100644 zonetree-dev/content/docs/usage/reads-and-writes.md create mode 100644 zonetree-dev/content/docs/usage/reads-and-writes.mdx delete mode 100644 zonetree-dev/content/docs/usage/transactions.md create mode 100644 zonetree-dev/content/docs/usage/transactions.mdx create mode 100644 zonetree-dev/content/ui/guide-styles.ts diff --git a/zonetree-dev/README.md b/zonetree-dev/README.md index a2b1341..9b06c21 100644 --- a/zonetree-dev/README.md +++ b/zonetree-dev/README.md @@ -38,6 +38,9 @@ Restore those from your private copy before building in a fresh checkout. - `content/ui/skin.ts` defines the shared light and dark emerald palettes. - `content/ui/styles.ts` generates responsive landing and documentation styles through PureStack's typed style builder. +- `content/docs/getting-started.mdx` and `content/docs/usage/` provide runnable + introductions and task-focused API guides. `content/ui/guide-styles.ts` shares + their typography, callouts, examples, and onward navigation. - `content/ui/components/hero.ts` contains the SVG storage lifecycle: parallel writes, mutable segment rollover, read-only segments, compaction, and multipart disk storage. Live query requests and results run alongside the storage transitions. diff --git a/zonetree-dev/content/docs/_nav.json b/zonetree-dev/content/docs/_nav.json index ab73105..2103c8e 100644 --- a/zonetree-dev/content/docs/_nav.json +++ b/zonetree-dev/content/docs/_nav.json @@ -2,8 +2,9 @@ "pageLinks": true, "icons": { "index.mdx": "iconoir:home", - "getting-started.md": "iconoir:rocket", + "getting-started.mdx": "iconoir:rocket", "usage/": "iconoir:database-script", + "usage/index.mdx": "iconoir:database-script", "concepts/": "iconoir:book", "storage/": "iconoir:database", "benchmark/": "iconoir:stats-report", @@ -15,14 +16,15 @@ }, "sequence": [ "index.mdx", - "getting-started.md", + "getting-started.mdx", "usage/", - "usage/opening-a-tree.md", - "usage/reads-and-writes.md", - "usage/atomic-operations.md", - "usage/transactions.md", - "usage/iteration-and-range-scans.md", - "usage/maintenance.md", + "usage/index.mdx", + "usage/opening-a-tree.mdx", + "usage/reads-and-writes.mdx", + "usage/atomic-operations.mdx", + "usage/transactions.mdx", + "usage/iteration-and-range-scans.mdx", + "usage/maintenance.mdx", "concepts/", "concepts/lsm-tree.mdx", "concepts/segments.md", diff --git a/zonetree-dev/content/docs/building-systems/partitioning-and-replication.md b/zonetree-dev/content/docs/building-systems/partitioning-and-replication.md index 387e871..d79dd8e 100644 --- a/zonetree-dev/content/docs/building-systems/partitioning-and-replication.md +++ b/zonetree-dev/content/docs/building-systems/partitioning-and-replication.md @@ -268,7 +268,7 @@ The iterator merges the collected layers into key order and emits the current va `ReadOnlyRegion` offers a similar fixed collected view without advancing the mutable segment; it excludes records still in that segment. `AutoRefresh` is useful for live scans, but keys inserted behind its position can be missed. Use the snapshot view plus a captured operation stream for a controlled online transfer. -Dispose export iterators promptly: they pin persistent segments and can delay reclamation of files replaced by compaction. For full iterator behavior, see [iteration and range scans](../usage/iteration-and-range-scans.md). +Dispose export iterators promptly: they pin persistent segments and can delay reclamation of files replaced by compaction. For full iterator behavior, see [iteration and range scans](../usage/iteration-and-range-scans.mdx). ### Join the export to the operation stream @@ -329,7 +329,7 @@ ZoneTree's local APIs give the node several useful choices: | Stable export | Snapshot iterator | The collected engine view | | Recovery of local writes | Configured WAL mode and storage provider | Local recovery according to that mode and provider | -Ordinary `Upsert` does not participate in the atomic methods' coordination lock. Use a compatible write discipline for keys whose correctness depends on atomic read-modify-write. Transactions in one tree also do not commit changes across another tree or node. See [atomic operations](../usage/atomic-operations.md) and [transactions](../usage/transactions.md). +Ordinary `Upsert` does not participate in the atomic methods' coordination lock. Use a compatible write discipline for keys whose correctness depends on atomic read-modify-write. Transactions in one tree also do not commit changes across another tree or node. See [atomic operations](../usage/atomic-operations.mdx) and [transactions](../usage/transactions.mdx). Transactional trees expose `BeginTransaction`, `Prepare`, `Commit`, and `Rollback` as separate lifecycle operations, alongside convenience methods that combine steps. These give a coordinating layer explicit local transaction controls. In the built-in optimistic implementation, `Prepare` checks local dependencies and marks the transaction ready in memory. A distributed commit protocol must establish its own durable decision records and recovery rules around those calls. `ZoneTreeFactory.SetTransactionLog` also permits a custom `ITransactionLog` implementation when the integration needs a different transaction-log design. @@ -362,7 +362,7 @@ Treat callbacks as notifications and keep them short; enqueue expensive transfer Some maintenance record counts include multiple physical versions of a key across segments. Use them as storage-pressure measurements rather than unique-key counts. Coordinate aggregate memory and I/O budgets across hosted partitions, and release iterators and backup activity before removing a replica's files. -See [maintenance](../usage/maintenance.md), [file stream providers](../storage/file-stream-providers.md), and the [API overview](../reference/api-overview.md) for the individual controls. +See [maintenance](../usage/maintenance.mdx), [file stream providers](../storage/file-stream-providers.md), and the [API overview](../reference/api-overview.md) for the individual controls. ## Implementation references diff --git a/zonetree-dev/content/docs/concepts/key-ordering.md b/zonetree-dev/content/docs/concepts/key-ordering.md index 250b157..d9df55e 100644 --- a/zonetree-dev/content/docs/concepts/key-ordering.md +++ b/zonetree-dev/content/docs/concepts/key-ordering.md @@ -86,4 +86,4 @@ Changing field order, comparison logic, culture rules, or binary encoding change ZoneTree supports forward and reverse iterators. Reverse iteration is useful for latest-first time-series queries, descending leaderboards, and tail scans over ordered data. -See [iteration and range scans](../usage/iteration-and-range-scans.md). +See [iteration and range scans](../usage/iteration-and-range-scans.mdx). diff --git a/zonetree-dev/content/docs/concepts/lsm-tree.mdx b/zonetree-dev/content/docs/concepts/lsm-tree.mdx index 5bc0535..5b26802 100644 --- a/zonetree-dev/content/docs/concepts/lsm-tree.mdx +++ b/zonetree-dev/content/docs/concepts/lsm-tree.mdx @@ -40,7 +40,7 @@ An `Upsert` enters the current mutable segment. Its concurrent B+Tree keeps keys Recent writes are also recorded through the configured write-ahead log. WAL mode determines the logging path and durability boundary. The mutable segment is where the application reads and updates current in-memory records; the WAL supports their recovery. -The default B+Tree lock mode uses node-level synchronization. Multiple application threads can write to the same ZoneTree, and independent reads can run concurrently. Operations that need coordinated read-modify-write behavior have separate [atomic APIs](../usage/atomic-operations.md). +The default B+Tree lock mode uses node-level synchronization. Multiple application threads can write to the same ZoneTree, and independent reads can run concurrently. Operations that need coordinated read-modify-write behavior have separate [atomic APIs](../usage/atomic-operations.mdx). ### 2. Move a full segment to read-only memory @@ -76,7 +76,7 @@ An iterator merges ordered streams from its included segments into one forward o The default iterator can refresh its segment set after mutable-segment rollover. For a stable scan, `IteratorType.Snapshot` moves the mutable segment forward and scans the resulting read-only and persistent region, excluding later writes. Dispose iterators promptly: they retain the disk segments they use, which can delay physical removal of replaced files. -See [iteration and range scans](../usage/iteration-and-range-scans.md) for visibility options, seeking, and iterator lifetime. +See [iteration and range scans](../usage/iteration-and-range-scans.mdx) for visibility options, seeking, and iterator lifetime. ## Multipart storage keeps merge work local @@ -132,7 +132,7 @@ Bottom merging has its own explicit entry point, `maintainer.StartBottomSegments Watch read-only segment accumulation alongside merge duration and write rate. A growing queue means frozen memory is arriving faster than maintenance is clearing it. Budget compaction resources and record limits together; the mutable-segment limit alone does not cap the tree's total memory use. -See [maintenance](../usage/maintenance.md) and [memory usage](../storage/memory-usage.md) for operational controls. +See [maintenance](../usage/maintenance.mdx) and [memory usage](../storage/memory-usage.md) for operational controls. ## The controls behind the shape diff --git a/zonetree-dev/content/docs/getting-started.md b/zonetree-dev/content/docs/getting-started.md deleted file mode 100644 index 23795cf..0000000 --- a/zonetree-dev/content/docs/getting-started.md +++ /dev/null @@ -1,73 +0,0 @@ -# Getting Started - -This guide creates a persistent ZoneTree, writes and reads a record, and scans -an ordered key range. - -## Install - -```bash -dotnet add package ZoneTree -``` - -## Open A Tree - -```csharp -using ZoneTree; - -using var zoneTree = new ZoneTreeFactory() - .SetDataDirectory("data/my-zone-tree") - .OpenOrCreate(); - -using var maintainer = zoneTree.CreateMaintainer(); -``` - -`OpenOrCreate()` opens the existing tree in `data/my-zone-tree`, or creates it -when it does not exist. - -`CreateMaintainer()` creates ZoneTree's ready-to-use background maintenance -worker and automates merge scheduling and inactive-cache cleanup. Creating a -maintainer is optional; without one, those jobs do not run automatically. - -## Write And Read - -```csharp -zoneTree.Upsert(1, "Hello ZoneTree"); - -if (zoneTree.TryGet(1, out var value)) - Console.WriteLine(value); -``` - -`Upsert` inserts a new key or replaces the current value. `TryGet` returns the -newest live value for the key. - -## Scan An Ordered Range - -```csharp -using var iterator = zoneTree.CreateIterator(); - -iterator.Seek(100); -while (iterator.Next() && iterator.CurrentKey < 200) - Console.WriteLine($"{iterator.CurrentKey}: {iterator.CurrentValue}"); -``` - -A forward iterator returns keys in comparer order. `Seek(100)` positions it so -the next successful `Next()` returns the first key greater than or equal to -`100`. - -Dispose iterators when the scan is complete. - -## Shutdown - -The tree, maintainer, and iterators are disposable. The `using` declarations in -the examples dispose them when their scopes end. Maintainer disposal waits for -its tracked merge threads to finish. See [maintenance](usage/maintenance.md) -when you need to configure maintenance or observe merge completion explicitly. - -## Next Steps - -* [Opening and configuring a tree](usage/opening-a-tree.md) -* [Reads and writes](usage/reads-and-writes.md) -* [Iteration and range scans](usage/iteration-and-range-scans.md) -* [Maintenance](usage/maintenance.md) -* [WAL modes](durability/wal-modes.md) -* [Production checklist](operations/production-checklist.md) diff --git a/zonetree-dev/content/docs/getting-started.mdx b/zonetree-dev/content/docs/getting-started.mdx new file mode 100644 index 0000000..9c56eac --- /dev/null +++ b/zonetree-dev/content/docs/getting-started.mdx @@ -0,0 +1,165 @@ +--- +title: Getting Started +description: Create a C# console app that opens a persistent ZoneTree, writes and reads records, scans an ordered range, and reopens the same data directory. +--- + + + + + One package +

ZoneTree runs inside your .NET process.

 + One directory +

Open existing data or create a new tree.

 + Ordered keys +

Read a single record or scan a range.

 + + +## 1. Create a console app + +You need a .NET SDK installed. From a terminal, create a project and add the package: + +```sh +dotnet new console -n ZoneTreeQuickstart +cd ZoneTreeQuickstart +dotnet add package ZoneTree +``` + +The example uses `int` keys and `string` values. ZoneTree supplies the comparer and serializers for these types. + +## 2. Replace Program.cs + +Paste this complete program into `Program.cs`: + +```csharp +using ZoneTree; + +using var tree = new ZoneTreeFactory() + .SetDataDirectory("data/getting-started") + .OpenOrCreate(); +using var maintainer = tree.CreateMaintainer(); + +tree.Upsert(10, "Ada"); +tree.Upsert(20, "Grace"); +tree.Upsert(30, "Linus"); + +if (tree.TryGet(20, out var name)) + Console.WriteLine($"Read 20: {name}"); + +using var iterator = tree.CreateIterator(); +iterator.Seek(10); + +while (iterator.Next()) +{ + if (iterator.CurrentKey >= 30) + break; + + Console.WriteLine($"{iterator.CurrentKey}: {iterator.CurrentValue}"); +} + +tree.TryDelete(30, out _); +Console.WriteLine($"Contains 30: {tree.ContainsKey(30)}"); +``` + + +

+ The maintainer schedules background merges and cleans inactive read caches. + Keep it alive while the tree is in use. The using declarations + dispose the iterator, maintainer, and tree in reverse order. +

+ + +## 3. Run and inspect the result + +```sh +dotnet run +``` + +With a fresh data directory, the output is: + +
+```text +Read 20: Grace +10: Ada +20: Grace +Contains 30: False +``` +
+ +Here is what the program does: + +| Operation | What to expect | +| --- | --- | +| `OpenOrCreate()` | Creates the tree on the first run and opens it on later runs. | +| `Upsert(key, value)` | Inserts a key or replaces its current value. Repeating the sample updates the same keys. | +| `TryGet(20, out name)` | Returns `true` and the latest live value for key `20`. | +| `Seek(10)` followed by `Next()` | Starts at the first key greater than or equal to `10`. The upper-bound check makes the scan `[10, 30)`. | +| `TryDelete(30, out _)` | Writes a deletion marker when the key exists. Later ordinary reads hide the deleted record. | + +The path is relative to the application's working directory. Keep that working directory unchanged when running the example again. + +## 4. Open the persisted data + +To read without writing the sample records again, replace `Program.cs` with: + +```csharp +using ZoneTree; + +using var tree = new ZoneTreeFactory() + .SetDataDirectory("data/getting-started") + .Open(); +using var maintainer = tree.CreateMaintainer(); + +if (tree.TryGet(20, out var name)) + Console.WriteLine($"Persisted 20: {name}"); +``` + +Run `dotnet run` from the same project directory. It prints `Persisted 20: Grace`. `Open()` requires existing data, so a path mistake fails instead of silently creating another tree. + + +

+ This example uses the default WAL configuration. A normal close-and-reopen + demonstrates persistence; it does not establish what survives a process or + machine failure. Read WAL modes to + choose the durability boundary your application needs. +

+ + +## Choose what to learn next + +Continue with one of these guides, or [browse the Usage overview](usage/index.mdx) to compare reads, writes, atomic operations, transactions, scans, and maintenance. + + + + + diff --git a/zonetree-dev/content/docs/operations/troubleshooting.md b/zonetree-dev/content/docs/operations/troubleshooting.md index 414ac48..9f7b5f9 100644 --- a/zonetree-dev/content/docs/operations/troubleshooting.md +++ b/zonetree-dev/content/docs/operations/troubleshooting.md @@ -230,7 +230,7 @@ First actions: * inspect `CommitResult.IsPendingTransactions`, * roll back abandoned transaction ids with `RollbackUncommittedTransactionIdsBefore(...)` during operational cleanup. -See [transactions](../usage/transactions.md). +See [transactions](../usage/transactions.mdx). ## Live Backup Generation Is Skipped diff --git a/zonetree-dev/content/docs/usage/atomic-operations.md b/zonetree-dev/content/docs/usage/atomic-operations.md deleted file mode 100644 index 667319c..0000000 --- a/zonetree-dev/content/docs/usage/atomic-operations.md +++ /dev/null @@ -1,169 +0,0 @@ -# Atomic Operations - -Atomic methods are ZoneTree's single-key read-modify-write tools. - -Use them when the next value depends on the value that is already visible for the same key: - -| Need | Use | -| --- | --- | -| Increment a counter | atomic method | -| Append to the current value | atomic method | -| Compare current value before replacing it | atomic method | -| Initialize if absent, update if present | `TryAtomicAddOrUpdate` | -| Simple insert or replace | `Upsert` | -| Several keys must change together | transaction | - -`Upsert`, `TryAdd`, `TryDelete`, and `ForceDelete` are the normal high-throughput write APIs. They are thread-safe, but they do not join the atomic-method lock. If a key's correctness depends on read-modify-write behavior, keep every write for that key on the atomic path. - -## What Atomic Means - -ZoneTree is an LSM-tree. The newest value for a key may be in the mutable segment, a read-only in-memory segment, the disk segment, or a bottom segment until maintenance merges older layers away. - -Atomic methods synchronize this sequence: - -1. Read the current visible value across the LSM-tree layers. -2. Decide whether to add, update, cancel, or replace. -3. Write the new value to the mutable segment. - -Atomic methods are ordered with other atomic methods on the same tree. Regular writes do not participate in that ordering. That is why `AtomicUpsert` exists: it is the upsert form to use when a key also participates in atomic read-modify-write operations. - -## Counter - -```csharp -const string key = "stats:user:42:views"; - -zoneTree.TryAtomicAddOrUpdate( - key, - valueToAdd: 1, - valueUpdater: (ref long value) => - { - value++; - return true; - }); -``` - -If the key is absent, ZoneTree writes `1`. If it is present, the updater receives the current value and writes back the incremented value. - -## Compare And Set - -```csharp -var found = zoneTree.TryAtomicGetAndUpdate( - "order:123:state", - out var current, - (ref OrderState state) => - { - if (state != OrderState.Pending) - return false; - - state = OrderState.Confirmed; - return true; - }); -``` - -`found` is `false` only when the key is not found. If the key is found and the delegate returns `false`, the method still returns `true` because the read succeeded, but no new value is written. - -## Initialize Or Update With A Callback - -```csharp -zoneTree.TryAtomicAddOrUpdate( - key: "counter:global", - valueToAdd: 1, - valueUpdater: (ref long value) => - { - value++; - return true; - }, - result: (in long value, long opIndex, OperationResult result) => - { - Console.WriteLine($"{result}: {value} at {opIndex}"); - }); -``` - -The result callback reports: - -| Result | Meaning | -| --- | --- | -| `Added` | The key was absent and a new value was written. | -| `Updated` | The key existed and the updated value was written. | -| `Cancelled` | The adder or updater returned `false`; no value was written. | - -When an operation is cancelled, the callback receives `opIndex` `0`. - -`TryAtomicAddOrUpdate` returns `true` when it adds a new key and `false` when it updates or cancels. Use the result callback when the caller needs to distinguish `Updated` from `Cancelled`. - -## Method Guide - -| Method | Behavior | -| --- | --- | -| `TryAtomicAdd(key, value, out opIndex)` | Adds only when the key is absent. Returns `false` and `opIndex = 0` when the key already exists. | -| `TryAtomicUpdate(key, value, out opIndex)` | Replaces only when the key exists. Returns `false` and `opIndex = 0` when the key is absent. | -| `AtomicUpsert(key, value)` | Adds or replaces under the atomic-method lock and returns the operation index. | -| `TryAtomicGetAndUpdate(key, out value, updater)` | Reads the current value and lets the updater decide whether to write a replacement. Returns `false` when the key is absent. | -| `TryAtomicAddOrUpdate(key, valueToAdd, updater, result)` | Adds `valueToAdd` when absent; otherwise updates the current value. | -| `TryAtomicAddOrUpdate(key, adder, updater, result)` | Lets the adder create the absent value and the updater change the existing value. Either delegate can cancel by returning `false`. | - -All successful writes are assigned a normal operation index. Operation indexes preserve ZoneTree's producer write order and are useful for replay, replication, audit, and restore workflows. - -## Delegate Rules - -`ValueUpdaterDelegate` and `ValueAdderDelegate` receive a local `TValue` variable by `ref`. - -```csharp -public delegate bool ValueUpdaterDelegate(ref TValue value); -public delegate bool ValueAdderDelegate(ref TValue value); -``` - -Return `true` to commit the local value. Return `false` to cancel the write. - -Keep delegates and result callbacks short and deterministic. `TryAtomicAddOrUpdate` may retry when the mutable segment is frozen or full, so adder/updater delegates can be invoked more than once before the method finishes. Avoid external side effects inside those delegates unless repeating the side effect is acceptable. - -For mutable reference types, assigning a new object to the `ref` parameter is the safest pattern. In-place mutation can be acceptable when the change is idempotent, repeatable, and never cancelled after mutation. It still changes the shared object immediately, before ZoneTree has necessarily accepted the write, assigned an operation index, and appended the WAL record. - -Returning `false` prevents ZoneTree from writing a new value, but it cannot undo in-place changes already made to a shared object reference. - -```csharp -zoneTree.TryAtomicGetAndUpdate(1, out var user, (ref UserSnapshot value) => -{ - if (!ShouldRename(value)) - return false; - - value = value with { Name = "Bob" }; - return true; -}); -``` - -See [value mutability](../concepts/value-mutability.md). - -## Mixing Write Modes - -Mixing write modes is fine when they protect different keys or different invariants: - -* `stats:user:123` uses atomic increments, -* `profile:user:123` uses regular `Upsert`, -* secondary index entries use regular `Upsert` or `ForceDelete`. - -Do not mix regular writes and atomic writes for the same read-modify-write invariant: - -```csharp -// Avoid this shape for the same counter key. -zoneTree.TryAtomicAddOrUpdate("counter", 1, (ref long value) => -{ - value++; - return true; -}); - -zoneTree.Upsert("counter", 0); -``` - -The `Upsert` is thread-safe, but it is not synchronized with the atomic read-decision-write sequence. If the key belongs to an atomic workflow, use `AtomicUpsert` for unconditional replacement. - -## When To Use Transactions - -Atomic methods coordinate one key's current value. Use transactions when a correctness rule spans multiple keys: - -* transfer from one account key to another, -* append an event and update a separate stream pointer, -* update a record and several secondary indexes as one unit, -* reserve a global sequence number and write the payload in the same commit. - -For one-key counters, compare-and-set logic, and initialize-or-update behavior, atomic methods are the smaller and faster tool. diff --git a/zonetree-dev/content/docs/usage/atomic-operations.mdx b/zonetree-dev/content/docs/usage/atomic-operations.mdx new file mode 100644 index 0000000..6baa4d9 --- /dev/null +++ b/zonetree-dev/content/docs/usage/atomic-operations.mdx @@ -0,0 +1,154 @@ +--- +title: Atomic Operations +description: Use ZoneTree's atomic add, update, and read-modify-write APIs; distinguish found, added, updated, and cancelled results; and keep competing writes on one coordination path. +--- + + + + +

+ Ordinary Upsert, TryAdd, TryDelete, + and ForceDelete do not join the atomic-method lock. If + correctness depends on an atomic update, every competing writer for that key + must respect the same coordination path. +

+ + +## Build a counter + +This complete program uses string keys and `long` values. The counter starts at `1` when absent and increments when it already exists: + +```csharp +using ZoneTree; + +using var tree = new ZoneTreeFactory() + .SetDataDirectory("data/atomic-example") + .OpenOrCreate(); +using var maintainer = tree.CreateMaintainer(); + +tree.TryAtomicAddOrUpdate( + key: "stats:user:42:views", + valueToAdd: 1, + valueUpdater: (ref long value) => + { + value++; + return true; + }, + result: (in long value, long opIndex, OperationResult result) => + { + Console.WriteLine($"{result}: {value} at operation {opIndex}"); + }); +``` + +Run it again against the same directory to increment the persisted counter. The callback reports `Added` on initialization and `Updated` on a later increment. + +## Understand the coordination boundary + +The newest value may be in mutable memory, a read-only segment, or a disk layer. Atomic methods synchronize this sequence across those layers: + +1. Find the current live value for the key. +2. Decide whether to add, replace, update, or cancel. +3. Write the accepted value to the mutable segment. + +The coordination lock is shared by atomic methods on the tree. It is not a separate lock per key, so long-running callbacks can delay unrelated atomic work in that tree. + +Use ordinary `Upsert` when the new value is independent of the old value and does not participate in an atomic invariant. For an unconditional replacement of a key in an atomic workflow, use `AtomicUpsert`: + +```csharp +tree.AtomicUpsert("stats:user:42:views", 1); +``` + +## Choose an atomic method + +| Method | When to use it | Result | +| --- | --- | --- | +| `TryAtomicAdd(key, value, out opIndex)` | Add only if no live value exists. | `true` when added; otherwise `false` and operation index `0`. | +| `TryAtomicUpdate(key, value, out opIndex)` | Replace only if a live value exists. | `true` when replaced; otherwise `false` and operation index `0`. | +| `AtomicUpsert(key, value)` | Add or replace while respecting atomic-method coordination. | The assigned operation index. | +| `TryAtomicGetAndUpdate(key, out value, updater)` | Decide on a replacement from the current value. | Whether a live key was found, even if the updater cancels. | +| `TryAtomicAddOrUpdate(key, valueToAdd, updater, result)` | Initialize when absent and update when present. | `true` only when added; use the callback to distinguish update from cancellation. | +| `TryAtomicAddOrUpdate(key, adder, updater, result)` | Compute both the initial value and later updates. | Either delegate can cancel by returning `false`. | + +Successful writes receive normal operation indexes. See [operation indexes](../concepts/op-index.md) for ordering and replay uses. + +## Compare before replacing + +This example advances a stored revision only when the current revision is `3`: + +```csharp +tree.AtomicUpsert("job:42:revision", 3); + +var replaced = false; +var found = tree.TryAtomicGetAndUpdate( + "job:42:revision", + out var revision, + (ref long value) => + { + if (value != 3) + return false; + + value = 4; + replaced = true; + return true; + }); + +Console.WriteLine($"Found: {found}; replaced: {replaced}"); +``` + +`found` answers whether the key existed. It does not mean the update was accepted. In this example, the local `replaced` flag records that separate decision. + +## Interpret cancellation and callback results + +An updater or adder receives a local `TValue` variable by `ref`. Return `true` to write that value, or `false` to cancel. + +| Callback result | Meaning | Operation index | +| --- | --- | --- | +| `Added` | A previously absent key was initialized. | Assigned to the write. | +| `Updated` | The existing value was replaced. | Assigned to the write. | +| `Cancelled` | The adder or updater declined the write. | `0`. | + +`TryAtomicAddOrUpdate` returns `false` for both `Updated` and `Cancelled`. Do not treat that boolean as a general success flag. + +Keep callbacks short. Adder/updater delegates in `TryAtomicAddOrUpdate` can run more than once if a segment freezes or fills before the write is accepted. Avoid network calls, external writes, and other effects that cannot safely repeat. The result callback also runs under atomic-method coordination. + +## Replace mutable objects instead of changing them in place + +If `TValue` is a reference type, copying the local variable does not clone the object. A property change can alter the shared object before ZoneTree accepts a write. Returning `false` cannot undo that mutation. + +Prefer immutable values. For a record value, construct a replacement with a `with` expression and assign it to the `ref` parameter. Make the decision before creating the replacement, and write through the API so operation indexing and WAL behavior remain explicit. See [value mutability](../concepts/value-mutability.md). + +## Know when to use a transaction + +A one-key atomic update does not commit related changes to other keys. Use a [transaction](transactions.mdx) for an invariant such as transferring value between two keys or updating a record and its index entries in the same transactional tree. + +Different key groups can use different disciplines. A profile key can use independent replacement writes while a separate counter uses atomic methods. The important boundary is every writer that can change the same invariant. + + + Need more than one key? +

+ Transactions add a commit boundary, conflict handling, and explicit retry + decisions for related keys in one tree. +

+ TransactionsOrdinary writes + diff --git a/zonetree-dev/content/docs/usage/index.mdx b/zonetree-dev/content/docs/usage/index.mdx new file mode 100644 index 0000000..92ed01a --- /dev/null +++ b/zonetree-dev/content/docs/usage/index.mdx @@ -0,0 +1,102 @@ +--- +title: Usage +description: Choose a ZoneTree API by coordination scope, and find guides for initialization, reads and writes, atomic operations, transactions, ordered scans, and maintenance. +layout: + fullWidth: true + showToc: false +--- + +
+ + + + Independent writes +

+ Use the ordinary API when a new value does not depend on the current + value. +

+ One-key decisions +

+ Use atomic methods when the current value determines the accepted + update. +

+ Related keys +

+ Use a transaction when several keys in one tree must share a commit + boundary. +

+ + + + + + + + + + + + + New to ZoneTree? +

+ Run the complete first-tree example before deciding which API your + application needs. +

+ Getting started + +
diff --git a/zonetree-dev/content/docs/usage/iteration-and-range-scans.md b/zonetree-dev/content/docs/usage/iteration-and-range-scans.md deleted file mode 100644 index 4ad823e..0000000 --- a/zonetree-dev/content/docs/usage/iteration-and-range-scans.md +++ /dev/null @@ -1,158 +0,0 @@ -# Iteration And Range Scans - -ZoneTree stores keys in comparer order. Iterators expose that order for full -scans, range scans, prefix layouts, and reverse reads. - -## Forward Iteration - -```csharp -using var iterator = zoneTree.CreateIterator(); - -while (iterator.Next()) -{ - Console.WriteLine($"{iterator.CurrentKey}: {iterator.CurrentValue}"); -} -``` - -## Seek - -Use `Seek` to position a forward iterator at the target when it exists, or at -the following key in configured comparer order when it does not. - -With an ascending integer comparer, this example scans the half-open range -`[100, 200)`: - -```csharp -using var iterator = zoneTree.CreateIterator(); - -iterator.Seek(100); - -while (iterator.Next()) -{ - if (iterator.CurrentKey >= 200) - break; - - Console.WriteLine($"{iterator.CurrentKey}: {iterator.CurrentValue}"); -} -``` - -The range boundaries follow the configured comparer. For structured keys, use -lower and upper keys that represent the intended range in that ordering. - -## Reverse Iteration - -Create a reverse iterator to scan the configured comparer order backward: - -```csharp -using var iterator = zoneTree.CreateReverseIterator(); - -while (iterator.Next()) -{ - Console.WriteLine($"{iterator.CurrentKey}: {iterator.CurrentValue}"); -} -``` - -Use `Seek` to position a reverse iterator at the target when it exists, or at -the preceding key in configured comparer order when it does not. With an -ascending integer comparer, this example scans from `1000` down through `900`: - -```csharp -using var iterator = zoneTree.CreateReverseIterator(); - -iterator.Seek(1000); - -while (iterator.Next()) -{ - if (iterator.CurrentKey < 900) - break; - - Console.WriteLine($"{iterator.CurrentKey}: {iterator.CurrentValue}"); -} -``` - -Reverse iteration is useful for latest-first time-series queries, descending -indexes, and high-key ranges. - -## Iterator Types - -ZoneTree iterators support different refresh and visibility behavior. - -| Type | Behavior | -| --- | --- | -| `AutoRefresh` | Default. Scans all available segments and refreshes when the mutable segment moves forward. Later writes may appear if their key position has not already been passed. | -| `NoRefresh` | Does not automatically include segments created by a later mutable-segment move. It still includes the current mutable segment and may observe later writes whose key position has not been passed. Call `Refresh()` to collect the latest segments manually. | -| `Snapshot` | Moves the mutable segment forward when the iterator is created, then scans the resulting read-only region and persistent segments. It does not see later writes. | -| `ReadOnlyRegion` | Like `Snapshot`, but does not move the mutable segment forward. It scans only data already in the read-only region and persistent segments. | - -Use the default iterator for ordinary scans. Use `Snapshot` when the scan needs -a stable view containing all writes visible when the iterator is created: - -```csharp -using var iterator = zoneTree.CreateIterator(IteratorType.Snapshot); -``` - -Use `ReadOnlyRegion` when the scan should exclude records still in the active -mutable segment and creating the iterator must not move that segment forward. - -## Deleted Records - -By default, iterators return live records and hide deletion markers. Include -deleted records for inspection, diagnostics, backup, restore, or -replication workflows: - -```csharp -using var iterator = zoneTree.CreateIterator( - IteratorType.NoRefresh, - includeDeletedRecords: true); -``` - -## Block Cache Contribution - -Disk-segment iterator reads do not contribute to the block cache by default. -This avoids retaining the decompressed blocks touched by a scan after the -iterator moves on. - -Enable cache contribution explicitly when the iterator should add decompressed -blocks to the shared block cache: - -```csharp -using var iterator = zoneTree.CreateIterator( - IteratorType.AutoRefresh, - includeDeletedRecords: false, - contributeToTheBlockCache: true); -``` - -Contributing retains the scanned blocks in memory until inactive block-cache -cleanup releases them. - -## Disk Segment Prefetch - -`DiskSegmentPrefetchSize` controls how many consecutive disk records an -iterator may read as one batch. The default is `0`; values below `2` disable -prefetching. - -```csharp -using var iterator = zoneTree.CreateIterator(new IteratorOptions -{ - IteratorType = IteratorType.AutoRefresh, - DiskSegmentPrefetchSize = 16 -}); -``` - -Prefetching can reduce per-record overhead during disk scans and may read beyond -an early range termination. - -## Dispose Iterators - -Iterators retain the segments they scan. Always dispose an iterator when its -scan is complete: - -```csharp -using var iterator = zoneTree.CreateIterator(); -``` - -Long-lived iterators can delay physical deletion of segments replaced by a -merge. Keep each iterator scoped to the scan that uses it. - -See [read-path caching](../tuning/read-path-caching.md) for prefetch and block -cache tuning. diff --git a/zonetree-dev/content/docs/usage/iteration-and-range-scans.mdx b/zonetree-dev/content/docs/usage/iteration-and-range-scans.mdx new file mode 100644 index 0000000..7d53926 --- /dev/null +++ b/zonetree-dev/content/docs/usage/iteration-and-range-scans.mdx @@ -0,0 +1,174 @@ +--- +title: Iteration And Range Scans +description: Seek and scan ordered key ranges, iterate in reverse, choose snapshot visibility, and configure disk prefetch and block-cache contribution. +--- + + + +## Scan an ordered range + +This complete program inserts records out of order and scans the half-open interval `[10, 30)`. With the default ascending integer comparer, it prints keys `10` and `20`: + +```csharp +using ZoneTree; + +using var tree = new ZoneTreeFactory() + .SetDataDirectory("data/iteration-example") + .OpenOrCreate(); +using var maintainer = tree.CreateMaintainer(); + +tree.Upsert(30, "Linus"); +tree.Upsert(10, "Ada"); +tree.Upsert(20, "Grace"); + +using var iterator = tree.CreateIterator(); +iterator.Seek(10); + +while (iterator.Next()) +{ + if (iterator.CurrentKey >= 30) + break; + + Console.WriteLine($"{iterator.CurrentKey}: {iterator.CurrentValue}"); +} +``` + + +

+ After Seek, call Next() before reading + CurrentKey or CurrentValue. A range scan needs its + own upper-bound check; the iterator does not stop at an application-defined + endpoint automatically. +

+ + +## Read the whole tree + +Create an iterator without seeking to start at the beginning of comparer order: + +```csharp +using var iterator = tree.CreateIterator(); +while (iterator.Next()) + Console.WriteLine($"{iterator.CurrentKey}: {iterator.CurrentValue}"); +``` + +The iterator merges visible records across LSM layers. Ordinary iteration returns the newest live value for a key and hides records considered deleted. + +## Seek forward or backward + +| Direction | `Seek(target)` positions the next read at | Ascending integer example | +| --- | --- | --- | +| Forward | The target key, or the next key in comparer order. | `Seek(15)` reads `20` first when keys are `10`, `20`, `30`. | +| Reverse | The target key, or the preceding key in comparer order. | `Seek(25)` reads `20` first for those same keys. | + +All boundaries are defined by your comparer, rather than by a universal numeric or string order. For composite keys, construct lower and upper keys that enclose the intended range. See [key ordering](../concepts/key-ordering.md) before designing prefix or time-series layouts. + +## Scan in reverse + +Use `CreateReverseIterator()` and keep calling `Next()`. Direction belongs to the iterator, not to a different movement method: + +```csharp +using var iterator = tree.CreateReverseIterator(); +iterator.Seek(30); + +while (iterator.Next()) +{ + if (iterator.CurrentKey < 10) + break; + + Console.WriteLine($"{iterator.CurrentKey}: {iterator.CurrentValue}"); +} +``` + +For the sample data, this returns `30`, `20`, then `10`. Reverse scans are useful when the latest timestamp or highest key should appear first. + +## Choose visibility under concurrent writes + +| Iterator type | Includes the active mutable segment? | What later writes can do | +| --- | --- | --- | +| `AutoRefresh` | Yes. This is the default. | Later writes may appear if their key positions have not been passed. Segment movement refreshes the iterator's sources. | +| `NoRefresh` | Yes. | Later writes in a collected mutable segment may appear. New segments are not collected automatically; call `Refresh()` explicitly when needed. | +| `Snapshot` | Moves it forward first, then scans the resulting read-only region and disk layers. | Later writes are excluded. | +| `ReadOnlyRegion` | No; it does not move the mutable segment forward. | Later writes are excluded, as are records still only in the active mutable segment at creation. | + +Use a snapshot for a scan that must exclude later writes: + +```csharp +using var iterator = tree.CreateIterator(IteratorType.Snapshot); +while (iterator.Next()) + Console.WriteLine($"{iterator.CurrentKey}: {iterator.CurrentValue}"); +``` + +Creating a snapshot moves the mutable segment into the read-only layer. Repeated snapshot creation under write load can increase segment movement and maintenance work. `NoRefresh` is not a snapshot: it can still see changes in a mutable segment it already holds, and initially collects its sources on the first seek, read, or explicit `Refresh()`. + +The table describes an iterator without a later manual refresh. Calling `Refresh()` recollects sources; even a snapshot iterator can then include newer read-only segments. Snapshot iteration also does not clone mutable reference values or freeze a time-dependent deletion predicate. Keep stored values immutable and account for custom deletion rules when defining a stable application-level view. + +## Control disk-scan memory and prefetch + +The default iterator does not contribute disk reads to the shared block cache. That avoids retaining each decompressed block touched by a one-off scan. + +Use `IteratorOptions` when the scan should configure both visibility and disk access: + +```csharp +using var iterator = tree.CreateIterator(new IteratorOptions +{ + IteratorType = IteratorType.AutoRefresh, + ContributeToTheBlockCache = false, + DiskSegmentPrefetchSize = 16 +}); + +while (iterator.Next()) + Console.WriteLine(iterator.CurrentKey); +``` + +`DiskSegmentPrefetchSize` defaults to `0`; values below `2` disable it. Prefetch batches consecutive disk records and can reduce per-record overhead. It can also read beyond a range where your loop stops early. + +Set `ContributeToTheBlockCache = true` when later reads are expected to reuse scanned blocks. Those blocks remain eligible for retention until inactive-cache cleanup releases them. Measure the tradeoff with your workload; see [read-path caching](../tuning/read-path-caching.md). + +## Include deletion markers when inspecting storage + +For diagnostics or a workflow that explicitly understands deletion markers: + +```csharp +using var iterator = tree.CreateIterator( + IteratorType.NoRefresh, + includeDeletedRecords: true); +``` + +This changes record visibility; it does not turn an ordinary scan into a complete backup or replication protocol. Those workflows also need their own ordering, metadata, and lifecycle rules. See [backups](../durability/backups.md) and [partitioning and replication](../building-systems/partitioning-and-replication.md). + +## Dispose at the end of the scan + +An iterator retains the segments it reads. A long-lived iterator can delay deletion of files replaced by compaction and keep read resources alive. + +Use a local `using` scope, finish the scan, and dispose before shutting down the tree. Create a new iterator for a later scan rather than leaving an old one open between unrelated requests. + + + Make ordered access part of the data model +

+ The comparer and key layout determine which queries can seek directly to a + useful range. +

+ Key orderingSecondary indexes + diff --git a/zonetree-dev/content/docs/usage/maintenance.md b/zonetree-dev/content/docs/usage/maintenance.md deleted file mode 100644 index 280a83c..0000000 --- a/zonetree-dev/content/docs/usage/maintenance.md +++ /dev/null @@ -1,279 +0,0 @@ -# Maintenance - -ZoneTree exposes two maintenance layers: - -* `CreateMaintainer()` returns an optional, ready-to-use maintenance - coordinator. -* `zoneTree.Maintenance` exposes segment state, counters, lifecycle events, and - operations for custom maintenance policy. - -ZoneTree can operate without a maintainer. Without one, the maintainer's -event-driven merge policy, merge-thread tracking, periodic cache cleanup, and -disposal coordination do not run. Omitting the maintainer transfers -responsibility for maintenance policy and coordination to the caller. -`zoneTree.Maintenance` is the supported surface for that custom implementation, -exposing the required state, counters, events, and operations. - -## Create A Maintainer - -```csharp -using var zoneTree = new ZoneTreeFactory() - .SetDataDirectory("data/app") - .OpenOrCreate(); - -using var maintainer = zoneTree.CreateMaintainer(); -``` - -The maintainer: - -* subscribes to ZoneTree lifecycle events, -* starts normal merges according to configurable thresholds, -* retries or schedules follow-up merges for retryable merge results, -* tracks normal and bottom merge threads for waiting and cancellation, -* periodically releases inactive read buffers and circular-cache entries, -* provides explicit merge, bottom-merge, and eviction methods, -* waits for its tracked merge threads during disposal. - -The returned maintainer is caller-owned and disposable. Disposal waits for its -tracked merge threads to finish. - -## Merge Triggers - -The default maintainer starts a normal merge after the mutable segment moves -forward and either configured limit is exceeded: - -| Setting | Default | Meaning | -| --- | ---: | --- | -| `ThresholdForMergeOperationStart` | `0` records | merge when read-only records exceed this count | -| `MaximumReadOnlySegmentCount` | `64` | merge when read-only segment count exceeds this count | - -With the default threshold of `0`, any non-empty read-only layer can start a -merge after segment movement. - -```csharp -maintainer.ThresholdForMergeOperationStart = 500_000; -maintainer.MaximumReadOnlySegmentCount = 32; -``` - -Use a higher record threshold when you want larger merge batches. Use a lower -read-only segment count limit when memory pressure should trigger merge work -sooner. - -## Evict Current Data - -`EvictToDisk()` moves the current mutable segment forward and starts a normal -merge. - -```csharp -maintainer.EvictToDisk(); -maintainer.WaitForBackgroundThreads(); -``` - -Use it before controlled shutdowns, exports, or explicit maintenance points when -you want the current in-memory records to enter the merge pipeline immediately. -Call `WaitForBackgroundThreads()` when the caller needs the merge to finish -before continuing. - -## Waiting And Cancellation - -The maintainer tracks the merge threads it starts. - -```csharp -maintainer.WaitForBackgroundThreads(); -``` - -For async callers: - -```csharp -await maintainer.WaitForBackgroundThreadsAsync(); -``` - -To request a faster shutdown: - -```csharp -maintainer.TryCancelBackgroundThreads(); -maintainer.WaitForBackgroundThreads(); -``` - -`TryCancelBackgroundThreads()` asks active normal and bottom-segment merges to -cancel. The merge threads finish when they observe the cancellation request. - -`Dispose()` waits for tracked merge threads, stops periodic cleanup, and detaches -event handlers. Call `WaitForBackgroundThreads()` when later code must observe -merge completion before disposal. Call `TryCancelBackgroundThreads()` before -disposal to request cancellation of current merge work. - -## Cache Cleanup - -The default maintainer starts inactive cache cleanup automatically. - -| Setting | Default | -| --- | ---: | -| `EnableJobForCleaningInactiveCaches` | `true` | -| `BlockCacheLifeTime` | `1 minute` | -| `InactiveBlockCacheCleanupInterval` | `30 seconds` | - -```csharp -maintainer.BlockCacheLifeTime = TimeSpan.FromMinutes(2); -maintainer.InactiveBlockCacheCleanupInterval = TimeSpan.FromSeconds(30); -``` - -Longer cache lifetime can help repeated disk reads. Shorter cache lifetime -reduces retained read-cache memory. The cleanup job releases inactive -decompressed blocks and expired circular key/value cache records. - -For read-cache details, see [read-path caching](../tuning/read-path-caching.md). - -## Bottom Segment Merge - -Bottom segment merge is an explicit operation. Run it when your service wants to -compact a range of bottom segments. - -```csharp -maintainer.StartBottomSegmentsMerge(); -maintainer.WaitForBackgroundThreads(); -``` - -To merge a selected range: - -```csharp -maintainer.StartBottomSegmentsMerge(fromIndex: 0, toIndex: 4); -maintainer.WaitForBackgroundThreads(); -``` - -The range uses the current bottom segment order. A broad range such as -`0..int.MaxValue` asks ZoneTree to merge as much of the bottom layer as possible. - -## Custom Maintenance Control - -`zoneTree.Maintenance` exposes the state, lifecycle events, and lower-level -operations used to build custom maintenance policy. Use it to integrate -maintenance with a scheduler, monitoring system, or storage orchestrator. - -```csharp -zoneTree.Maintenance.MoveMutableSegmentForward(); - -var thread = zoneTree.Maintenance.StartMergeOperation(); -thread?.Join(); -``` - -Useful direct operations: - -| Operation | Purpose | -| --- | --- | -| `MoveMutableSegmentForward()` | move the current mutable segment into the read-only layer | -| `StartMergeOperation()` | start a normal merge thread | -| `StartBottomSegmentsMergeOperation(fromIndex, toIndex)` | start a bottom-segment merge thread | -| `TryCancelMergeOperation()` | request cancellation for the active normal merge | -| `TryCancelBottomSegmentsMergeOperation()` | request cancellation for the active bottom-segment merge | -| `SaveMetaData()` | refresh the JSON metadata file and clear pending metadata records | -| `ReleaseReadBuffers(ticks)` | release inactive decompressed disk blocks | -| `ReleaseCircularKeyCacheRecords()` | release expired key-cache records | -| `ReleaseCircularValueCacheRecords()` | release expired value-cache records | - -`StartMergeOperation()` and `StartBottomSegmentsMergeOperation(...)` return the -created thread, or `null` when a merge of the same kind is already active. - -## Merge Results - -Merge completion is reported through maintenance events. - -```csharp -zoneTree.Maintenance.OnMergeOperationEnded += (_, result) => -{ - Console.WriteLine(result); -}; -``` - -| Result | Meaning | -| --- | --- | -| `SUCCESS` | merge completed | -| `NOTHING_TO_MERGE` | no eligible read-only segments were available | -| `ANOTHER_MERGE_IS_RUNNING` | a merge of the same kind was already active | -| `RETRY_READONLY_SEGMENTS_ARE_NOT_READY` | read-only segments were still preparing | -| `CANCELLED_BY_USER` | cancellation was requested | -| `FAILURE` | an exception occurred; inspect the logger | - -The default maintainer retries `RETRY_READONLY_SEGMENTS_ARE_NOT_READY`. If it sees -`ANOTHER_MERGE_IS_RUNNING`, it starts another merge after the active merge -finishes. - -## Counters - -Maintenance counters are useful for dashboards and health checks. - -| Counter | Meaning | -| --- | --- | -| `MutableSegmentRecordCount` | records in the current mutable segment | -| `ReadOnlySegmentsCount` | read-only in-memory segment count | -| `ReadOnlySegmentsRecordCount` | records across read-only in-memory segments | -| `InMemoryRecordCount` | mutable plus read-only record count | -| `TotalRecordCount` | physical records across memory and disk layers | -| `IsMerging` | normal merge is active | -| `IsBottomSegmentsMerging` | bottom-segment merge is active | - -`TotalRecordCount` is a physical storage counter. Use `Count()` or -`CountFullScan()` for live-record counts. - -## Events - -Use events for monitoring, scheduling, and cleanup reporting. - -| Event | Use | -| --- | --- | -| `OnMutableSegmentMovedForward` | observe mutable segment movement | -| `OnMergeOperationStarted` | observe normal merge start | -| `OnMergeOperationEnded` | observe normal merge result | -| `OnBottomSegmentsMergeOperationStarted` | observe bottom merge start | -| `OnBottomSegmentsMergeOperationEnded` | observe bottom merge result | -| `OnDiskSegmentCreated` | observe created disk segment files | -| `OnDiskSegmentActivated` | observe the active disk segment change | -| `OnCanNotDropReadOnlySegment` | report cleanup failure for read-only segment files | -| `OnCanNotDropDiskSegment` | report cleanup failure for disk segment files | -| `OnCanNotDropDiskSegmentCreator` | report cleanup failure for unfinished merge output | - -Failed drop events mean obsolete files or temporary output stayed behind after a -cleanup attempt failed. Log the exception and investigate the file-system or -provider error. - -## Iterator Lifetime - -Dispose iterators as soon as scans finish. Long-lived iterators can keep segments -alive, which delays cleanup of old segment files and read buffers. - -Snapshot iterators move the mutable segment forward when they are created. Heavy -snapshot-iterator usage under write load can increase read-only segment pressure. - -## Practical Patterns - -For the default maintenance model: - -```csharp -using var zoneTree = new ZoneTreeFactory() - .SetDataDirectory("data/app") - .OpenOrCreate(); - -using var maintainer = zoneTree.CreateMaintainer(); - -// run application -``` - -For a controlled checkpoint: - -```csharp -maintainer.EvictToDisk(); -maintainer.WaitForBackgroundThreads(); -zoneTree.Maintenance.SaveMetaData(); -``` - -For a custom maintenance window: - -```csharp -maintainer.StartMerge(); -maintainer.StartBottomSegmentsMerge(); -maintainer.WaitForBackgroundThreads(); -``` - -For related tuning, see [memory usage](../storage/memory-usage.md), -[disk segment tuning](../tuning/disk-segments.md), and -[read-path caching](../tuning/read-path-caching.md). diff --git a/zonetree-dev/content/docs/usage/maintenance.mdx b/zonetree-dev/content/docs/usage/maintenance.mdx new file mode 100644 index 0000000..b7eab28 --- /dev/null +++ b/zonetree-dev/content/docs/usage/maintenance.mdx @@ -0,0 +1,308 @@ +--- +title: Maintenance +description: Run ZoneTree's standard maintainer, control merges and cache cleanup, wait for background work, and inspect lifecycle events and storage counters. +--- + + + +## Start with a maintainer + +Create one maintainer alongside the tree. This complete program writes a record, requests a merge of current in-memory data, and waits for tracked background work: + +```csharp +using ZoneTree; + +using var tree = new ZoneTreeFactory() + .SetDataDirectory("data/maintenance-example") + .OpenOrCreate(); +using var maintainer = tree.CreateMaintainer(); + +tree.Upsert(42, "Ada"); + +maintainer.EvictToDisk(); +await maintainer.WaitForBackgroundThreadsAsync(); + +Console.WriteLine($"Read-only segments: {tree.Maintenance.ReadOnlySegmentsCount}"); +``` + +For normal service operation, keep the tree and maintainer alive and let the event-driven policy schedule merges. Explicit eviction and waiting are useful at controlled maintenance points; they are not needed after every write. + + +

+ ZoneTree can operate without CreateMaintainer(). Without it, + you own merge scheduling, retries, cache cleanup, and background-work + coordination through tree.Maintenance. Keeping the default + maintainer is the starting point for applications that do not need a custom + policy. +

+ + +## Know which layer to use + +| Surface | Responsibility | +| --- | --- | +| `tree.CreateMaintainer()` | Standard merge policy, merge-thread tracking, periodic cache cleanup, and disposal coordination. Returns a caller-owned `IMaintainer`. | +| `tree.Maintenance` | Segment state, counters, lifecycle events, and lower-level operations for custom policy. | + +The maintainer schedules normal merges when mutable segments move forward. Bottom-segment compaction remains an explicit operation. Read [LSM-tree architecture](../concepts/lsm-tree.mdx) for the distinction between mutable, read-only, disk, and bottom layers. + +## Tune merge triggers + +After the mutable segment moves forward, the default maintainer starts a normal merge when either limit is exceeded: + +| Setting | Default | Trigger | +| --- | ---: | --- | +| `ThresholdForMergeOperationStart` | `0` | Read-only record count exceeds the threshold. | +| `MaximumReadOnlySegmentCount` | `64` | Read-only segment count exceeds the limit. | + +With the default record threshold, any non-empty read-only layer can trigger a merge after movement. To collect larger batches or bound the number of queued segments: + +```csharp +maintainer.ThresholdForMergeOperationStart = 500_000; +maintainer.MaximumReadOnlySegmentCount = 32; +``` + +Higher thresholds can reduce the frequency of small merges, but allow more records to remain in memory. Choose them alongside record size, write rate, and available memory; a record count is not a byte budget. + +## Request a merge and wait for it + +`EvictToDisk()` moves the active mutable segment forward and starts a normal merge: + +```csharp +maintainer.EvictToDisk(); +maintainer.WaitForBackgroundThreads(); +``` + +`StartMerge()` requests a normal merge without explicitly moving the current mutable segment. `WaitForBackgroundThreads()` waits for the merge threads tracked by this maintainer; it does not stop application writers or create an application-wide checkpoint by itself. + +For asynchronous code, use `await maintainer.WaitForBackgroundThreadsAsync()`. + + +

+ Eviction reorganizes storage. It does not replace choosing WAL and + failure-recovery guarantees. To establish a controlled maintenance boundary, + coordinate application writers as well as background threads. See + WAL modes and + live backups for those separate + workflows. +

+ + +## Compact bottom segments + +Run a bottom merge when your policy wants to compact persistent bottom-layer segments: + +```csharp +maintainer.StartBottomSegmentsMerge(); +maintainer.WaitForBackgroundThreads(); +``` + +To request a selected range in the current bottom-segment order: + +```csharp +maintainer.StartBottomSegmentsMerge(fromIndex: 0, toIndex: 4); +maintainer.WaitForBackgroundThreads(); +``` + +The default bounds are `0` and `int.MaxValue`, asking the engine to merge as much of the current bottom layer as possible. The indexes refer to segment order, not to record keys. Schedule this work around the CPU, I/O, and latency budget of your service. + +## Control inactive-cache cleanup + +The standard maintainer starts periodic cleanup automatically: + +| Setting | Default | Effect | +| --- | --- | --- | +| `EnableJobForCleaningInactiveCaches` | `true` | Enables the periodic cleanup job. | +| `BlockCacheLifeTime` | `1 minute` | Controls how long inactive decompressed disk blocks remain eligible for retention. | +| `InactiveBlockCacheCleanupInterval` | `30 seconds` | Sets the timer period when the cleanup job starts. Restart the job to apply a changed interval. | + +```csharp +maintainer.EnableJobForCleaningInactiveCaches = false; +maintainer.BlockCacheLifeTime = TimeSpan.FromMinutes(2); +maintainer.InactiveBlockCacheCleanupInterval = TimeSpan.FromSeconds(15); +maintainer.EnableJobForCleaningInactiveCaches = true; +``` + +The cleanup interval is captured when the periodic timer is created; assigning the property alone does not reconfigure an already running timer. `BlockCacheLifeTime` is read on each cleanup pass. + +Longer retention can help repeated reads and also retain more memory. Cleanup releases inactive block buffers and expired circular key/value cache entries. The .NET runtime may retain released memory for reuse rather than immediately returning it to the operating system. + +See [memory usage](../storage/memory-usage.md) and [read-path caching](../tuning/read-path-caching.md) when interpreting process memory or changing the cache policy. + +## Observe progress and failures + +Subscribe to merge completion to record the outcome: + +```csharp +tree.Maintenance.OnMergeOperationEnded += (_, result) => +{ + Console.WriteLine($"Merge ended: {result}"); +}; +``` + +| Result | Interpretation | +| --- | --- | +| `SUCCESS` | The merge completed. | +| `NOTHING_TO_MERGE` | There were no eligible read-only segments. | +| `ANOTHER_MERGE_IS_RUNNING` | A merge of the same kind was already active. | +| `RETRY_READONLY_SEGMENTS_ARE_NOT_READY` | Read-only segments were still preparing. | +| `CANCELLED_BY_USER` | A cancellation request was observed. | +| `FAILURE` | The merge failed; inspect the configured logger. | + +The standard maintainer retries `RETRY_READONLY_SEGMENTS_ARE_NOT_READY`. After `ANOTHER_MERGE_IS_RUNNING`, it schedules follow-up work after the active normal merge finishes. + +Useful counters on `tree.Maintenance` include: + +| Counter | What it measures | +| --- | --- | +| `MutableSegmentRecordCount` | Records in the active mutable segment. | +| `ReadOnlySegmentsCount` | Queued read-only in-memory segments. | +| `ReadOnlySegmentsRecordCount` | Records across those read-only segments. | +| `InMemoryRecordCount` | Mutable and read-only records combined. | +| `TotalRecordCount` | Physical records across memory and disk layers. | +| `IsMerging` / `IsBottomSegmentsMerging` | Whether each merge kind is active. | + +Physical records can include multiple versions and deletion markers. Use `Count()` or `CountFullScan()` for live-record counts, with the cost and visibility requirements of your workload in mind. + +## Own shutdown explicitly + +Stop accepting application work and finish scans before closing storage. Short-lived iterators let old segments and files be reclaimed; long-lived iterators can pin them after a merge. + +Maintainer disposal waits for tracked merge threads, requests cancellation of periodic cleanup, and detaches event handlers. The wait methods join tracked merge threads; they do not join the cache-cleanup task or arbitrary application work. For a cancellation-first shutdown: + +```csharp +maintainer.TryCancelBackgroundThreads(); +maintainer.WaitForBackgroundThreads(); +``` + +Cancellation is a request. The threads still need to observe it and finish. After the wait, dispose the maintainer and then the tree. The `using` declaration order in the first example handles that disposal sequence. + +## Build a custom policy when needed + +Use `tree.Maintenance` when integrating with your own scheduler or storage orchestrator. A direct merge returns a thread, or `null` if a merge of the same kind is already active: + +```csharp +tree.Maintenance.MoveMutableSegmentForward(); +var mergeThread = tree.Maintenance.StartMergeOperation(); +mergeThread?.Join(); +``` + +A `null` return is not proof that maintenance is finished. Observe the existing work through the appropriate state and events. Directly started threads are outside the standard maintainer's thread tracking, so your custom policy must coordinate them. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OperationPurpose
MoveMutableSegmentForward()Freeze the current mutable segment into the read-only layer.
StartMergeOperation()Start a normal merge thread.
+ StartBottomSegmentsMergeOperation(fromIndex, toIndex) + Start a bottom-layer merge thread.
TryCancelMergeOperation()Request cancellation of the active normal merge.
TryCancelBottomSegmentsMergeOperation()Request cancellation of the active bottom merge.
SaveMetaData() + Refresh the JSON metadata file and clear pending metadata records. +
ReleaseReadBuffers(ticks)Release inactive decompressed disk blocks.
+ ReleaseCircularKeyCacheRecords() / + ReleaseCircularValueCacheRecords() + Release expired circular-cache entries.
+ + + +
    +
  • + OnMutableSegmentMovedForward: observe mutable-segment + movement. +
    • +
  • + OnMergeOperationStarted / OnMergeOperationEnded: + observe normal merges. +
    • +
  • + OnBottomSegmentsMergeOperationStarted / + OnBottomSegmentsMergeOperationEnded: observe bottom merges. +
    • +
  • + OnDiskSegmentCreated / OnDiskSegmentActivated: + observe disk-segment publication. +
    • +
  • + OnCanNotDropReadOnlySegment, + OnCanNotDropDiskSegment, and + OnCanNotDropDiskSegmentCreator: investigate failed cleanup of + obsolete files or unfinished merge output. +
    • +
+

+ For failed-drop events, log the exception and investigate the file-system or + storage-provider error. +

+ + + + Turn maintenance observations into an operating policy +

+ Use diagnostics to identify the bottleneck, then change thresholds or + storage settings against a measured workload. +

+ DiagnosticsProduction checklist + diff --git a/zonetree-dev/content/docs/usage/opening-a-tree.md b/zonetree-dev/content/docs/usage/opening-a-tree.md deleted file mode 100644 index f50e4c1..0000000 --- a/zonetree-dev/content/docs/usage/opening-a-tree.md +++ /dev/null @@ -1,79 +0,0 @@ -# Opening A Tree - -ZoneTree instances are created with `ZoneTreeFactory`. The factory collects configuration and opens or creates the underlying storage. - -## Open Or Create - -```csharp -using ZoneTree; - -using var zoneTree = new ZoneTreeFactory() - .SetDataDirectory("data/app") - .OpenOrCreate(); -``` - -Use `OpenOrCreate` for normal application startup when the database may or may not already exist. - -## Create - -```csharp -using var zoneTree = new ZoneTreeFactory() - .SetDataDirectory("data/app") - .Create(); -``` - -Use `Create` when a database must not already exist. - -## Open Existing - -```csharp -using var zoneTree = new ZoneTreeFactory() - .SetDataDirectory("data/app") - .Open(); -``` - -Use `Open` when an existing database is required. - -## Transactional Tree - -```csharp -using var zoneTree = new ZoneTreeFactory() - .SetDataDirectory("data/tx-app") - .OpenOrCreateTransactional(); -``` - -Transactional trees support coordinated multi-key operations. Use a non-transactional tree for the fastest simple read/write workloads. - -## Configure Before Opening - -Set serializers, comparers, WAL options, deletion delegates, and storage options before opening the tree. These choices define the storage format and behavior. - -```csharp -using ZoneTree; -using ZoneTree.Comparers; -using ZoneTree.Serializers; - -using var zoneTree = new ZoneTreeFactory() - .SetDataDirectory("data/app") - .SetComparer(new Int32ComparerAscending()) - .SetKeySerializer(new Int32Serializer()) - .SetValueSerializer(new Utf8StringSerializer()) - .SetMutableSegmentMaxItemCount(100_000) - .OpenOrCreate(); -``` - -ZoneTree fills serializers, comparers, and hashers for supported known types. -When Bloom filtering is enabled with a custom comparer, ensure comparer-equal -keys produce the same hash. - -## Dispose - -Always dispose ZoneTree instances when they are no longer needed. - -```csharp -using var zoneTree = new ZoneTreeFactory() - .SetDataDirectory("data/app") - .OpenOrCreate(); -``` - -For long-running services, keep the tree open for the service lifetime and dispose during shutdown. diff --git a/zonetree-dev/content/docs/usage/opening-a-tree.mdx b/zonetree-dev/content/docs/usage/opening-a-tree.mdx new file mode 100644 index 0000000..53e77ec --- /dev/null +++ b/zonetree-dev/content/docs/usage/opening-a-tree.mdx @@ -0,0 +1,132 @@ +--- +title: Opening A Tree +description: Choose OpenOrCreate, Open, or Create; configure a ZoneTreeFactory; and manage the lifetime of a tree, its maintainer, and its iterators. +--- + + + +## Start with the application lifetime + +This complete example opens a persistent tree and starts the standard maintenance coordinator: + +```csharp +using ZoneTree; + +using var tree = new ZoneTreeFactory() + .SetDataDirectory("data/opening-example") + .OpenOrCreate(); +using var maintainer = tree.CreateMaintainer(); + +tree.Upsert(42, "Ada"); +if (tree.TryGet(42, out var name)) + Console.WriteLine(name); +``` + +In a service, create the tree at startup and share it with the application code that needs it. Dispose it during shutdown after requests and scans have finished. Opening a new tree for every request repeats initialization and complicates resource ownership. + +## Choose an opening mode + +| Method | Directory contains a tree | No tree exists | Use it for | +| --- | --- | --- | --- | +| `OpenOrCreate()` | Opens it. | Creates it. | Applications that initialize their own storage. | +| `Open()` | Opens it. | Throws `DatabaseNotFoundException`. | Services or tools that must use an existing database. | +| `Create()` | Throws `DatabaseAlreadyExistsException`. | Creates it. | Explicit provisioning and fresh test databases. | + +Each method returns an opened `IZoneTree`. Configure the factory before calling it. + + +

+ Relative paths resolve from the application's working directory. For a + deployed service, use a stable configured path. Use Open() when + a missing or mistyped path should fail startup. +

+ + +## Configure before opening + +The factory fills supported serializers, comparers, and hashers for known types. Specify components explicitly when the stored representation or ordering needs to be controlled: + +```csharp +using ZoneTree; +using ZoneTree.Comparers; +using ZoneTree.Serializers; + +using var tree = new ZoneTreeFactory() + .SetDataDirectory("data/configured-example") + .SetComparer(new Int32ComparerAscending()) + .SetKeySerializer(new Int32Serializer()) + .SetValueSerializer(new Utf8StringSerializer()) + .SetMutableSegmentMaxItemCount(100_000) + .OpenOrCreate(); +using var maintainer = tree.CreateMaintainer(); +``` + +`MutableSegmentMaxItemCount` controls the number of records in the active mutable segment. It is not a byte limit, and repeated writes to an existing key do not each create another record in that segment. + +Treat serializers, key comparison, and deletion rules as part of the storage design. Reopening existing data with incompatible rules can make it unreadable or incorrectly ordered. Plan a migration when changing that design. + +| Decision | Read next | +| --- | --- | +| What determines equality and ordering? | [Key ordering](../concepts/key-ordering.md) | +| How are keys and values encoded? | [Serializers, comparers, and hashers](../concepts/serializers-and-comparers.md) | +| Which values represent deleted records? | [Deletion markers and TTL](../concepts/deletion-markers-and-ttl.md) | +| When are writes recoverable after a failure? | [WAL modes](../durability/wal-modes.md) | +| Which other settings can be configured? | [Configuration reference](../reference/configuration.md) | + +When Bloom filtering is enabled with a custom comparer, comparer-equal keys must produce the same hash. Supply a compatible key hasher with your comparer. + +## Open a transactional tree + +Use the transactional factory methods when several keys in the same tree must participate in a transaction: + +```csharp +using ZoneTree; + +using var tree = new ZoneTreeFactory() + .SetDataDirectory("data/transactional-example") + .OpenOrCreateTransactional(); +using var maintainer = tree.CreateMaintainer(); +``` + +`OpenTransactional()` requires existing data; `CreateTransactional()` requires that a tree does not already exist. Their opening rules match the non-transactional methods above. + +The returned `ITransactionalZoneTree` uses transaction IDs and commit handling. Choose it because an invariant spans multiple keys, rather than merely because several application threads will write. See [transactions](transactions.mdx) for a complete example. + +## Close resources in the right order + +| Resource | Lifetime and ownership | +| --- | --- | +| Tree | Application-owned. Keep it open while operations need it; dispose at shutdown. | +| Maintainer | Caller-owned. Keep it with the tree; disposal waits for tracked merge threads. | +| Iterator | Scan-owned. Dispose as soon as that scan finishes so replaced segments can be reclaimed. | + +The examples declare the tree first and the maintainer second. C# disposes them in reverse order, so maintenance finishes before the tree closes. A later iterator declaration is disposed before both. + +Creating a maintainer is optional. Omitting it means your application must supply the merge policy, cache cleanup, and background-work coordination through `tree.Maintenance`. + + + Continue with data operations +

+ Start with ordinary reads and writes, then add coordination only where the + data rule requires it. +

+ Reads and writesMaintenance + diff --git a/zonetree-dev/content/docs/usage/reads-and-writes.md b/zonetree-dev/content/docs/usage/reads-and-writes.md deleted file mode 100644 index a92ebd1..0000000 --- a/zonetree-dev/content/docs/usage/reads-and-writes.md +++ /dev/null @@ -1,81 +0,0 @@ -# Reads And Writes - -ZoneTree provides multiple write APIs because different operations need different coordination costs. Use the simplest API that matches the data rule you need. - -## API Choice - -| Need | Use | -| --- | --- | -| Set or replace a value | `Upsert` | -| Add only if the key does not exist | `TryAdd` | -| Delete a key if it exists | `TryDelete` | -| Write a deletion marker without checking existence | `ForceDelete` | -| Update based on the current value | atomic methods | -| Coordinate multiple keys | transactions | - -## Upsert - -`Upsert` is the normal high-throughput write path. - -```csharp -var opIndex = zoneTree.Upsert(42, "value"); -``` - -It adds the key if it does not exist or replaces the latest value if it does. - -## TryGet - -```csharp -if (zoneTree.TryGet(42, out var value)) -{ - Console.WriteLine(value); -} -``` - -`TryGet` searches the newest layers first and ignores values considered deleted by the configured deletion logic. - -Treat returned values as immutable snapshots. If `TValue` is a mutable reference type, do not mutate the returned object as a way to update the database. Create a new value and write it back. - -See [value mutability](../concepts/value-mutability.md). - -## TryAdd - -```csharp -if (zoneTree.TryAdd(42, "value", out var opIndex)) -{ - Console.WriteLine("added"); -} -``` - -Use `TryAdd` when duplicate keys should not be overwritten. - -## TryDelete And ForceDelete - -```csharp -zoneTree.TryDelete(42, out var opIndex); -zoneTree.ForceDelete(42); -``` - -`TryDelete` checks whether the key exists. `ForceDelete` writes a deletion marker directly. - -## Thread Safety - -Regular write APIs are designed for normal concurrent use. Atomic methods are not the "safe version" of regular writes; they are the synchronized read-modify-write tools. - -Use `Upsert` for simple concurrent writes. Use atomic methods only when the new value depends on the existing value. - -## Mixed Write Modes - -You can mix write modes in the same tree. For example: - -* ordinary records use fast `Upsert`, -* counter keys use atomic read-modify-write, -* multi-key changes use transactions. - -This lets each workflow pay only for the coordination it needs. - -## Value Shape - -Small immutable structs or readonly record structs are often a good fit for ZoneTree values. They reduce accidental shared mutation and can reduce GC pressure for in-memory segments. - -For larger or complex values, use immutable reference types, records, serialized payloads, or clone-and-upsert patterns. diff --git a/zonetree-dev/content/docs/usage/reads-and-writes.mdx b/zonetree-dev/content/docs/usage/reads-and-writes.mdx new file mode 100644 index 0000000..3117406 --- /dev/null +++ b/zonetree-dev/content/docs/usage/reads-and-writes.mdx @@ -0,0 +1,144 @@ +--- +title: Reads And Writes +description: Read, upsert, add, and delete ZoneTree records; interpret operation indexes; and choose atomic operations or transactions when writes need coordination. +--- + + + +## A complete read and write example + +This program uses an integer key and a string value. Run it in a console project with the `ZoneTree` package installed: + +```csharp +using ZoneTree; + +using var tree = new ZoneTreeFactory() + .SetDataDirectory("data/read-write-example") + .OpenOrCreate(); +using var maintainer = tree.CreateMaintainer(); + +var writtenAt = tree.Upsert(42, "Ada"); +if (tree.TryGet(42, out var name)) + Console.WriteLine($"42: {name}"); + +if (tree.TryAdd(7, "Grace", out var addedAt)) + Console.WriteLine($"Added key 7 at operation {addedAt}"); + +if (tree.TryDelete(7, out var deletedAt)) + Console.WriteLine($"Deleted key 7 at operation {deletedAt}"); +``` + +For setup and resource lifetime, see [opening a tree](opening-a-tree.mdx). + +## Choose an operation + +| Your data rule | API | Important detail | +| --- | --- | --- | +| Read the latest live value. | `TryGet` | Returns `false` when missing or considered deleted. | +| Insert or replace without consulting the old value. | `Upsert` | The ordinary write path for independent updates. | +| Check for absence, then add. | `TryAdd` | The check and write are not serialized with competing writers. | +| Check for existence, then delete. | `TryDelete` | Also uses a separate existence check and write. | +| Write a deletion marker without checking. | `ForceDelete` | Avoids the existence read, even if the key was already absent. | +| Enforce a conditional change against other atomic operations. | `TryAtomicAdd`, `TryAtomicUpdate`, or an updater method | Every competing write for that invariant must use the atomic path. | +| Coordinate changes to several keys in one tree. | Transactional API | Requires commit and conflict handling. | + +## Insert or replace with Upsert + +```csharp +var opIndex = tree.Upsert(42, "Updated name"); +``` + +The latest write becomes the value returned by subsequent ordinary reads. If the key already exists in an older LSM layer, the new record shadows it until maintenance merges the layers. + +`Upsert` returns the assigned operation index. Use that value when following the producer's write sequence; it is not a transaction ID or confirmation that the storage device has flushed the write. See [operation indexes](../concepts/op-index.md) and [WAL modes](../durability/wal-modes.md). + +## Read with TryGet + +```csharp +if (tree.TryGet(42, out var value)) + Console.WriteLine(value); +else + Console.WriteLine("Key not found"); +``` + +The read searches the newest layers first and applies the configured deletion logic. Check the boolean before using the output value. A separate `ContainsKey` call is unnecessary when you also need the value, and another writer can change the key between two calls. + + +

+ A mutable object can be shared with an in-memory segment. Changing its + properties is not a database update and does not produce a WAL record or + operation index. Construct a replacement value and write it through the + appropriate API. See + value mutability. +

+ + +## Add only when absent + +For a workflow without competing writers to the same key: + +```csharp +if (!tree.TryAdd(99, "New record", out var opIndex)) + Console.WriteLine("The key already existed at the existence check."); +``` + +If the key exists at that check, `TryAdd` returns `false` and sets the operation index to `0`. It does not protect the check-and-write sequence from another writer. + +When concurrent inserts must agree on a single winner, use `TryAtomicAdd` and keep competing writes on the atomic path: + +```csharp +if (tree.TryAtomicAdd(100, "Reserved", out var opIndex)) + Console.WriteLine("Reservation added."); +``` + +Ordinary `Upsert` does not participate in the atomic-method lock. See [atomic operations](atomic-operations.mdx) for the coordination rules. + +## Delete a record + +```csharp +var deleted = tree.TryDelete(42, out var deleteIndex); +var markerIndex = tree.ForceDelete(99); +``` + +`TryDelete` returns `false` and operation index `0` if its existence check finds no live key. `ForceDelete` writes a deletion marker directly. Ordinary reads and iterators hide records considered deleted. + +A deletion marker does not immediately remove old values from every segment. Maintenance eventually reclaims obsolete storage. For custom value types, configure compatible deletion delegates; see [deletion markers and TTL](../concepts/deletion-markers-and-ttl.md). + +## Concurrency and value design + +The ordinary APIs support concurrent use, but a sequence of calls is not a single atomic action. A `TryGet` followed by `Upsert` can lose an intervening update. Use [single-key atomic operations](atomic-operations.mdx) when the next value depends on the current value. + +Use [transactions](transactions.mdx) when the invariant spans several keys in one transactional tree. Choose immutable structs, records, or replacement objects so values are not modified through shared references. + +Different key groups can use different write disciplines. For example, profile replacement can use `Upsert` while a separate counter uses atomic methods. Keep the discipline consistent for every writer that can change the same invariant. + + + Choose the next operation +

+ Read a range, update one key based on its current value, or coordinate a + group of keys. +

+ Scan a rangeAtomic operationsTransactions + diff --git a/zonetree-dev/content/docs/usage/transactions.md b/zonetree-dev/content/docs/usage/transactions.md deleted file mode 100644 index de6f5d2..0000000 --- a/zonetree-dev/content/docs/usage/transactions.md +++ /dev/null @@ -1,130 +0,0 @@ -# Transactions - -Transactions coordinate changes across multiple keys. Use them when a workflow must read or write several keys as one logical unit. - -For simple high-throughput writes, prefer the non-transactional API. - -## Open A Transactional Tree - -```csharp -using ZoneTree; - -using var zoneTree = new ZoneTreeFactory() - .SetDataDirectory("data/tx") - .OpenOrCreateTransactional(); -``` - -## Basic Transaction - -```csharp -var tx = zoneTree.BeginTransaction(); - -zoneTree.Upsert(tx, 1, "first"); -zoneTree.Upsert(tx, 2, "second"); - -var result = zoneTree.PrepareAndCommit(tx); - -Console.WriteLine(result.Result); -``` - -`Prepare` and `PrepareAndCommit` report commit state: - -| Result | Meaning | -| --- | --- | -| `Committed` | the transaction committed | -| `ReadyToCommit` | `Prepare` succeeded and a separate `Commit` call can finish it | -| `PendingTransactions` | this transaction depends on uncommitted transactions | -| `Aborted` | the transaction was aborted | - -## When To Use Transactions - -Use transactions for: - -* moving value from one key to another, -* updating a primary record and secondary index together, -* ensuring multiple keys change together, -* checking multiple keys before writing. - -Do not use transactions merely because writes are concurrent. Regular writes are designed for concurrent workloads. - -## Atomic Methods Vs Transactions - -| Requirement | Use | -| --- | --- | -| Current value of one key decides the next value | atomic method | -| Multiple keys must commit together | transaction | -| Simple replace/add/delete | regular write API | - -## Keep Transactions Short - -Keep transaction work focused. Long-running transactions increase conflict windows and can make other transactions wait or retry. - -## No-Throw APIs - -Most transactional APIs have `NoThrow` variants. Read/write methods return `TransactionResult` or `TransactionResult` instead of throwing `TransactionAbortedException`; prepare and commit methods return `CommitResult`. - -```csharp -var tx = zoneTree.BeginTransaction(); - -var write = zoneTree.UpsertNoThrow(tx, 1, "value"); -if (write.IsAborted) -{ - return; -} - -var commit = zoneTree.PrepareAndCommitNoThrow(tx); -if (commit.IsPendingTransactions) -{ - Console.WriteLine("Waiting on: " + - string.Join(", ", commit.PendingTransactionList)); -} -``` - -## Fluent Transactions - -`BeginFluentTransaction` wraps retry handling for aborted and pending transactions. - -```csharp -using var tx = zoneTree.BeginFluentTransaction(); - -var result = await tx - .Do(id => zoneTree.UpsertNoThrow(id, 1, "first")) - .Do(id => zoneTree.UpsertNoThrow(id, 2, "second")) - .CommitAsync(); - -Console.WriteLine(result.Succeeded); -``` - -The fluent API is useful when retrying a short unit of work is easier than managing prepare, pending dependencies, and rollback directly. - -## Read-Committed Reads - -Transactional trees expose read-committed helpers for reading committed data without starting a new transaction. - -```csharp -if (zoneTree.ReadCommittedTryGet(1, out var value)) -{ - Console.WriteLine(value); -} -``` - -Pass the current transaction id when a transaction should see its own uncommitted writes and committed writes from others. - -## Auto-Commit Helpers - -`UpsertAutoCommit` and `DeleteAutoCommit` start a transaction, perform one write, and commit it. They are convenient when all writes must pass through the transaction log, but they are heavier than regular `Upsert` and `ForceDelete`. - -## Transaction Maintenance - -Transactional maintenance exposes the underlying tree and transaction log: - -* `zoneTree.Maintenance.ZoneTree`, -* `zoneTree.Maintenance.TransactionLog`, -* `zoneTree.Maintenance.UncommittedTransactionIds`. - -Roll back abandoned transactions during operational cleanup: - -```csharp -zoneTree.Maintenance.RollbackUncommittedTransactionIdsBefore( - DateTime.UtcNow.AddMinutes(-10)); -``` diff --git a/zonetree-dev/content/docs/usage/transactions.mdx b/zonetree-dev/content/docs/usage/transactions.mdx new file mode 100644 index 0000000..c108339 --- /dev/null +++ b/zonetree-dev/content/docs/usage/transactions.mdx @@ -0,0 +1,188 @@ +--- +title: Transactions +description: Open a transactional ZoneTree, commit related writes, handle aborts and pending dependencies, and use fluent retries or explicit transaction control. +--- + + + + +

+ Transactions coordinate keys inside one ITransactionalZoneTree. + They do not atomically commit changes to separate ZoneTree instances, + another database, or an external service. Related index entries must be in + the same transactional tree to share its commit boundary. +

+ + +## Start with a fluent transaction + +This complete console example writes two related keys. It checks the returned result and handles a transaction-abort exception from the pending-wait path: + +```csharp +using ZoneTree; +using ZoneTree.Exceptions; + +using var tree = new ZoneTreeFactory() + .SetDataDirectory("data/transactions-example") + .OpenOrCreateTransactional(); +using var maintainer = tree.CreateMaintainer(); + +try +{ + using var transaction = tree.BeginFluentTransaction(); + var result = await transaction + .Do(id => tree.UpsertNoThrow(id, "order:42:state", "confirmed")) + .Do(id => tree.UpsertNoThrow(id, "order:42:receipt", "receipt-42")) + .CommitAsync(); + + Console.WriteLine($"Committed: {result.Succeeded}"); + if (result.Succeeded && tree.ReadCommittedTryGet("order:42:state", out var state)) + Console.WriteLine(state); +} +catch (TransactionAbortedException) +{ + Console.WriteLine("Transaction aborted; the unit of work did not commit."); +} +``` + +`OpenOrCreateTransactional()` returns the transactional API. It differs from the ordinary tree: reads and writes in a transaction take a transaction ID, and success depends on the commit outcome. + +## Choose the smallest coordination scope + +| Data rule | API | +| --- | --- | +| Replace a value without reading its previous state. | [Ordinary writes](reads-and-writes.mdx). | +| Compute one key's new value from its current value. | [Atomic operations](atomic-operations.mdx). | +| Read or change several related keys as one transaction. | Transactional tree. | + +Concurrent application threads alone do not require transactions. Use them for a multi-key invariant, and make the reads that decide the writes part of that transaction. + +## Understand the transaction lifecycle + +1. `BeginTransaction()` allocates a transaction ID. +2. Reads and writes use that ID. +3. `PrepareAndCommit` attempts to prepare and commit the work. +4. A pending transaction must be resolved, or rolled back. An aborted unit of work must be retried from its reads if the application still wants to perform it. + +The engine uses optimistic coordination. Conflicts and dependencies can prevent an immediate commit, so checking only the individual write results is insufficient. + +| Commit state | What it means | What the caller does | +| --- | --- | --- | +| `Committed` | The transaction committed. | Continue with work that depends on success. | +| `ReadyToCommit` | An explicit `Prepare` succeeded. | Call `Commit` to finish, or roll back. | +| `PendingTransactions` | There are unresolved transaction dependencies. | Apply a bounded wait/retry policy or roll back. Inspect `PendingTransactionList` when diagnosing dependencies. | +| `Aborted` | The transaction cannot commit. | Retry the whole unit of work if appropriate. | + +`PrepareAndCommit` and other throwing APIs can raise `TransactionAbortedException`. Their `NoThrow` counterparts report abort status in a result. `NoThrow` describes transaction-abort reporting; it does not suppress every possible exception. + +## Keep retryable work safe to repeat + +The fluent transaction reruns its registered jobs when retrying an attempt that aborts before entering the pending wait. Read current values again inside those jobs; do not reuse a decision from a failed attempt. + +If a dependency aborts during the pending wait, `CommitAsync()` can throw `TransactionAbortedException` rather than return a failed result. Handle both outcomes. Using `NoThrow` operations inside the jobs does not make `CommitAsync()` exception-free. + +Keep network requests, notifications, and other external side effects outside the retryable jobs. Perform success-dependent follow-up only after the commit succeeds, with the delivery guarantees your application requires. + +Attempt and pending-wait limits can be configured: + +```csharp +using var transaction = tree.BeginFluentTransaction() + .SetRetryCountForAbortedTransactions(5) + .SetRetryCountForPendingTransactions(20); + +var result = await transaction + .Do(id => tree.UpsertNoThrow(id, "order:43:state", "confirmed")) + .CommitAsync(); +``` + +In the current implementation, the values above allow at most five complete transaction attempts and twenty pending-prepare checks. The aborted setting counts total attempts, despite the setter's retry-oriented name. + +Fluent transactions are caller-owned. Dispose them so unfinished work can be rolled back. A failed result or an abort exception is still a failed transaction; do not report success because retries were attempted. + +## Use explicit control when you need it + +Use explicit transaction IDs when your application owns pending-dependency handling and rollback. This example chooses to roll back any attempt that has not committed, including a pending one: + +```csharp +using ZoneTree.Transactional; + +var transactionId = tree.BeginTransaction(); +try +{ + var write = tree.UpsertNoThrow(transactionId, "order:44:state", "confirmed"); + if (!write.IsAborted) + { + var commit = tree.PrepareAndCommitNoThrow(transactionId); + Console.WriteLine(commit.Result); + } +} +finally +{ + if (tree.GetTransactionState(transactionId) == TransactionState.Uncommitted) + tree.Rollback(transactionId); +} +``` + +For a wait-and-retry policy, keep ownership of the transaction until it commits or is rolled back. Do not abandon a `PendingTransactions` result; its writes and dependency state still need resolution. + +## Read committed data + +Use the read-committed helpers when you need committed values without beginning a new transaction: + +```csharp +if (tree.ReadCommittedTryGet("order:42:state", out var state)) + Console.WriteLine(state); +``` + +Passing `currentTransactionId` lets that transaction see its own uncommitted writes while reading committed data from others. Separate read-committed calls do not establish a single snapshot across several keys. Use transactional reads when those values drive a multi-key decision. + +## Auto-commit and maintenance + +`UpsertAutoCommit` and `DeleteAutoCommit` start and commit a transaction around one write. They are convenient for applications whose writes must use the transactional API, but they add transaction work compared with an ordinary tree's write methods. + +Transactional maintenance exposes: + +| Surface | Purpose | +| --- | --- | +| `tree.Maintenance.ZoneTree` | The underlying storage tree. Do not bypass transaction coordination for transactional records. | +| `tree.Maintenance.TransactionLog` | Transaction log state and services. | +| `tree.Maintenance.UncommittedTransactionIds` | Unfinished transactions to investigate. | + +For abandoned work, an application-controlled cleanup policy can use: + +```csharp +tree.Maintenance.RollbackUncommittedTransactionIdsBefore( + DateTime.UtcNow.AddMinutes(-10)); +``` + +Choose a cutoff appropriate for your application; age alone does not prove that a transaction has been abandoned. WAL settings still determine storage durability. See [WAL modes](../durability/wal-modes.md) before choosing failure guarantees. + + + Keep transaction and storage lifetimes aligned +

+ Finish or roll back application work before closing the tree, and keep a + maintainer available while the service is running. +

+ MaintenanceProduction checklist + diff --git a/zonetree-dev/content/ui/guide-styles.ts b/zonetree-dev/content/ui/guide-styles.ts new file mode 100644 index 0000000..f2ae3dc --- /dev/null +++ b/zonetree-dev/content/ui/guide-styles.ts @@ -0,0 +1,62 @@ +import { styleBuilder, themes } from '@purestack/ts-style' +import { zoneTreeColors } from './skin' + +type GuideStyleContext = { + root: ReturnType + colors: typeof zoneTreeColors.light | typeof zoneTreeColors.dark +} + +export function registerGuideStyles() { + themes.forEach((theme) => { + const context = { root: styleBuilder.get(theme), colors: zoneTreeColors[theme] } + registerGuideTypography(context) + registerGuideExamples(context) + registerGuideNavigation(context) + }) +} + +function registerGuideTypography({ root, colors }: GuideStyleContext) { + root.select('.zt-guide-intro').marginBottom('28px') + root.select('.zt-guide-intro > .text-eyebrow').color(colors.accentInk).fontSize('10px') + .fontWeight('700').letterSpacing('1.5px').marginBottom('16px') + root.select('.zt-guide-intro > h1').margin('0').fontSize('clamp(32px, 3.5vw, 44px)') + .lineHeight('1.15').letterSpacing('-1.3px').fontWeight('700').textWrap('balance') + root.select('.zt-guide-intro > .text-tagline').marginBlock('20px 0').maxWidth('70ch') + .fontSize('18px').lineHeight('1.8').color(colors.muted) + root.select('.doc-content:has(.zt-guide-intro) > h2').paddingTop('24px').marginTop('40px') + .borderTop(`1px solid ${colors.subtle}`).fontSize('25px').fontWeight('650') + root.select('.zt-guide-outcomes').marginBlock('28px 32px') + root.select('.zt-guide-outcomes strong').fontSize('14px').fontWeight('650') + root.select('.zt-guide-outcomes p').fontSize('13px').lineHeight('1.7').margin('8px 0 0').color(colors.muted) + root.select('.zt-guide-callout').marginBlock('28px').fontSize('14px').lineHeight('1.8') + root.select('.zt-guide-callout p').margin('0') + root.select('.zt-guide-callout .flex').minWidth('0') + root.select('.doc-content:has(.zt-guide-intro) table').fontSize('13px').lineHeight('1.7') + root.select('.doc-content:has(.zt-guide-intro) :is(th, td)').verticalAlign('top') + root.select('.doc-content:has(.zt-guide-intro) td code').fontSize('12px').overflowWrap('anywhere').whiteSpace('normal') + root.media('(max-width: 600px)').select('.zt-guide-intro > .text-tagline').fontSize('16px') + root.media('(max-width: 600px)').select('.doc-content:has(.zt-guide-intro) > h2').fontSize('22px') +} + +function registerGuideExamples({ root, colors }: GuideStyleContext) { + root.select('.doc-content:has(.zt-guide-intro) pre').maxWidth('100%').overflowX('auto') + .fontSize('12.5px').lineHeight('1.8').border(`1px solid ${colors.subtle}`) + root.select('.zt-guide-output pre').background(colors.surfaceAlt).borderColor(colors.accentBorder) + root.select('.zt-guide-reference').marginBlock('16px') + root.select('.zt-guide-reference :is(p, li)').fontSize('14px').lineHeight('1.8').overflowWrap('anywhere') + root.select('.zt-guide-reference table').margin('0') + root.media('(max-width: 600px)').select('.doc-content:has(.zt-guide-intro) pre').padding('16px').fontSize('12px') +} + +function registerGuideNavigation({ root, colors }: GuideStyleContext) { + root.select('.zt-guide-next').marginBlock('24px 32px') + root.select('.zt-guide-next .docs-card').padding('24px') + root.select('.zt-guide-next .docs-card-meta').marginBottom('20px') + root.select('.zt-guide-next .docs-card h3').fontSize('20px').lineHeight('1.3') + root.select('.zt-guide-next .docs-card p').fontSize('13px').marginBlock('14px 24px') + root.select('.zt-guide-next-panel').marginBlock('40px 16px') + root.select('.zt-guide-next-panel strong').fontSize('17px').lineHeight('1.4').fontWeight('650') + root.select('.zt-guide-next-panel p').marginBlock('12px 20px').fontSize('14px').lineHeight('1.8').color(colors.muted) + root.select('.doc-content:has(.zt-guide-intro) a:focus-visible').outline(`2px solid ${colors.accentInk}`) + .outlineOffset('4px').borderRadius('4px') +} diff --git a/zonetree-dev/content/ui/styles.ts b/zonetree-dev/content/ui/styles.ts index cbbe063..821c553 100644 --- a/zonetree-dev/content/ui/styles.ts +++ b/zonetree-dev/content/ui/styles.ts @@ -6,11 +6,13 @@ import { registerBenchmarkReportStyles } from './benchmark-styles' import { registerImageGalleryStyles } from './image-gallery-styles' import { registerErrorPageStyles } from './error-styles' import { registerDocsOverviewStyles } from './docs-overview-styles' +import { registerGuideStyles } from './guide-styles' export function registerDocsStyles() { registerErrorPageStyles() registerImageGalleryStyles() registerDocsOverviewStyles() + registerGuideStyles() registerBenchmarkReportStyles() themes.forEach((theme, palette) => { const context = createDocsStyleContext(theme, palette) diff --git a/zonetree-dev/package.json b/zonetree-dev/package.json index 9e0c604..db76bfb 100644 --- a/zonetree-dev/package.json +++ b/zonetree-dev/package.json @@ -9,7 +9,7 @@ "build": "purestack build --content ./content --clean", "serve": "purestack serve --content ./content --port 4900 --clean", "publish": "purestack publish --content ./content", - "deploy": "pgdeploy deploy --host https://zonetree.dev --package ./dist/web --allow-path 'pagefind/*,.ts-ssg/manifest.json,_assets/benchmarks/*.json'" + "deploy": "pgdeploy deploy --host https://zonetree.dev --package ./dist/web --allow-path 'pagefind/*,.ts-ssg/manifest.json,assets/benchmarks/*.json' --skip-path '.purestack-output'" }, "dependencies": { "@purestack/ts-components": "1.1.27",