// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. using System; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Text; using CommunityToolkit.HighPerformance.Helpers; #if NET6_0_OR_GREATER using BitOperations = System.Numerics.BitOperations; #else using BitOperations = CommunityToolkit.HighPerformance.Helpers.Internals.BitOperations; #endif namespace CommunityToolkit.HighPerformance.Buffers; /// /// A configurable pool for instances. This can be used to minimize allocations /// when creating multiple instances from buffers of values. /// The method provides a best-effort alternative to just creating /// a new instance every time, in order to minimize the number of duplicated instances. /// The type will internally manage a highly efficient priority queue for the /// cached instances, so that when the full capacity is reached, the least frequently /// used values will be automatically discarded to leave room for new values to cache. /// public sealed class StringPool { /// /// The size used by default by the parameterless constructor. /// private const int DefaultSize = 2048; /// /// The minimum size for instances. /// private const int MinimumSize = 32; /// /// The current array of instances in use. /// private readonly FixedSizePriorityMap[] maps; /// /// The total number of maps in use. /// private readonly int numberOfMaps; /// /// Initializes a new instance of the class. /// public StringPool() : this(DefaultSize) { } /// /// Initializes a new instance of the class. /// /// The minimum size for the pool to create. public StringPool(int minimumSize) { if (minimumSize <= 0) { ThrowArgumentOutOfRangeException(); } // Set the minimum size minimumSize = Math.Max(minimumSize, MinimumSize); // Calculates the rounded up factors for a specific size/factor pair static void FindFactors(int size, int factor, out uint x, out uint y) { double a = Math.Sqrt((double)size / factor); double b = factor * a; x = BitOperations.RoundUpToPowerOf2((uint)a); y = BitOperations.RoundUpToPowerOf2((uint)b); } // We want to find two powers of 2 factors that produce a number // that is at least equal to the requested size. In order to find the // combination producing the optimal factors (with the product being as // close as possible to the requested size), we test a number of ratios // that we consider acceptable, and pick the best results produced. // The ratio between maps influences the number of objects being allocated, // as well as the multithreading performance when locking on maps. // We still want to contraint this number to avoid situations where we // have a way too high number of maps compared to total size. FindFactors(minimumSize, 2, out uint x2, out uint y2); FindFactors(minimumSize, 3, out uint x3, out uint y3); FindFactors(minimumSize, 4, out uint x4, out uint y4); uint p2 = x2 * y2; uint p3 = x3 * y3; uint p4 = x4 * y4; if (p3 < p2) { p2 = p3; x2 = x3; y2 = y3; } if (p4 < p2) { p2 = p4; x2 = x4; y2 = y4; } Span span = this.maps = new FixedSizePriorityMap[x2]; // We preallocate the maps in advance, since each bucket only contains the // array field, which is not preinitialized, so the allocations are minimal. // This lets us lock on each individual maps when retrieving a string instance. foreach (ref FixedSizePriorityMap map in span) { map = new FixedSizePriorityMap((int)y2); } this.numberOfMaps = (int)x2; Size = (int)p2; } /// /// Gets the shared instance. /// /// /// The shared pool provides a singleton, reusable instance that /// can be accessed directly, and that pools instances for the entire /// process. Since is thread-safe, the shared instance can be used /// concurrently by multiple threads without the need for manual synchronization. /// public static StringPool Shared { get; } = new(); /// /// Gets the total number of that can be stored in the current instance. /// public int Size { get; } /// /// Stores a instance in the internal cache. /// /// The input instance to cache. public void Add(string value) { if (value.Length == 0) { return; } int hashcode = GetHashCode(value.AsSpan()); int bucketIndex = hashcode & (this.numberOfMaps - 1); ref FixedSizePriorityMap map = ref this.maps.DangerousGetReferenceAt(bucketIndex); lock (map.SyncRoot) { map.Add(value, hashcode); } } /// /// Gets a cached instance matching the input content, or stores the input one. /// /// The input instance with the contents to use. /// A instance with the contents of , cached if possible. public string GetOrAdd(string value) { if (value.Length == 0) { return string.Empty; } int hashcode = GetHashCode(value.AsSpan()); int bucketIndex = hashcode & (this.numberOfMaps - 1); ref FixedSizePriorityMap map = ref this.maps.DangerousGetReferenceAt(bucketIndex); lock (map.SyncRoot) { return map.GetOrAdd(value, hashcode); } } /// /// Gets a cached instance matching the input content, or creates a new one. /// /// The input with the contents to use. /// A instance with the contents of , cached if possible. public string GetOrAdd(ReadOnlySpan span) { if (span.IsEmpty) { return string.Empty; } int hashcode = GetHashCode(span); int bucketIndex = hashcode & (this.numberOfMaps - 1); ref FixedSizePriorityMap map = ref this.maps.DangerousGetReferenceAt(bucketIndex); lock (map.SyncRoot) { return map.GetOrAdd(span, hashcode); } } /// /// Gets a cached instance matching the input content (converted to Unicode), or creates a new one. /// /// The input with the contents to use, in a specified encoding. /// The instance to use to decode the contents of . /// A instance with the contents of , cached if possible. public unsafe string GetOrAdd(ReadOnlySpan span, Encoding encoding) { if (span.IsEmpty) { return string.Empty; } int maxLength = encoding.GetMaxCharCount(span.Length); using SpanOwner buffer = SpanOwner.Allocate(maxLength); fixed (byte* source = span) fixed (char* destination = &buffer.DangerousGetReference()) { int effectiveLength = encoding.GetChars(source, span.Length, destination, maxLength); return GetOrAdd(new ReadOnlySpan(destination, effectiveLength)); } } /// /// Tries to get a cached instance matching the input content, if present. /// /// The input with the contents to use. /// The resulting cached instance, if present /// Whether or not the target instance was found. public bool TryGet(ReadOnlySpan span, [NotNullWhen(true)] out string? value) { if (span.IsEmpty) { value = string.Empty; return true; } int hashcode = GetHashCode(span); int bucketIndex = hashcode & (this.numberOfMaps - 1); ref FixedSizePriorityMap map = ref this.maps.DangerousGetReferenceAt(bucketIndex); lock (map.SyncRoot) { return map.TryGet(span, hashcode, out value); } } /// /// Resets the current instance and its associated maps. /// public void Reset() { foreach (ref FixedSizePriorityMap map in this.maps.AsSpan()) { lock (map.SyncRoot) { map.Reset(); } } } /// /// A configurable map containing a group of cached instances. /// /// /// Instances of this type are stored in an array within and they are /// always accessed by reference - essentially as if this type had been a class. The type is /// also private, so there's no risk for users to directly access it and accidentally copy an /// instance, which would lead to bugs due to values becoming out of sync with the internal state /// (that is, because instances would be copied by value, so primitive fields would not be shared). /// The reason why we're using a struct here is to remove an indirection level and improve cache /// locality when accessing individual buckets from the methods in the type. /// private struct FixedSizePriorityMap { /// /// The index representing the end of a given list. /// private const int EndOfList = -1; /// /// The array of 1-based indices for the items stored in . /// private readonly int[] buckets; /// /// The array of currently cached entries (ie. the lists for each hash group). /// private readonly MapEntry[] mapEntries; /// /// The array of priority values associated to each item stored in . /// private readonly HeapEntry[] heapEntries; /// /// The current number of items stored in the map. /// private int count; /// /// The current incremental timestamp for the items stored in . /// private uint timestamp; /// /// A type representing a map entry, ie. a node in a given list. /// private struct MapEntry { /// /// The precomputed hashcode for . /// public int HashCode; /// /// The instance cached in this entry. /// public string? Value; /// /// The 0-based index for the next node in the current list. /// public int NextIndex; /// /// The 0-based index for the heap entry corresponding to the current node. /// public int HeapIndex; } /// /// A type representing a heap entry, used to associate priority to each item. /// private struct HeapEntry { /// /// The timestamp for the current entry (ie. the priority for the item). /// public uint Timestamp; /// /// The 0-based index for the map entry corresponding to the current item. /// public int MapIndex; } /// /// Initializes a new instance of the struct. /// /// The fixed capacity of the current map. public FixedSizePriorityMap(int capacity) { this.buckets = new int[capacity]; this.mapEntries = new MapEntry[capacity]; this.heapEntries = new HeapEntry[capacity]; this.count = 0; this.timestamp = 0; } /// /// Gets an that can be used to synchronize access to the current instance. /// public object SyncRoot { [MethodImpl(MethodImplOptions.AggressiveInlining)] get => this.buckets; } /// /// Implements for the current instance. /// /// The input instance to cache. /// The precomputed hashcode for . [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Add(string value, int hashcode) { ref string target = ref TryGet(value.AsSpan(), hashcode); if (Unsafe.IsNullRef(ref target)) { Insert(value, hashcode); } else { target = value; } } /// /// Implements for the current instance. /// /// The input instance with the contents to use. /// The precomputed hashcode for . /// A instance with the contents of . [MethodImpl(MethodImplOptions.AggressiveInlining)] public string GetOrAdd(string value, int hashcode) { ref string result = ref TryGet(value.AsSpan(), hashcode); if (!Unsafe.IsNullRef(ref result)) { return result; } Insert(value, hashcode); return value; } /// /// Implements for the current instance. /// /// The input with the contents to use. /// The precomputed hashcode for . /// A instance with the contents of , cached if possible. [MethodImpl(MethodImplOptions.AggressiveInlining)] public string GetOrAdd(ReadOnlySpan span, int hashcode) { ref string result = ref TryGet(span, hashcode); if (!Unsafe.IsNullRef(ref result)) { return result; } string value = span.ToString(); Insert(value, hashcode); return value; } /// /// Implements for the current instance. /// /// The input with the contents to use. /// The precomputed hashcode for . /// The resulting cached instance, if present /// Whether or not the target instance was found. [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool TryGet(ReadOnlySpan span, int hashcode, [NotNullWhen(true)] out string? value) { ref string result = ref TryGet(span, hashcode); if (!Unsafe.IsNullRef(ref result)) { value = result; return true; } value = null; return false; } /// /// Resets the current instance and throws away all the cached values. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Reset() { this.buckets.AsSpan().Clear(); this.mapEntries.AsSpan().Clear(); this.heapEntries.AsSpan().Clear(); this.count = 0; this.timestamp = 0; } /// /// Tries to get a target instance, if it exists, and returns a reference to it. /// /// The input with the contents to use. /// The precomputed hashcode for . /// A reference to the slot where the target instance could be. [MethodImpl(MethodImplOptions.NoInlining)] private unsafe ref string TryGet(ReadOnlySpan span, int hashcode) { ref MapEntry mapEntriesRef = ref this.mapEntries.DangerousGetReference(); ref MapEntry entry = ref Unsafe.NullRef(); int length = this.buckets.Length; int bucketIndex = hashcode & (length - 1); for (int i = this.buckets.DangerousGetReferenceAt(bucketIndex) - 1; (uint)i < (uint)length; i = entry.NextIndex) { entry = ref Unsafe.Add(ref mapEntriesRef, (nint)(uint)i); if (entry.HashCode == hashcode && entry.Value!.AsSpan().SequenceEqual(span)) { UpdateTimestamp(ref entry.HeapIndex); return ref entry.Value!; } } return ref Unsafe.NullRef(); } /// /// Inserts a new instance in the current map, freeing up a space if needed. /// /// The new instance to store. /// The precomputed hashcode for . [MethodImpl(MethodImplOptions.NoInlining)] private void Insert(string value, int hashcode) { ref int bucketsRef = ref this.buckets.DangerousGetReference(); ref MapEntry mapEntriesRef = ref this.mapEntries.DangerousGetReference(); ref HeapEntry heapEntriesRef = ref this.heapEntries.DangerousGetReference(); int entryIndex, heapIndex; // If the current map is full, first get the oldest value, which is // always the first item in the heap. Then, free up a slot by // removing that, and insert the new value in that empty location. if (this.count == this.mapEntries.Length) { entryIndex = heapEntriesRef.MapIndex; heapIndex = 0; ref MapEntry removedEntry = ref Unsafe.Add(ref mapEntriesRef, (nint)(uint)entryIndex); // The removal logic can be extremely optimized in this case, as we // can retrieve the precomputed hashcode for the target entry by doing // a lookup on the target map node, and we can also skip all the comparisons // while traversing the target chain since we know in advance the index of // the target node which will contain the item to remove from the map. Remove(removedEntry.HashCode, entryIndex); } else { // If the free list is not empty, get that map node and update the field entryIndex = this.count; heapIndex = this.count; } int bucketIndex = hashcode & (this.buckets.Length - 1); ref int targetBucket = ref Unsafe.Add(ref bucketsRef, (nint)(uint)bucketIndex); ref MapEntry targetMapEntry = ref Unsafe.Add(ref mapEntriesRef, (nint)(uint)entryIndex); ref HeapEntry targetHeapEntry = ref Unsafe.Add(ref heapEntriesRef, (nint)(uint)heapIndex); // Assign the values in the new map entry targetMapEntry.HashCode = hashcode; targetMapEntry.Value = value; targetMapEntry.NextIndex = targetBucket - 1; targetMapEntry.HeapIndex = heapIndex; // Update the bucket slot and the current count targetBucket = entryIndex + 1; this.count++; // Link the heap node with the current entry targetHeapEntry.MapIndex = entryIndex; // Update the timestamp and balance the heap again UpdateTimestamp(ref targetMapEntry.HeapIndex); } /// /// Removes a specified instance from the map to free up one slot. /// /// The precomputed hashcode of the instance to remove. /// The index of the target map node to remove. /// The input instance needs to already exist in the map. [MethodImpl(MethodImplOptions.NoInlining)] private void Remove(int hashcode, int mapIndex) { ref MapEntry mapEntriesRef = ref this.mapEntries.DangerousGetReference(); int bucketIndex = hashcode & (this.buckets.Length - 1); int entryIndex = this.buckets.DangerousGetReferenceAt(bucketIndex) - 1; int lastIndex = EndOfList; // We can just have an undefined loop, as the input // value we're looking for is guaranteed to be present while (true) { ref MapEntry candidate = ref Unsafe.Add(ref mapEntriesRef, (nint)(uint)entryIndex); // Check the current value for a match if (entryIndex == mapIndex) { // If this was not the first list node, update the parent as well if (lastIndex != EndOfList) { ref MapEntry lastEntry = ref Unsafe.Add(ref mapEntriesRef, (nint)(uint)lastIndex); lastEntry.NextIndex = candidate.NextIndex; } else { // Otherwise, update the target index from the bucket slot this.buckets.DangerousGetReferenceAt(bucketIndex) = candidate.NextIndex + 1; } this.count--; return; } // Move to the following node in the current list lastIndex = entryIndex; entryIndex = candidate.NextIndex; } } /// /// Updates the timestamp of a heap node at the specified index (which is then synced back). /// /// The index of the target heap node to update. [MethodImpl(MethodImplOptions.NoInlining)] private void UpdateTimestamp(ref int heapIndex) { int currentIndex = heapIndex; int count = this.count; ref MapEntry mapEntriesRef = ref this.mapEntries.DangerousGetReference(); ref HeapEntry heapEntriesRef = ref this.heapEntries.DangerousGetReference(); ref HeapEntry root = ref Unsafe.Add(ref heapEntriesRef, (nint)(uint)currentIndex); uint timestamp = this.timestamp; // Check if incrementing the current timestamp for the heap node to update // would result in an overflow. If that happened, we could end up violating // the min-heap property (the value of each node has to always be <= than that // of its child nodes), eg. if we were updating a node that was not the root. // In that scenario, we could end up with a node somewhere along the heap with // a value lower than that of its parent node (as the timestamp would be 0). // To guard against this, we just check the current timestamp value, and if // the maximum value has been reached, we reinitialize the entire heap. This // is done in a non-inlined call, so we don't increase the codegen size in this // method. The reinitialization simply traverses the heap in breadth-first order // (ie. level by level), and assigns incrementing timestamps to all nodes starting // from 0. The value of the current timestamp is then just set to the current size. if (timestamp == uint.MaxValue) { // We use a goto here as this path is very rarely taken. Doing so // causes the generated asm to contain a forward jump to the fallback // path if this branch is taken, whereas the normal execution path will // not need to execute any jumps at all. This is done to reduce the overhead // introduced by this check in all the invocations where this point is not reached. goto Fallback; } Downheap: // Assign a new timestamp to the target heap node. We use a // local incremental timestamp instead of using the system timer // as this greatly reduces the overhead and the time spent in system calls. // The uint type provides a large enough range and it's unlikely users would ever // exhaust it anyway (especially considering each map has a separate counter). root.Timestamp = this.timestamp = timestamp + 1; // Once the timestamp is updated (which will cause the heap to become // unbalanced), start a sift down loop to balance the heap again. while (true) { // The heap is 0-based (so that the array length can remain the same // as the power of 2 value used for the other arrays in this type). // This means that children of each node are at positions: // - left: (2 * n) + 1 // - right: (2 * n) + 2 ref HeapEntry minimum = ref root; int left = (currentIndex * 2) + 1; int right = (currentIndex * 2) + 2; int targetIndex = currentIndex; // Check and update the left child, if necessary if (left < count) { ref HeapEntry child = ref Unsafe.Add(ref heapEntriesRef, (nint)(uint)left); if (child.Timestamp < minimum.Timestamp) { minimum = ref child; targetIndex = left; } } // Same check as above for the right child if (right < count) { ref HeapEntry child = ref Unsafe.Add(ref heapEntriesRef, (nint)(uint)right); if (child.Timestamp < minimum.Timestamp) { minimum = ref child; targetIndex = right; } } // If no swap is pending, we can just stop here. // Before returning, we update the target index as well. if (Unsafe.AreSame(ref root, ref minimum)) { heapIndex = targetIndex; return; } // Update the indices in the respective map entries (accounting for the swap) Unsafe.Add(ref mapEntriesRef, (nint)(uint)root.MapIndex).HeapIndex = targetIndex; Unsafe.Add(ref mapEntriesRef, (nint)(uint)minimum.MapIndex).HeapIndex = currentIndex; currentIndex = targetIndex; // Swap the parent and child (so that the minimum value bubbles up) HeapEntry temp = root; root = minimum; minimum = temp; // Update the reference to the root node root = ref Unsafe.Add(ref heapEntriesRef, (nint)(uint)currentIndex); } Fallback: UpdateAllTimestamps(); // After having updated all the timestamps, if the heap contains N items, the // node in the bottom right corner will have a value of N - 1. Since the timestamp // is incremented by 1 before starting the downheap execution, here we simply // update the local timestamp to be N - 1, so that the code above will set the // timestamp of the node currently being updated to exactly N. timestamp = (uint)(count - 1); goto Downheap; } /// /// Updates the timestamp of all the current heap nodes in incrementing order. /// The heap is always guaranteed to be complete binary tree, so when it contains /// a given number of nodes, those are all contiguous from the start of the array. /// [MethodImpl(MethodImplOptions.NoInlining)] private void UpdateAllTimestamps() { int count = this.count; ref HeapEntry heapEntriesRef = ref this.heapEntries.DangerousGetReference(); for (int i = 0; i < count; i++) { Unsafe.Add(ref heapEntriesRef, (nint)(uint)i).Timestamp = (uint)i; } } } /// /// Gets the (positive) hashcode for a given instance. /// /// The input instance. /// The hashcode for . [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int GetHashCode(ReadOnlySpan span) { return HashCode.Combine(span); } /// /// Throws an when the requested size exceeds the capacity. /// private static void ThrowArgumentOutOfRangeException() { throw new ArgumentOutOfRangeException("minimumSize", "The requested size must be greater than 0"); } }