mirror of
https://github.com/navidrome/navidrome.git
synced 2026-10-16 20:32:30 +08:00
perf(artwork): cap the stale-absent recheck at 100 items per kind per hour (#6007)
* feat(artwork): drip the stale-absent recheck instead of bursting it daily Each hourly housekeeping tick now re-queues at most 100 absent states per kind, oldest attempts first, instead of everything older than 24h at once. External agents see a flat ~100 requests/hour per agent instead of hourly bursts of ~2,000, and the effective recheck interval self-scales with the size of the absent pool (~4 days at 10k absent artists) while small libraries keep the 24h floor. * feat(artwork): trust an absent artwork state for a week before rechecking With the recheck now dripped at 100 items per kind per hour, the 24h floor only governed small libraries, where the drip cap never binds; they still re-asked every agent daily. A 7-day floor cuts that cost 7x and, for large libraries, becomes the binding limit over the drip cycle (~5.7k calls/day instead of ~9.6k at 10k absent artists). Among comparable servers, this is still the second-most-eager recheck: gonic retries misses every 30 days, Jellyfin and Funkwhale never do. * refactor(artwork): state the drip's backpressure contract where it bites Review follow-ups: the recheck limit deliberately caps the *selection*, not the insertions — already-queued rows use up budget, so a stalled drain admits no new work instead of building a recovery burst. Say so in the interface doc, mirror it in the mock by truncating the sorted candidates (matching the SQL's LIMIT-before-ON CONFLICT), and teach `artwork status` and the worker doc the post-drip wording. Also pin the one cmd fixture that still assumed a 24h recheck window.
This commit is contained in:
@@ -18,7 +18,11 @@ import (
|
||||
)
|
||||
|
||||
// StaleAbsentAge is how long an absent state is trusted before a recheck retries it.
|
||||
const StaleAbsentAge = 24 * time.Hour
|
||||
const StaleAbsentAge = 7 * 24 * time.Hour
|
||||
|
||||
// StaleAbsentRecheckBatch caps how many absent states each hourly tick re-queues per kind,
|
||||
// oldest first, so external agents see a flat drip instead of a daily burst.
|
||||
const StaleAbsentRecheckBatch = 100
|
||||
|
||||
// RecheckKinds omits media files: they resolve embedded-only, at scan or on view.
|
||||
var RecheckKinds = []model.Kind{
|
||||
@@ -125,7 +129,7 @@ func enqueueStaleAbsentAll(ctx context.Context, ds model.DataStore) error {
|
||||
cutoff := time.Now().Add(-StaleAbsentAge)
|
||||
queue := ds.ArtworkQueue(ctx)
|
||||
for _, kind := range RecheckKinds {
|
||||
if _, err := queue.EnqueueStaleAbsent(kind, cutoff); err != nil {
|
||||
if _, err := queue.EnqueueStaleAbsent(kind, cutoff, StaleAbsentRecheckBatch); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package artwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
@@ -235,8 +236,8 @@ var _ = Describe("Housekeeping", func() {
|
||||
})
|
||||
|
||||
It("enqueues only absent entries older than the recheck window, across all kinds", func() {
|
||||
old := time.Now().Add(-48 * time.Hour)
|
||||
recent := time.Now().Add(-time.Hour)
|
||||
old := time.Now().Add(-StaleAbsentAge - time.Hour)
|
||||
recent := time.Now().Add(-StaleAbsentAge + time.Hour)
|
||||
|
||||
artRepo.ItemData["ar-stale"] = model.ItemArtwork{ItemKind: "ar", ItemID: "ar1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old}
|
||||
artRepo.ItemData["al-stale"] = model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old}
|
||||
@@ -259,6 +260,20 @@ var _ = Describe("Housekeeping", func() {
|
||||
Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ar", "ar2")).To(BeNil())
|
||||
Expect(findQueued(queueRepo.MockArtworkQueueRepo, "al", "al2")).To(BeNil())
|
||||
})
|
||||
|
||||
It("caps each tick at the recheck batch, oldest attempts first", func() {
|
||||
for i := range StaleAbsentRecheckBatch + 1 {
|
||||
id := fmt.Sprintf("ar%d", i)
|
||||
artRepo.ItemData[id] = model.ItemArtwork{ItemKind: "ar", ItemID: id, ImageType: model.ImageTypePrimary,
|
||||
Hash: "", AttemptedAt: time.Now().Add(-StaleAbsentAge - time.Duration(i+1)*time.Minute)}
|
||||
}
|
||||
|
||||
Expect(enqueueStaleAbsentAll(ctx, ds)).To(Succeed())
|
||||
|
||||
Expect(queueRepo.Data).To(HaveLen(StaleAbsentRecheckBatch))
|
||||
// ar0 has the newest attempted_at of the cohort, so it is the one left out.
|
||||
Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ar", "ar0")).To(BeNil())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("EnqueueMissingAll", func() {
|
||||
|
||||
@@ -137,7 +137,8 @@ func (w *Worker) Backfill(ctx context.Context) (bool, error) {
|
||||
return backfill(ctx, w.proc.ds)
|
||||
}
|
||||
|
||||
// EnqueueStaleAbsentAll requeues known-absent entries older than StaleAbsentAge.
|
||||
// EnqueueStaleAbsentAll requeues known-absent entries older than StaleAbsentAge, at most
|
||||
// StaleAbsentRecheckBatch per kind, oldest first.
|
||||
func (w *Worker) EnqueueStaleAbsentAll(ctx context.Context) error {
|
||||
return enqueueStaleAbsentAll(ctx, w.proc.ds)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user