checker: add tiered RetentionPolicy
Introduce a pure RetentionPolicy.Decide function that partitions check executions into keep/drop sets according to a tiered policy: - 0..7 days -> every execution - 7..30 days -> 2 per day per (checker, target) - 30..D/2 -> 1 per week per (checker, target) - D/2..D days -> 1 per month per (checker, target) - > D days -> dropped The function is intentionally storage-agnostic so the upcoming janitor goroutine can call it on any execution slice and so it can be unit tested directly. All thresholds are configurable to allow per-user overrides via UserQuota.
This commit is contained in:
parent
5ec22c6678
commit
bf9159d967
2 changed files with 311 additions and 0 deletions
183
internal/usecase/checker/retention.go
Normal file
183
internal/usecase/checker/retention.go
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
// This file is part of the happyDomain (R) project.
|
||||
// Copyright (c) 2020-2026 happyDomain
|
||||
// Authors: Pierre-Olivier Mercier, et al.
|
||||
//
|
||||
// This program is offered under a commercial and under the AGPL license.
|
||||
// For commercial licensing, contact us at <contact@happydomain.org>.
|
||||
//
|
||||
// For AGPL licensing:
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package checker
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"git.happydns.org/happyDomain/model"
|
||||
)
|
||||
|
||||
// RetentionPolicy describes how check executions are thinned out as they age.
|
||||
//
|
||||
// The policy is intentionally tiered: users care about full detail for recent
|
||||
// runs, but only need sparse historical samples to spot long-term trends.
|
||||
//
|
||||
// Default behaviour, given a RetentionDays of D:
|
||||
//
|
||||
// age window | kept
|
||||
// ------------------------- | ------------------------------------------
|
||||
// 0 .. 7 days | every execution
|
||||
// 7 .. 30 days | up to 2 executions per day per (checker,target)
|
||||
// 30 .. D/2 days | up to 1 execution per week per (checker,target)
|
||||
// D/2 .. D days | up to 1 execution per month per (checker,target)
|
||||
// > D days | dropped
|
||||
//
|
||||
// All thresholds and bucket counts are configurable so the policy can be
|
||||
// tuned per-user via the admin UserQuota.
|
||||
type RetentionPolicy struct {
|
||||
// RetentionDays is the hard cap on age. Executions older than this are
|
||||
// always dropped. Must be > 0.
|
||||
RetentionDays int
|
||||
|
||||
// FullDetailDays: every execution kept under this age.
|
||||
FullDetailDays int
|
||||
// DailyBucketDays: between FullDetailDays and DailyBucketDays, keep
|
||||
// PerDayKept executions per UTC day per (checker,target).
|
||||
DailyBucketDays int
|
||||
PerDayKept int
|
||||
// WeeklyBucketDays: between DailyBucketDays and WeeklyBucketDays, keep
|
||||
// PerWeekKept executions per ISO week per (checker,target).
|
||||
WeeklyBucketDays int
|
||||
PerWeekKept int
|
||||
// Beyond WeeklyBucketDays and up to RetentionDays, keep PerMonthKept
|
||||
// executions per calendar month per (checker,target).
|
||||
PerMonthKept int
|
||||
}
|
||||
|
||||
// DefaultRetentionPolicy returns the standard tiered policy for the given
|
||||
// retention horizon.
|
||||
func DefaultRetentionPolicy(retentionDays int) RetentionPolicy {
|
||||
if retentionDays <= 0 {
|
||||
retentionDays = 365
|
||||
}
|
||||
return RetentionPolicy{
|
||||
RetentionDays: retentionDays,
|
||||
FullDetailDays: 7,
|
||||
DailyBucketDays: 30,
|
||||
PerDayKept: 2,
|
||||
WeeklyBucketDays: max(retentionDays/2, 31),
|
||||
PerWeekKept: 1,
|
||||
PerMonthKept: 1,
|
||||
}
|
||||
}
|
||||
|
||||
// Decide partitions executions into the ones to keep and the ones to drop
|
||||
// according to the policy. The function is pure: it does not touch storage.
|
||||
//
|
||||
// Executions are grouped by (CheckerID, Target) and ordered most-recent-first
|
||||
// inside each group, so the newest execution in a bucket is the one preserved.
|
||||
func (p RetentionPolicy) Decide(executions []*happydns.Execution, now time.Time) (keep, drop []happydns.Identifier) {
|
||||
if len(executions) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Group by (checker, target).
|
||||
groups := map[string][]*happydns.Execution{}
|
||||
for _, e := range executions {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
key := e.CheckerID + "|" + e.Target.String()
|
||||
groups[key] = append(groups[key], e)
|
||||
}
|
||||
|
||||
hardCutoff := now.AddDate(0, 0, -p.RetentionDays)
|
||||
fullCutoff := now.AddDate(0, 0, -p.FullDetailDays)
|
||||
dailyCutoff := now.AddDate(0, 0, -p.DailyBucketDays)
|
||||
weeklyCutoff := now.AddDate(0, 0, -p.WeeklyBucketDays)
|
||||
|
||||
for _, group := range groups {
|
||||
// Most recent first.
|
||||
sort.Slice(group, func(i, j int) bool {
|
||||
return group[i].StartedAt.After(group[j].StartedAt)
|
||||
})
|
||||
|
||||
dayBuckets := map[string]int{}
|
||||
weekBuckets := map[string]int{}
|
||||
monthBuckets := map[string]int{}
|
||||
|
||||
for _, e := range group {
|
||||
t := e.StartedAt
|
||||
switch {
|
||||
case t.Before(hardCutoff):
|
||||
drop = append(drop, e.Id)
|
||||
case !t.Before(fullCutoff):
|
||||
// 0 .. FullDetailDays — keep everything.
|
||||
keep = append(keep, e.Id)
|
||||
case !t.Before(dailyCutoff):
|
||||
k := t.UTC().Format("2006-01-02")
|
||||
if dayBuckets[k] < p.PerDayKept {
|
||||
dayBuckets[k]++
|
||||
keep = append(keep, e.Id)
|
||||
} else {
|
||||
drop = append(drop, e.Id)
|
||||
}
|
||||
case !t.Before(weeklyCutoff):
|
||||
y, w := t.UTC().ISOWeek()
|
||||
k := isoWeekKey(y, w)
|
||||
if weekBuckets[k] < p.PerWeekKept {
|
||||
weekBuckets[k]++
|
||||
keep = append(keep, e.Id)
|
||||
} else {
|
||||
drop = append(drop, e.Id)
|
||||
}
|
||||
default:
|
||||
k := t.UTC().Format("2006-01")
|
||||
if monthBuckets[k] < p.PerMonthKept {
|
||||
monthBuckets[k]++
|
||||
keep = append(keep, e.Id)
|
||||
} else {
|
||||
drop = append(drop, e.Id)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return keep, drop
|
||||
}
|
||||
|
||||
func isoWeekKey(year, week int) string {
|
||||
return time.Date(year, 1, 1, 0, 0, 0, 0, time.UTC).Format("2006") + "-W" + twoDigits(week)
|
||||
}
|
||||
|
||||
func twoDigits(n int) string {
|
||||
if n < 10 {
|
||||
return "0" + itoa(n)
|
||||
}
|
||||
return itoa(n)
|
||||
}
|
||||
|
||||
func itoa(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
var buf [4]byte
|
||||
i := len(buf)
|
||||
for n > 0 {
|
||||
i--
|
||||
buf[i] = byte('0' + n%10)
|
||||
n /= 10
|
||||
}
|
||||
return string(buf[i:])
|
||||
}
|
||||
128
internal/usecase/checker/retention_test.go
Normal file
128
internal/usecase/checker/retention_test.go
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
// This file is part of the happyDomain (R) project.
|
||||
// Copyright (c) 2020-2026 happyDomain
|
||||
// Authors: Pierre-Olivier Mercier, et al.
|
||||
//
|
||||
// This program is offered under a commercial and under the AGPL license.
|
||||
// For commercial licensing, contact us at <contact@happydomain.org>.
|
||||
//
|
||||
// For AGPL licensing:
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
package checker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.happydns.org/happyDomain/model"
|
||||
)
|
||||
|
||||
func mkExec(id string, age time.Duration, now time.Time) *happydns.Execution {
|
||||
return &happydns.Execution{
|
||||
Id: happydns.Identifier(id),
|
||||
CheckerID: "ping",
|
||||
Target: happydns.CheckTarget{DomainId: "example.com"},
|
||||
StartedAt: now.Add(-age),
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecide_Empty(t *testing.T) {
|
||||
p := DefaultRetentionPolicy(365)
|
||||
keep, drop := p.Decide(nil, time.Now())
|
||||
if len(keep) != 0 || len(drop) != 0 {
|
||||
t.Fatalf("expected empty results, got keep=%d drop=%d", len(keep), len(drop))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecide_FullDetailWindow(t *testing.T) {
|
||||
now := time.Date(2026, 4, 8, 12, 0, 0, 0, time.UTC)
|
||||
p := DefaultRetentionPolicy(365)
|
||||
|
||||
var execs []*happydns.Execution
|
||||
for i := 0; i < 50; i++ {
|
||||
execs = append(execs, mkExec(fmt.Sprintf("e%d", i), time.Duration(i)*time.Hour, now))
|
||||
}
|
||||
|
||||
keep, drop := p.Decide(execs, now)
|
||||
if len(drop) != 0 {
|
||||
t.Fatalf("expected no drops in <7d window, got %d", len(drop))
|
||||
}
|
||||
if len(keep) != 50 {
|
||||
t.Fatalf("expected 50 keeps, got %d", len(keep))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecide_DailyBucket(t *testing.T) {
|
||||
now := time.Date(2026, 4, 8, 12, 0, 0, 0, time.UTC)
|
||||
p := DefaultRetentionPolicy(365)
|
||||
|
||||
// 10 executions on the same day, ~10 days ago (inside daily window).
|
||||
var execs []*happydns.Execution
|
||||
for i := 0; i < 10; i++ {
|
||||
execs = append(execs, mkExec(fmt.Sprintf("e%d", i), 10*24*time.Hour+time.Duration(i)*time.Hour, now))
|
||||
}
|
||||
|
||||
keep, drop := p.Decide(execs, now)
|
||||
if len(keep) != p.PerDayKept {
|
||||
t.Fatalf("expected %d keeps in daily bucket, got %d", p.PerDayKept, len(keep))
|
||||
}
|
||||
if len(drop) != 10-p.PerDayKept {
|
||||
t.Fatalf("expected %d drops, got %d", 10-p.PerDayKept, len(drop))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecide_HardCutoff(t *testing.T) {
|
||||
now := time.Date(2026, 4, 8, 12, 0, 0, 0, time.UTC)
|
||||
p := DefaultRetentionPolicy(30)
|
||||
|
||||
execs := []*happydns.Execution{
|
||||
mkExec("recent", 1*24*time.Hour, now),
|
||||
mkExec("old", 100*24*time.Hour, now),
|
||||
}
|
||||
|
||||
keep, drop := p.Decide(execs, now)
|
||||
if len(keep) != 1 || string(keep[0]) != "recent" {
|
||||
t.Fatalf("expected 'recent' to be kept, got %v", keep)
|
||||
}
|
||||
if len(drop) != 1 || string(drop[0]) != "old" {
|
||||
t.Fatalf("expected 'old' to be dropped, got %v", drop)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecide_GroupedByTarget(t *testing.T) {
|
||||
now := time.Date(2026, 4, 8, 12, 0, 0, 0, time.UTC)
|
||||
p := DefaultRetentionPolicy(365)
|
||||
|
||||
// 5 executions same day, 10 days ago, two different targets.
|
||||
mk := func(id, dom string) *happydns.Execution {
|
||||
return &happydns.Execution{
|
||||
Id: happydns.Identifier(id),
|
||||
CheckerID: "ping",
|
||||
Target: happydns.CheckTarget{DomainId: dom},
|
||||
StartedAt: now.Add(-10 * 24 * time.Hour),
|
||||
}
|
||||
}
|
||||
var execs []*happydns.Execution
|
||||
for i := 0; i < 5; i++ {
|
||||
execs = append(execs, mk(fmt.Sprintf("a%d", i), "a.example"))
|
||||
execs = append(execs, mk(fmt.Sprintf("b%d", i), "b.example"))
|
||||
}
|
||||
|
||||
keep, _ := p.Decide(execs, now)
|
||||
// PerDayKept per group => 2 * 2 groups = 4
|
||||
if len(keep) != 2*p.PerDayKept {
|
||||
t.Fatalf("expected %d keeps, got %d", 2*p.PerDayKept, len(keep))
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue