Add admin API and frontend for scheduler management
Some checks are pending
continuous-integration/drone/push Build is running

This commit is contained in:
nemunaire 2026-02-11 11:21:20 +07:00
commit e451f35385
12 changed files with 1115 additions and 21 deletions

View file

@ -0,0 +1,102 @@
// This file is part of the happyDomain (R) project.
// Copyright (c) 2020-2026 happyDomain
// Authors: Pierre-Olivier Mercier, et al.
//
// This program is offered under a commercial and under the AGPL license.
// For commercial licensing, contact us at <contact@happydomain.org>.
//
// For AGPL licensing:
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package controller
import (
"net/http"
"github.com/gin-gonic/gin"
"git.happydns.org/happyDomain/model"
)
// AdminSchedulerController handles admin operations on the test scheduler
type AdminSchedulerController struct {
scheduler happydns.AdminSchedulerUsecase
}
func NewAdminSchedulerController(scheduler happydns.AdminSchedulerUsecase) *AdminSchedulerController {
return &AdminSchedulerController{scheduler: scheduler}
}
// GetSchedulerStatus returns the current scheduler state
//
// @Summary Get scheduler status
// @Description Returns the current state of the test scheduler including worker count, queue size, and upcoming schedules
// @Tags scheduler
// @Produce json
// @Success 200 {object} happydns.SchedulerStatus
// @Router /scheduler [get]
func (ctrl *AdminSchedulerController) GetSchedulerStatus(c *gin.Context) {
c.JSON(http.StatusOK, ctrl.scheduler.GetSchedulerStatus())
}
// EnableScheduler enables the test scheduler at runtime
//
// @Summary Enable scheduler
// @Description Enables the test scheduler at runtime without restarting the server
// @Tags scheduler
// @Success 200 {object} happydns.SchedulerStatus
// @Failure 500 {object} happydns.ErrorResponse
// @Router /scheduler/enable [post]
func (ctrl *AdminSchedulerController) EnableScheduler(c *gin.Context) {
if err := ctrl.scheduler.SetEnabled(true); err != nil {
c.JSON(http.StatusInternalServerError, happydns.ErrorResponse{Message: err.Error()})
return
}
c.JSON(http.StatusOK, ctrl.scheduler.GetSchedulerStatus())
}
// DisableScheduler disables the test scheduler at runtime
//
// @Summary Disable scheduler
// @Description Disables the test scheduler at runtime without restarting the server
// @Tags scheduler
// @Success 200 {object} happydns.SchedulerStatus
// @Failure 500 {object} happydns.ErrorResponse
// @Router /scheduler/disable [post]
func (ctrl *AdminSchedulerController) DisableScheduler(c *gin.Context) {
if err := ctrl.scheduler.SetEnabled(false); err != nil {
c.JSON(http.StatusInternalServerError, happydns.ErrorResponse{Message: err.Error()})
return
}
c.JSON(http.StatusOK, ctrl.scheduler.GetSchedulerStatus())
}
// RescheduleUpcoming randomizes the next run time of all enabled schedules
// within their respective intervals to spread load evenly.
//
// @Summary Reschedule upcoming tests
// @Description Randomizes the next run time of all enabled schedules within their intervals to spread load
// @Tags scheduler
// @Produce json
// @Success 200 {object} map[string]int
// @Failure 500 {object} happydns.ErrorResponse
// @Router /scheduler/reschedule-upcoming [post]
func (ctrl *AdminSchedulerController) RescheduleUpcoming(c *gin.Context) {
n, err := ctrl.scheduler.RescheduleUpcomingTests()
if err != nil {
c.JSON(http.StatusInternalServerError, happydns.ErrorResponse{Message: err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"rescheduled": n})
}

View file

@ -36,6 +36,7 @@ func DeclareRoutes(cfg *happydns.Options, router *gin.Engine, s storage.Storage,
declareDomainRoutes(apiRoutes, dependancies, s)
declarePluginsRoutes(apiRoutes, dependancies, s)
declareProviderRoutes(apiRoutes, dependancies, s)
declareSchedulerRoutes(apiRoutes, dependancies)
declareSessionsRoutes(cfg, apiRoutes, s)
declareUserAuthsRoutes(apiRoutes, dependancies, s)
declareUsersRoutes(apiRoutes, dependancies, s)

View file

@ -0,0 +1,45 @@
// This file is part of the happyDomain (R) project.
// Copyright (c) 2020-2026 happyDomain
// Authors: Pierre-Olivier Mercier, et al.
//
// This program is offered under a commercial and under the AGPL license.
// For commercial licensing, contact us at <contact@happydomain.org>.
//
// For AGPL licensing:
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package route
import (
"github.com/gin-gonic/gin"
"git.happydns.org/happyDomain/internal/api-admin/controller"
apicontroller "git.happydns.org/happyDomain/internal/api/controller"
"git.happydns.org/happyDomain/model"
)
// schedulerDepsProvider is satisfied by App which exposes TestScheduler()
type schedulerDepsProvider interface {
TestScheduler() apicontroller.TestSchedulerInterface
}
func declareSchedulerRoutes(router *gin.RouterGroup, dependancies happydns.UsecaseDependancies) {
ctrl := controller.NewAdminSchedulerController(dependancies.TestScheduler())
schedulerRoute := router.Group("/scheduler")
schedulerRoute.GET("", ctrl.GetSchedulerStatus)
schedulerRoute.POST("/enable", ctrl.EnableScheduler)
schedulerRoute.POST("/disable", ctrl.DisableScheduler)
schedulerRoute.POST("/reschedule-upcoming", ctrl.RescheduleUpcoming)
}

View file

@ -45,6 +45,9 @@ type TestResultController struct {
// TestSchedulerInterface defines the interface for triggering on-demand tests
type TestSchedulerInterface interface {
TriggerOnDemandTest(pluginName string, targetType happydns.TestScopeType, targetID happydns.Identifier, userID happydns.Identifier, options happydns.PluginOptions) (happydns.Identifier, error)
GetSchedulerStatus() happydns.SchedulerStatus
SetEnabled(enabled bool) error
RescheduleUpcomingTests() (int, error)
}
func NewTestResultController(

View file

@ -31,7 +31,6 @@ import (
"github.com/gin-gonic/gin"
api "git.happydns.org/happyDomain/internal/api/route"
"git.happydns.org/happyDomain/internal/api/controller"
"git.happydns.org/happyDomain/internal/mailer"
"git.happydns.org/happyDomain/internal/newsletter"
"git.happydns.org/happyDomain/internal/session"
@ -83,7 +82,7 @@ type App struct {
router *gin.Engine
srv *http.Server
insights *insightsCollector
testScheduler controller.TestSchedulerInterface
testScheduler happydns.AdminSchedulerUsecase
plugins happydns.PluginManager
store storage.Storage
usecases Usecases
@ -157,6 +156,10 @@ func (a *App) TestScheduleUsecase() happydns.TestScheduleUsecase {
return a.usecases.testSchedule
}
func (a *App) TestScheduler() happydns.AdminSchedulerUsecase {
return a.testScheduler
}
func (a *App) UserUsecase() happydns.UserUsecase {
return a.usecases.user
}
@ -173,10 +176,6 @@ func (a *App) ZoneUsecase() happydns.ZoneUsecase {
return a.usecases.zone
}
func (a *App) TestScheduler() controller.TestSchedulerInterface {
return a.testScheduler
}
func (a *App) ZoneServiceUsecase() happydns.ZoneServiceUsecase {
return a.usecases.zoneService
}
@ -313,7 +312,7 @@ func (app *App) initUsecases() {
app.usecases.session = sessionService
app.usecases.testPlugin = pluginUC.NewTestPluginUsecase(app.cfg, app.plugins, app.store)
app.usecases.testResult = testresultUC.NewTestResultUsecase(app.store, app.cfg)
app.usecases.testSchedule = testresultUC.NewTestScheduleUsecase(app.store, app.cfg)
app.usecases.testSchedule = testresultUC.NewTestScheduleUsecase(app.store, app.cfg, app.store, app.usecases.testPlugin)
app.usecases.orchestrator = orchestrator.NewOrchestrator(
domainLogService,

View file

@ -35,10 +35,11 @@ import (
)
const (
SchedulerCheckInterval = 1 * time.Minute // How often to check for due tests
SchedulerCleanupInterval = 24 * time.Hour // How often to clean up old executions
TestExecutionTimeout = 5 * time.Minute // Max time for a single test
MaxRetries = 3 // Max retry attempts for failed tests
SchedulerCheckInterval = 1 * time.Minute // How often to check for due tests
SchedulerCleanupInterval = 24 * time.Hour // How often to clean up old executions
SchedulerDiscoveryInterval = 1 * time.Hour // How often to auto-discover new targets
TestExecutionTimeout = 5 * time.Minute // Max time for a single test
MaxRetries = 3 // Max retry attempts for failed tests
)
// Priority levels for test execution queue
@ -62,6 +63,8 @@ type testScheduler struct {
workers []*worker
mu sync.RWMutex
wg sync.WaitGroup
runtimeEnabled bool
running bool
}
// activeExecution tracks a running test execution
@ -147,6 +150,25 @@ func (d *disabledScheduler) TriggerOnDemandTest(pluginName string, targetType ha
return happydns.Identifier{}, fmt.Errorf("test scheduler is disabled in configuration")
}
// GetSchedulerStatus returns a status indicating the scheduler is disabled
func (d *disabledScheduler) GetSchedulerStatus() happydns.SchedulerStatus {
return happydns.SchedulerStatus{
ConfigEnabled: false,
RuntimeEnabled: false,
Running: false,
}
}
// SetEnabled returns an error since the scheduler is disabled in configuration
func (d *disabledScheduler) SetEnabled(enabled bool) error {
return fmt.Errorf("scheduler is disabled in configuration, cannot enable at runtime")
}
// RescheduleUpcomingTests returns an error since the scheduler is disabled
func (d *disabledScheduler) RescheduleUpcomingTests() (int, error) {
return 0, fmt.Errorf("test scheduler is disabled in configuration")
}
// newTestScheduler creates a new test scheduler
func newTestScheduler(
cfg *happydns.Options,
@ -163,12 +185,13 @@ func newTestScheduler(
store: store,
pluginUsecase: pluginUsecase,
resultUsecase: testresult.NewTestResultUsecase(store, cfg),
scheduleUsecase: testresult.NewTestScheduleUsecase(store, cfg),
scheduleUsecase: testresult.NewTestScheduleUsecase(store, cfg, store, pluginUsecase),
stop: make(chan bool),
runNowChan: make(chan *happydns.TestSchedule, 100),
queue: newPriorityQueue(),
activeExecutions: make(map[string]*activeExecution),
workers: make([]*worker, numWorkers),
runtimeEnabled: true,
}
// Create workers
@ -215,8 +238,27 @@ func (s *testScheduler) Run() {
return
}
s.mu.Lock()
s.running = true
s.mu.Unlock()
defer func() {
s.mu.Lock()
s.running = false
s.mu.Unlock()
}()
log.Printf("Starting test scheduler with %d workers...\n", len(s.workers))
// Reschedule overdue tests before starting workers so that tests missed
// during a server suspend or shutdown are spread into the near future
// instead of all firing at once.
if n, err := s.scheduleUsecase.RescheduleOverdueTests(); err != nil {
log.Printf("Warning: failed to reschedule overdue tests: %v\n", err)
} else if n > 0 {
log.Printf("Rescheduled %d overdue test(s) into the near future\n", n)
}
// Start workers
for _, w := range s.workers {
s.wg.Add(1)
@ -226,9 +268,15 @@ func (s *testScheduler) Run() {
// Main scheduling loop
checkTicker := time.NewTicker(SchedulerCheckInterval)
cleanupTicker := time.NewTicker(SchedulerCleanupInterval)
discoveryTicker := time.NewTicker(SchedulerDiscoveryInterval)
defer checkTicker.Stop()
defer cleanupTicker.Stop()
defer discoveryTicker.Stop()
// Initial discovery: create default schedules for all existing targets
if err := s.scheduleUsecase.DiscoverAndEnsureSchedules(); err != nil {
log.Printf("Warning: schedule discovery encountered errors: %v\n", err)
}
// Initial check
s.checkSchedules()
@ -240,6 +288,11 @@ func (s *testScheduler) Run() {
case <-cleanupTicker.C:
s.cleanup()
case <-discoveryTicker.C:
if err := s.scheduleUsecase.DiscoverAndEnsureSchedules(); err != nil {
log.Printf("Warning: schedule discovery encountered errors: %v\n", err)
}
case schedule := <-s.runNowChan:
s.queueOnDemandTest(schedule)
@ -251,6 +304,13 @@ func (s *testScheduler) Run() {
// checkSchedules checks for due tests and queues them
func (s *testScheduler) checkSchedules() {
s.mu.RLock()
enabled := s.runtimeEnabled
s.mu.RUnlock()
if !enabled {
return
}
dueSchedules, err := s.scheduleUsecase.ListDueSchedules()
if err != nil {
log.Printf("Error listing due schedules: %v\n", err)
@ -366,6 +426,41 @@ func (s *testScheduler) TriggerOnDemandTest(pluginName string, targetType happyd
return execution.Id, nil
}
// GetSchedulerStatus returns a snapshot of the current scheduler state
func (s *testScheduler) GetSchedulerStatus() happydns.SchedulerStatus {
s.mu.RLock()
activeCount := len(s.activeExecutions)
running := s.running
runtimeEnabled := s.runtimeEnabled
s.mu.RUnlock()
nextSchedules, _ := s.scheduleUsecase.ListUpcomingSchedules(20)
return happydns.SchedulerStatus{
ConfigEnabled: !s.cfg.DisableScheduler,
RuntimeEnabled: runtimeEnabled,
Running: running,
WorkerCount: len(s.workers),
QueueSize: s.queue.Len(),
ActiveCount: activeCount,
NextSchedules: nextSchedules,
}
}
// SetEnabled enables or disables the scheduler at runtime
func (s *testScheduler) SetEnabled(enabled bool) error {
s.mu.Lock()
s.runtimeEnabled = enabled
s.mu.Unlock()
return nil
}
// RescheduleUpcomingTests randomizes the next run time of all enabled schedules
// within their respective intervals, delegating to the schedule usecase.
func (s *testScheduler) RescheduleUpcomingTests() (int, error) {
return s.scheduleUsecase.RescheduleUpcomingTests()
}
// cleanup removes old execution records and expired test results
func (s *testScheduler) cleanup() {
log.Println("Running scheduler cleanup...")
@ -417,10 +512,21 @@ func (w *worker) executeTest(item *queueItem) {
execution := item.execution
schedule := item.schedule
// Always update schedule NextRun after execution, whether it succeeds or fails.
// This prevents the schedule from being re-queued on the next tick if the test fails.
if item.execution.ScheduleId != nil {
defer func() {
if err := w.scheduler.scheduleUsecase.UpdateScheduleAfterRun(*item.execution.ScheduleId); err != nil {
log.Printf("Worker %d: Error updating schedule after run: %v\n", w.id, err)
}
}()
}
// Mark execution as running
execution.Status = happydns.TestExecutionRunning
if err := w.scheduler.resultUsecase.UpdateTestExecution(execution); err != nil {
log.Printf("Worker %d: Error updating execution status: %v\n", w.id, err)
_ = w.scheduler.resultUsecase.FailTestExecution(execution.Id, err.Error())
return
}
@ -448,6 +554,13 @@ func (w *worker) executeTest(item *queueItem) {
return
}
// Merge options: global defaults < user opts < domain/service opts < schedule opts
mergedOptions, err := w.scheduler.scheduleUsecase.PrepareTestOptions(schedule)
if err != nil {
// Non-fatal: PrepareTestOptions already falls back to schedule-only options
log.Printf("Worker %d: warning, could not prepare plugin options for %s: %v\n", w.id, schedule.PluginName, err)
}
// Prepare metadata
meta := make(map[string]string)
meta["target_type"] = schedule.TargetType.String()
@ -464,7 +577,7 @@ func (w *worker) executeTest(item *queueItem) {
errorChan <- fmt.Errorf("plugin panicked: %v", r)
}
}()
result, err := plugin.RunTest(schedule.Options, meta)
result, err := plugin.RunTest(mergedOptions, meta)
if err != nil {
errorChan <- err
} else {
@ -527,13 +640,6 @@ func (w *worker) executeTest(item *queueItem) {
return
}
// Update schedule if this was a scheduled test
if item.execution.ScheduleId != nil {
if err := w.scheduler.scheduleUsecase.UpdateScheduleAfterRun(*item.execution.ScheduleId); err != nil {
log.Printf("Worker %d: Error updating schedule: %v\n", w.id, err)
}
}
log.Printf("Worker %d: Completed test %s for target %s (status: %s, duration: %v)\n",
log.Printf("Worker %d: Completed test %s for target %s (status: %d, duration: %v)\n",
w.id, schedule.PluginName, schedule.TargetId.String(), result.Status, duration)
}

View file

@ -102,3 +102,8 @@ type TestResultStorage interface {
// LastTestSchedulerRun retrieves the last time the scheduler ran
LastTestSchedulerRun() (*time.Time, error)
}
// DomainLister provides access to domain listings for schedule discovery.
type DomainLister interface {
ListAllDomains() (happydns.Iterator[happydns.Domain], error)
}

View file

@ -0,0 +1,471 @@
// This file is part of the happyDomain (R) project.
// Copyright (c) 2020-2026 happyDomain
// Authors: Pierre-Olivier Mercier, et al.
//
// This program is offered under a commercial and under the AGPL license.
// For commercial licensing, contact us at <contact@happydomain.org>.
//
// For AGPL licensing:
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package testresult
import (
"errors"
"fmt"
"math/rand"
"sort"
"time"
"git.happydns.org/happyDomain/model"
)
const (
// Default test intervals
DefaultUserTestInterval = 4 * time.Hour // 4 hours for domain tests
DefaultDomainTestInterval = 24 * time.Hour // 24 hours for domain tests
DefaultServiceTestInterval = 1 * time.Hour // 1 hour for service tests
MinimumTestInterval = 5 * time.Minute // Minimum interval allowed
)
// TestScheduleUsecase implements business logic for test schedules
type TestScheduleUsecase struct {
storage TestResultStorage
options *happydns.Options
domainLister DomainLister
pluginUsecase happydns.TestPluginUsecase
}
// NewTestScheduleUsecase creates a new test schedule usecase
func NewTestScheduleUsecase(storage TestResultStorage, options *happydns.Options, domainLister DomainLister, pluginUsecase happydns.TestPluginUsecase) *TestScheduleUsecase {
return &TestScheduleUsecase{
storage: storage,
options: options,
domainLister: domainLister,
pluginUsecase: pluginUsecase,
}
}
// ListUserSchedules retrieves all schedules for a specific user
func (u *TestScheduleUsecase) ListUserSchedules(userId happydns.Identifier) ([]*happydns.TestSchedule, error) {
return u.storage.ListTestSchedulesByUser(userId)
}
// ListSchedulesByTarget retrieves all schedules for a specific target
func (u *TestScheduleUsecase) ListSchedulesByTarget(targetType happydns.TestScopeType, targetId happydns.Identifier) ([]*happydns.TestSchedule, error) {
return u.storage.ListTestSchedulesByTarget(targetType, targetId)
}
// GetSchedule retrieves a specific schedule by ID
func (u *TestScheduleUsecase) GetSchedule(scheduleId happydns.Identifier) (*happydns.TestSchedule, error) {
return u.storage.GetTestSchedule(scheduleId)
}
// CreateSchedule creates a new test schedule with validation
func (u *TestScheduleUsecase) CreateSchedule(schedule *happydns.TestSchedule) error {
// Set default interval if not specified
if schedule.Interval == 0 {
schedule.Interval = u.getDefaultInterval(schedule.TargetType)
}
// Validate interval
if schedule.Interval < MinimumTestInterval {
return fmt.Errorf("test interval must be at least %v", MinimumTestInterval)
}
// Calculate next run time: pick a random offset within the interval
// to spread load evenly across all schedules
// TODO: Use a smarter load balance function in the future
if schedule.NextRun.IsZero() {
offset := time.Duration(rand.Int63n(int64(schedule.Interval)))
schedule.NextRun = time.Now().Add(offset)
}
return u.storage.CreateTestSchedule(schedule)
}
// UpdateSchedule updates an existing schedule
func (u *TestScheduleUsecase) UpdateSchedule(schedule *happydns.TestSchedule) error {
// Validate interval
if schedule.Interval < MinimumTestInterval {
return fmt.Errorf("test interval must be at least %v", MinimumTestInterval)
}
// Get existing schedule to preserve certain fields
existing, err := u.storage.GetTestSchedule(schedule.Id)
if err != nil {
return err
}
// Preserve LastRun if not explicitly changed
if schedule.LastRun == nil {
schedule.LastRun = existing.LastRun
}
// Recalculate next run time if interval changed
if schedule.Interval != existing.Interval {
if schedule.LastRun != nil {
schedule.NextRun = schedule.LastRun.Add(schedule.Interval)
} else {
schedule.NextRun = time.Now().Add(schedule.Interval)
}
}
return u.storage.UpdateTestSchedule(schedule)
}
// DeleteSchedule removes a schedule
func (u *TestScheduleUsecase) DeleteSchedule(scheduleId happydns.Identifier) error {
return u.storage.DeleteTestSchedule(scheduleId)
}
// EnableSchedule enables a schedule
func (u *TestScheduleUsecase) EnableSchedule(scheduleId happydns.Identifier) error {
schedule, err := u.storage.GetTestSchedule(scheduleId)
if err != nil {
return err
}
schedule.Enabled = true
// Reset next run time if it's in the past
if schedule.NextRun.Before(time.Now()) {
schedule.NextRun = time.Now().Add(schedule.Interval)
}
return u.storage.UpdateTestSchedule(schedule)
}
// DisableSchedule disables a schedule
func (u *TestScheduleUsecase) DisableSchedule(scheduleId happydns.Identifier) error {
schedule, err := u.storage.GetTestSchedule(scheduleId)
if err != nil {
return err
}
schedule.Enabled = false
return u.storage.UpdateTestSchedule(schedule)
}
// UpdateScheduleAfterRun updates a schedule after it has been executed
func (u *TestScheduleUsecase) UpdateScheduleAfterRun(scheduleId happydns.Identifier) error {
schedule, err := u.storage.GetTestSchedule(scheduleId)
if err != nil {
return err
}
now := time.Now()
schedule.LastRun = &now
schedule.NextRun = now.Add(schedule.Interval)
return u.storage.UpdateTestSchedule(schedule)
}
// ListDueSchedules retrieves all enabled schedules that are due to run
func (u *TestScheduleUsecase) ListDueSchedules() ([]*happydns.TestSchedule, error) {
schedules, err := u.storage.ListEnabledTestSchedules()
if err != nil {
return nil, err
}
now := time.Now()
var dueSchedules []*happydns.TestSchedule
for _, schedule := range schedules {
if schedule.NextRun.Before(now) {
dueSchedules = append(dueSchedules, schedule)
}
}
return dueSchedules, nil
}
// ListUpcomingSchedules retrieves the next `limit` enabled schedules sorted by NextRun ascending
func (u *TestScheduleUsecase) ListUpcomingSchedules(limit int) ([]*happydns.TestSchedule, error) {
schedules, err := u.storage.ListEnabledTestSchedules()
if err != nil {
return nil, err
}
sort.Slice(schedules, func(i, j int) bool {
return schedules[i].NextRun.Before(schedules[j].NextRun)
})
if limit > 0 && len(schedules) > limit {
schedules = schedules[:limit]
}
return schedules, nil
}
// getDefaultInterval returns the default test interval based on target type
func (u *TestScheduleUsecase) getDefaultInterval(targetType happydns.TestScopeType) time.Duration {
switch targetType {
case happydns.TestScopeUser:
return DefaultUserTestInterval
case happydns.TestScopeDomain:
return DefaultDomainTestInterval
case happydns.TestScopeService:
return DefaultServiceTestInterval
default:
return DefaultDomainTestInterval
}
}
// MergePluginOptions merges plugin options from different scopes
// Priority: schedule options > domain options > user options > global options
func (u *TestScheduleUsecase) MergePluginOptions(
globalOpts happydns.PluginOptions,
userOpts happydns.PluginOptions,
domainOpts happydns.PluginOptions,
scheduleOpts happydns.PluginOptions,
) happydns.PluginOptions {
merged := make(happydns.PluginOptions)
// Start with global options
for k, v := range globalOpts {
merged[k] = v
}
// Override with user options
for k, v := range userOpts {
merged[k] = v
}
// Override with domain options
for k, v := range domainOpts {
merged[k] = v
}
// Override with schedule options (highest priority)
for k, v := range scheduleOpts {
merged[k] = v
}
return merged
}
// ValidateScheduleOwnership checks if a user owns a schedule
func (u *TestScheduleUsecase) ValidateScheduleOwnership(scheduleId happydns.Identifier, userId happydns.Identifier) error {
schedule, err := u.storage.GetTestSchedule(scheduleId)
if err != nil {
return err
}
if !schedule.OwnerId.Equals(userId) {
return fmt.Errorf("user does not own this schedule")
}
return nil
}
// CreateDefaultSchedulesForTarget creates default schedules for a new target
func (u *TestScheduleUsecase) CreateDefaultSchedulesForTarget(
pluginName string,
targetType happydns.TestScopeType,
targetId happydns.Identifier,
ownerId happydns.Identifier,
enabled bool,
) error {
schedule := &happydns.TestSchedule{
PluginName: pluginName,
OwnerId: ownerId,
TargetType: targetType,
TargetId: targetId,
Interval: u.getDefaultInterval(targetType),
Enabled: enabled,
NextRun: time.Now().Add(u.getDefaultInterval(targetType)),
Options: make(happydns.PluginOptions),
}
return u.CreateSchedule(schedule)
}
// rescheduleTests reschedules each given schedule to a random time in [now, now+maxOffsetFn(schedule)].
func (u *TestScheduleUsecase) rescheduleTests(schedules []*happydns.TestSchedule, maxOffsetFn func(*happydns.TestSchedule) time.Duration) (int, error) {
count := 0
now := time.Now()
for _, schedule := range schedules {
maxOffset := maxOffsetFn(schedule)
if maxOffset <= 0 {
maxOffset = time.Second
}
schedule.NextRun = now.Add(time.Duration(rand.Int63n(int64(maxOffset))))
if err := u.storage.UpdateTestSchedule(schedule); err != nil {
return count, err
}
count++
}
return count, nil
}
// RescheduleUpcomingTests randomizes the next run time of all enabled schedules
// within their respective intervals to spread load evenly. Useful after a restart.
func (u *TestScheduleUsecase) RescheduleUpcomingTests() (int, error) {
schedules, err := u.storage.ListEnabledTestSchedules()
if err != nil {
return 0, err
}
return u.rescheduleTests(schedules, func(s *happydns.TestSchedule) time.Duration {
return s.Interval
})
}
// RescheduleOverdueTests reschedules tests whose NextRun is in the past,
// spreading them over a short window to avoid scheduler famine (e.g. after
// a long machine suspend or server downtime).
func (u *TestScheduleUsecase) RescheduleOverdueTests() (int, error) {
schedules, err := u.storage.ListEnabledTestSchedules()
if err != nil {
return 0, err
}
now := time.Now()
var overdue []*happydns.TestSchedule
for _, s := range schedules {
if s.NextRun.Before(now) {
overdue = append(overdue, s)
}
}
if len(overdue) == 0 {
return 0, nil
}
// Spread overdue tests over a small window proportional to their count,
// capped at MinimumTestInterval, to prevent all of them from running at once.
spreadWindow := time.Duration(len(overdue)) * 5 * time.Second
if spreadWindow > MinimumTestInterval {
spreadWindow = MinimumTestInterval
}
return u.rescheduleTests(overdue, func(s *happydns.TestSchedule) time.Duration {
return spreadWindow
})
}
// DeleteSchedulesForTarget removes all schedules for a target
func (u *TestScheduleUsecase) DeleteSchedulesForTarget(targetType happydns.TestScopeType, targetId happydns.Identifier) error {
schedules, err := u.storage.ListTestSchedulesByTarget(targetType, targetId)
if err != nil {
return err
}
for _, schedule := range schedules {
if err := u.storage.DeleteTestSchedule(schedule.Id); err != nil {
return err
}
}
return nil
}
// DiscoverAndEnsureSchedules creates default enabled schedules for all (plugin, domain)
// pairs that don't yet have an explicit schedule record. This implements the opt-out
// model: tests run automatically unless a schedule with Enabled=false has been saved.
// Non-fatal per-domain errors are collected and returned together.
func (u *TestScheduleUsecase) DiscoverAndEnsureSchedules() error {
if u.domainLister == nil || u.pluginUsecase == nil {
return nil
}
plugins, err := u.pluginUsecase.ListTestPlugins()
if err != nil {
return fmt.Errorf("listing test plugins for discovery: %w", err)
}
var domainPlugins []happydns.TestPlugin
for _, p := range plugins {
if p.Version().AvailableOn.ApplyToDomain {
domainPlugins = append(domainPlugins, p)
}
}
if len(domainPlugins) == 0 {
return nil
}
iter, err := u.domainLister.ListAllDomains()
if err != nil {
return fmt.Errorf("listing domains for schedule discovery: %w", err)
}
defer iter.Close()
var errs []error
for iter.Next() {
domain := iter.Item()
if domain == nil {
continue
}
for _, plugin := range domainPlugins {
pluginName := plugin.PluginEnvName()[0]
schedules, err := u.ListSchedulesByTarget(happydns.TestScopeDomain, domain.Id)
if err != nil {
errs = append(errs, fmt.Errorf("listing schedules for domain %s: %w", domain.Id, err))
continue
}
hasSchedule := false
for _, sched := range schedules {
if sched.PluginName == pluginName {
hasSchedule = true
break
}
}
if !hasSchedule {
if err := u.CreateSchedule(&happydns.TestSchedule{
PluginName: pluginName,
OwnerId: domain.Owner,
TargetType: happydns.TestScopeDomain,
TargetId: domain.Id,
Enabled: true,
}); err != nil {
errs = append(errs, fmt.Errorf("auto-creating schedule for domain %s / plugin %s: %w",
domain.Id, pluginName, err))
}
}
}
}
return errors.Join(errs...)
}
// PrepareTestOptions fetches and merges plugin options for a scheduled test execution.
// It combines stored options (global/user/domain/service scopes) with the
// schedule-specific overrides, returning the final merged options.
func (u *TestScheduleUsecase) PrepareTestOptions(schedule *happydns.TestSchedule) (happydns.PluginOptions, error) {
if u.pluginUsecase == nil {
return schedule.Options, nil
}
var domainId, serviceId *happydns.Identifier
switch schedule.TargetType {
case happydns.TestScopeDomain:
domainId = &schedule.TargetId
case happydns.TestScopeService:
serviceId = &schedule.TargetId
}
baseOptions, err := u.pluginUsecase.GetTestPluginOptions(schedule.PluginName, &schedule.OwnerId, domainId, serviceId)
if err != nil {
// Non-fatal: fall back to schedule-only options and surface as a warning
return schedule.Options, fmt.Errorf("could not fetch plugin options for %s: %w", schedule.PluginName, err)
}
if baseOptions != nil {
return u.MergePluginOptions(nil, nil, *baseOptions, schedule.Options), nil
}
return schedule.Options, nil
}

View file

@ -191,6 +191,30 @@ type TestExecution struct {
Options PluginOptions `json:"options,omitempty"`
}
// SchedulerStatus holds a snapshot of the scheduler state for monitoring
type SchedulerStatus struct {
// ConfigEnabled indicates if the scheduler is enabled in the configuration file
ConfigEnabled bool `json:"config_enabled"`
// RuntimeEnabled indicates if the scheduler is currently enabled at runtime
RuntimeEnabled bool `json:"runtime_enabled"`
// Running indicates if the scheduler goroutine is currently running
Running bool `json:"running"`
// WorkerCount is the number of worker goroutines
WorkerCount int `json:"worker_count"`
// QueueSize is the number of items currently waiting in the execution queue
QueueSize int `json:"queue_size"`
// ActiveCount is the number of tests currently being executed
ActiveCount int `json:"active_count"`
// NextSchedules contains the upcoming scheduled tests sorted by next run time
NextSchedules []*TestSchedule `json:"next_schedules"`
}
// TestResultUsecase defines business logic for test results
type TestResultUsecase interface {
// ListTestResultsByTarget retrieves test results for a specific target

View file

@ -37,6 +37,7 @@ type UsecaseDependancies interface {
TestPluginUsecase() TestPluginUsecase
TestResultUsecase() TestResultUsecase
TestScheduleUsecase() TestScheduleUsecase
TestScheduler() AdminSchedulerUsecase
UserUsecase() UserUsecase
ZoneCorrectionApplierUsecase() ZoneCorrectionApplierUsecase
ZoneImporterUsecase() ZoneImporterUsecase

View file

@ -104,6 +104,9 @@
<NavItem>
<NavLink href="/plugins" active={page && page.url.pathname.startsWith('/plugins')}>Plugins</NavLink>
</NavItem>
<NavItem>
<NavLink href="/scheduler" active={page && page.url.pathname.startsWith('/scheduler')}>Scheduler</NavLink>
</NavItem>
</Nav>
</Collapse>
</Navbar>

View file

@ -0,0 +1,334 @@
<!--
This file is part of the happyDomain (R) project.
Copyright (c) 2022-2026 happyDomain
Authors: Pierre-Olivier Mercier, et al.
This program is offered under a commercial and under the AGPL license.
For commercial licensing, contact us at <contact@happydomain.org>.
For AGPL licensing:
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
-->
<script lang="ts">
import { onMount } from "svelte";
import {
Badge,
Button,
Card,
CardBody,
CardHeader,
Col,
Container,
Icon,
Row,
Spinner,
Table,
} from "@sveltestrap/sveltestrap";
import { toasts } from "$lib/stores/toasts";
import { getScheduler, postSchedulerDisable, postSchedulerEnable, postSchedulerRescheduleUpcoming } from "$lib/api-admin/sdk.gen";
interface TestSchedule {
id: string;
plugin_name: string;
user_id: string;
target_type: number;
target_id: string;
interval: number;
enabled: boolean;
last_run?: string;
next_run: string;
}
interface SchedulerStatus {
config_enabled: boolean;
runtime_enabled: boolean;
running: boolean;
worker_count: number;
queue_size: number;
active_count: number;
next_schedules: TestSchedule[] | null;
}
let status = $state<SchedulerStatus | null>(null);
let loading = $state(true);
let actionInProgress = $state(false);
let rescheduleInProgress = $state(false);
let error = $state<string | null>(null);
async function fetchStatus() {
loading = true;
error = null;
try {
const { data, error: err } = await getScheduler();
if (err) throw new Error(String(err));
status = data as SchedulerStatus;
} catch (e: any) {
error = e.message ?? "Unknown error";
} finally {
loading = false;
}
}
async function setEnabled(enabled: boolean) {
actionInProgress = true;
const action = enabled ? "enable" : "disable";
try {
const { data, error: err } = await (enabled ? postSchedulerEnable() : postSchedulerDisable());
if (err) {
toasts.addErrorToast({ message: `Failed to ${action} scheduler: ${err}` });
return;
}
status = data as SchedulerStatus;
toasts.addToast({ message: `Scheduler ${action}d successfully`, color: "success" });
} catch (e: any) {
toasts.addErrorToast({ message: e.message ?? `Failed to ${action} scheduler` });
} finally {
actionInProgress = false;
}
}
async function rescheduleUpcoming() {
rescheduleInProgress = true;
try {
const { data, error: err } = await postSchedulerRescheduleUpcoming();
if (err) {
toasts.addErrorToast({ message: `Failed to reschedule: ${err}` });
return;
}
toasts.addToast({
message: `Rescheduled ${(data as any).rescheduled} schedule(s) successfully`,
color: "success",
});
await fetchStatus();
} catch (e: any) {
toasts.addErrorToast({ message: e.message ?? "Failed to reschedule upcoming tests" });
} finally {
rescheduleInProgress = false;
}
}
function formatDuration(ns: number): string {
const seconds = ns / 1e9;
if (seconds < 60) return `${Math.round(seconds)}s`;
const minutes = seconds / 60;
if (minutes < 60) return `${Math.round(minutes)}m`;
const hours = minutes / 60;
if (hours < 24) return `${Math.round(hours)}h`;
return `${Math.round(hours / 24)}d`;
}
function targetTypeName(t: number): string {
const names: Record<number, string> = {
0: "instance",
1: "user",
2: "domain",
3: "zone",
4: "service",
5: "ondemand",
};
return names[t] ?? "unknown";
}
onMount(fetchStatus);
</script>
<Container class="flex-fill my-5">
<Row class="mb-4">
<Col>
<h1 class="display-5">
<Icon name="clock-history"></Icon>
Test Scheduler
</h1>
<p class="text-muted lead">Monitor and control the background test scheduler</p>
</Col>
</Row>
{#if loading}
<div class="d-flex align-items-center gap-2">
<Spinner size="sm" />
<span>Loading scheduler status...</span>
</div>
{:else if error}
<Card color="danger" body>
<Icon name="exclamation-triangle-fill"></Icon>
Error loading scheduler status: {error}
<Button class="ms-3" size="sm" color="light" onclick={fetchStatus}>Retry</Button>
</Card>
{:else if status}
<!-- Status Card -->
<Card class="mb-4">
<CardHeader>
<div class="d-flex justify-content-between align-items-center">
<span><Icon name="info-circle-fill"></Icon> Scheduler Status</span>
<Button size="sm" color="secondary" outline onclick={fetchStatus}>
<Icon name="arrow-clockwise"></Icon> Refresh
</Button>
</div>
</CardHeader>
<CardBody>
<Row class="g-3 mb-3">
<Col sm={6} md={4}>
<div class="text-muted small">Config Enabled</div>
{#if status.config_enabled}
<Badge color="success">Yes</Badge>
{:else}
<Badge color="danger">No</Badge>
{/if}
</Col>
<Col sm={6} md={4}>
<div class="text-muted small">Runtime Enabled</div>
{#if status.runtime_enabled}
<Badge color="success">Yes</Badge>
{:else}
<Badge color="warning">Disabled</Badge>
{/if}
</Col>
<Col sm={6} md={4}>
<div class="text-muted small">Running</div>
{#if status.running}
<Badge color="success"><Icon name="play-fill"></Icon> Running</Badge>
{:else}
<Badge color="secondary"><Icon name="stop-fill"></Icon> Stopped</Badge>
{/if}
</Col>
<Col sm={6} md={4}>
<div class="text-muted small">Workers</div>
<strong>{status.worker_count}</strong>
</Col>
<Col sm={6} md={4}>
<div class="text-muted small">Queue Size</div>
<strong>{status.queue_size}</strong>
</Col>
<Col sm={6} md={4}>
<div class="text-muted small">Active Executions</div>
<strong>{status.active_count}</strong>
</Col>
</Row>
{#if status.config_enabled}
<div class="d-flex gap-2">
{#if status.runtime_enabled}
<Button
color="warning"
disabled={actionInProgress}
onclick={() => setEnabled(false)}
>
{#if actionInProgress}<Spinner size="sm" />{:else}<Icon
name="pause-fill"
></Icon>{/if}
Disable Scheduler
</Button>
{:else}
<Button
color="success"
disabled={actionInProgress}
onclick={() => setEnabled(true)}
>
{#if actionInProgress}<Spinner size="sm" />{:else}<Icon
name="play-fill"
></Icon>{/if}
Enable Scheduler
</Button>
{/if}
<Button
color="secondary"
outline
disabled={rescheduleInProgress}
onclick={rescheduleUpcoming}
>
{#if rescheduleInProgress}<Spinner size="sm" />{:else}<Icon
name="shuffle"
></Icon>{/if}
Spread Upcoming Tests
</Button>
</div>
{:else}
<p class="text-muted mb-0">
<Icon name="lock-fill"></Icon>
The scheduler is disabled in the server configuration and cannot be enabled at
runtime.
</p>
{/if}
</CardBody>
</Card>
<!-- Upcoming Scheduled Tests -->
<Card>
<CardHeader>
<Icon name="calendar-event-fill"></Icon>
Upcoming Scheduled Tests
{#if status.next_schedules}
<Badge color="secondary" class="ms-2">{status.next_schedules.length}</Badge>
{/if}
</CardHeader>
<CardBody class="p-0">
<div class="table-responsive">
<Table hover class="mb-0">
<thead>
<tr>
<th>Plugin</th>
<th>Target Type</th>
<th>Target ID</th>
<th>Interval</th>
<th>Last Run</th>
<th>Next Run</th>
</tr>
</thead>
<tbody>
{#if !status.next_schedules || status.next_schedules.length === 0}
<tr>
<td colspan="6" class="text-center text-muted py-3">
No scheduled tests
</td>
</tr>
{:else}
{#each status.next_schedules as schedule}
<tr>
<td><strong>{schedule.plugin_name}</strong></td>
<td
><Badge color="info"
>{targetTypeName(schedule.target_type)}</Badge
></td
>
<td><code class="small">{schedule.target_id}</code></td>
<td>{formatDuration(schedule.interval)}</td>
<td>
{#if schedule.last_run}
{new Date(schedule.last_run).toLocaleString()}
{:else}
<span class="text-muted">Never</span>
{/if}
</td>
<td>
{#if new Date(schedule.next_run) < new Date()}
<span class="text-danger">
<Icon name="exclamation-circle-fill"></Icon>
{new Date(schedule.next_run).toLocaleString()}
</span>
{:else}
{new Date(schedule.next_run).toLocaleString()}
{/if}
</td>
</tr>
{/each}
{/if}
</tbody>
</Table>
</div>
</CardBody>
</Card>
{/if}
</Container>