Add admin API and frontend for scheduler management

internal/api-admin/controller/scheduler_controller.go

@@ -0,0 +1,102 @@
+// This file is part of the happyDomain (R) project.
+// Copyright (c) 2020-2026 happyDomain
+// Authors: Pierre-Olivier Mercier, et al.
+//
+// This program is offered under a commercial and under the AGPL license.
+// For commercial licensing, contact us at <contact@happydomain.org>.
+//
+// For AGPL licensing:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package controller
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+
+	"git.happydns.org/happyDomain/model"
+)
+
+// AdminSchedulerController handles admin operations on the test scheduler
+type AdminSchedulerController struct {
+	scheduler happydns.AdminSchedulerUsecase
+}
+
+func NewAdminSchedulerController(scheduler happydns.AdminSchedulerUsecase) *AdminSchedulerController {
+	return &AdminSchedulerController{scheduler: scheduler}
+}
+
+// GetSchedulerStatus returns the current scheduler state
+//
+//	@Summary		Get scheduler status
+//	@Description	Returns the current state of the test scheduler including worker count, queue size, and upcoming schedules
+//	@Tags			scheduler
+//	@Produce		json
+//	@Success		200	{object}	happydns.SchedulerStatus
+//	@Router			/scheduler [get]
+func (ctrl *AdminSchedulerController) GetSchedulerStatus(c *gin.Context) {
+	c.JSON(http.StatusOK, ctrl.scheduler.GetSchedulerStatus())
+}
+
+// EnableScheduler enables the test scheduler at runtime
+//
+//	@Summary		Enable scheduler
+//	@Description	Enables the test scheduler at runtime without restarting the server
+//	@Tags			scheduler
+//	@Success		200	{object}	happydns.SchedulerStatus
+//	@Failure		500	{object}	happydns.ErrorResponse
+//	@Router			/scheduler/enable [post]
+func (ctrl *AdminSchedulerController) EnableScheduler(c *gin.Context) {
+	if err := ctrl.scheduler.SetEnabled(true); err != nil {
+		c.JSON(http.StatusInternalServerError, happydns.ErrorResponse{Message: err.Error()})
+		return
+	}
+	c.JSON(http.StatusOK, ctrl.scheduler.GetSchedulerStatus())
+}
+
+// DisableScheduler disables the test scheduler at runtime
+//
+//	@Summary		Disable scheduler
+//	@Description	Disables the test scheduler at runtime without restarting the server
+//	@Tags			scheduler
+//	@Success		200	{object}	happydns.SchedulerStatus
+//	@Failure		500	{object}	happydns.ErrorResponse
+//	@Router			/scheduler/disable [post]
+func (ctrl *AdminSchedulerController) DisableScheduler(c *gin.Context) {
+	if err := ctrl.scheduler.SetEnabled(false); err != nil {
+		c.JSON(http.StatusInternalServerError, happydns.ErrorResponse{Message: err.Error()})
+		return
+	}
+	c.JSON(http.StatusOK, ctrl.scheduler.GetSchedulerStatus())
+}
+
+// RescheduleUpcoming randomizes the next run time of all enabled schedules
+// within their respective intervals to spread load evenly.
+//
+//	@Summary		Reschedule upcoming tests
+//	@Description	Randomizes the next run time of all enabled schedules within their intervals to spread load
+//	@Tags			scheduler
+//	@Produce		json
+//	@Success		200	{object}	map[string]int
+//	@Failure		500	{object}	happydns.ErrorResponse
+//	@Router			/scheduler/reschedule-upcoming [post]
+func (ctrl *AdminSchedulerController) RescheduleUpcoming(c *gin.Context) {
+	n, err := ctrl.scheduler.RescheduleUpcomingTests()
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, happydns.ErrorResponse{Message: err.Error()})
+		return
+	}
+	c.JSON(http.StatusOK, gin.H{"rescheduled": n})
+}

internal/api-admin/route/route.go

@@ -36,6 +36,7 @@ func DeclareRoutes(cfg *happydns.Options, router *gin.Engine, s storage.Storage,
 	declareDomainRoutes(apiRoutes, dependancies, s)
 	declarePluginsRoutes(apiRoutes, dependancies, s)
 	declareProviderRoutes(apiRoutes, dependancies, s)
+	declareSchedulerRoutes(apiRoutes, dependancies)
 	declareSessionsRoutes(cfg, apiRoutes, s)
 	declareUserAuthsRoutes(apiRoutes, dependancies, s)
 	declareUsersRoutes(apiRoutes, dependancies, s)

internal/api-admin/route/scheduler.go

@@ -0,0 +1,45 @@
+// This file is part of the happyDomain (R) project.
+// Copyright (c) 2020-2026 happyDomain
+// Authors: Pierre-Olivier Mercier, et al.
+//
+// This program is offered under a commercial and under the AGPL license.
+// For commercial licensing, contact us at <contact@happydomain.org>.
+//
+// For AGPL licensing:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package route
+
+import (
+	"github.com/gin-gonic/gin"
+
+	"git.happydns.org/happyDomain/internal/api-admin/controller"
+	apicontroller "git.happydns.org/happyDomain/internal/api/controller"
+	"git.happydns.org/happyDomain/model"
+)
+
+// schedulerDepsProvider is satisfied by App which exposes TestScheduler()
+type schedulerDepsProvider interface {
+	TestScheduler() apicontroller.TestSchedulerInterface
+}
+
+func declareSchedulerRoutes(router *gin.RouterGroup, dependancies happydns.UsecaseDependancies) {
+	ctrl := controller.NewAdminSchedulerController(dependancies.TestScheduler())
+
+	schedulerRoute := router.Group("/scheduler")
+	schedulerRoute.GET("", ctrl.GetSchedulerStatus)
+	schedulerRoute.POST("/enable", ctrl.EnableScheduler)
+	schedulerRoute.POST("/disable", ctrl.DisableScheduler)
+	schedulerRoute.POST("/reschedule-upcoming", ctrl.RescheduleUpcoming)
+}

internal/api/controller/testresult_controller.go

@@ -45,6 +45,9 @@ type TestResultController struct {
 // TestSchedulerInterface defines the interface for triggering on-demand tests
 type TestSchedulerInterface interface {
 	TriggerOnDemandTest(pluginName string, targetType happydns.TestScopeType, targetID happydns.Identifier, userID happydns.Identifier, options happydns.PluginOptions) (happydns.Identifier, error)
+	GetSchedulerStatus() happydns.SchedulerStatus
+	SetEnabled(enabled bool) error
+	RescheduleUpcomingTests() (int, error)
 }
 
 func NewTestResultController(

internal/app/app.go

@@ -31,7 +31,6 @@ import (
 	"github.com/gin-gonic/gin"
 
 	api "git.happydns.org/happyDomain/internal/api/route"
-	"git.happydns.org/happyDomain/internal/api/controller"
 	"git.happydns.org/happyDomain/internal/mailer"
 	"git.happydns.org/happyDomain/internal/newsletter"
 	"git.happydns.org/happyDomain/internal/session"
@@ -83,7 +82,7 @@ type App struct {
 	router        *gin.Engine
 	srv           *http.Server
 	insights      *insightsCollector
-	testScheduler controller.TestSchedulerInterface
+	testScheduler happydns.AdminSchedulerUsecase
 	plugins       happydns.PluginManager
 	store         storage.Storage
 	usecases      Usecases
@@ -157,6 +156,10 @@ func (a *App) TestScheduleUsecase() happydns.TestScheduleUsecase {
 	return a.usecases.testSchedule
 }
 
+func (a *App) TestScheduler() happydns.AdminSchedulerUsecase {
+	return a.testScheduler
+}
+
 func (a *App) UserUsecase() happydns.UserUsecase {
 	return a.usecases.user
 }
@@ -173,10 +176,6 @@ func (a *App) ZoneUsecase() happydns.ZoneUsecase {
 	return a.usecases.zone
 }
 
-func (a *App) TestScheduler() controller.TestSchedulerInterface {
-	return a.testScheduler
-}
-
 func (a *App) ZoneServiceUsecase() happydns.ZoneServiceUsecase {
 	return a.usecases.zoneService
 }

internal/app/testscheduler.go

@@ -35,10 +35,11 @@ import (
 )
 
 const (
-	SchedulerCheckInterval   = 1 * time.Minute // How often to check for due tests
+	SchedulerCheckInterval     = 1 * time.Minute // How often to check for due tests
-	SchedulerCleanupInterval = 24 * time.Hour  // How often to clean up old executions
+	SchedulerCleanupInterval   = 24 * time.Hour  // How often to clean up old executions
-	TestExecutionTimeout     = 5 * time.Minute // Max time for a single test
+	SchedulerDiscoveryInterval = 1 * time.Hour   // How often to auto-discover new targets
-	MaxRetries               = 3               // Max retry attempts for failed tests
+	TestExecutionTimeout       = 5 * time.Minute // Max time for a single test
+	MaxRetries                 = 3               // Max retry attempts for failed tests
 )
 
 // Priority levels for test execution queue
@@ -62,6 +63,8 @@ type testScheduler struct {
 	workers          []*worker
 	mu               sync.RWMutex
 	wg               sync.WaitGroup
+	runtimeEnabled   bool
+	running          bool
 }
 
 // activeExecution tracks a running test execution
@@ -147,6 +150,25 @@ func (d *disabledScheduler) TriggerOnDemandTest(pluginName string, targetType ha
 	return happydns.Identifier{}, fmt.Errorf("test scheduler is disabled in configuration")
 }
 
+// GetSchedulerStatus returns a status indicating the scheduler is disabled
+func (d *disabledScheduler) GetSchedulerStatus() happydns.SchedulerStatus {
+	return happydns.SchedulerStatus{
+		ConfigEnabled:  false,
+		RuntimeEnabled: false,
+		Running:        false,
+	}
+}
+
+// SetEnabled returns an error since the scheduler is disabled in configuration
+func (d *disabledScheduler) SetEnabled(enabled bool) error {
+	return fmt.Errorf("scheduler is disabled in configuration, cannot enable at runtime")
+}
+
+// RescheduleUpcomingTests returns an error since the scheduler is disabled
+func (d *disabledScheduler) RescheduleUpcomingTests() (int, error) {
+	return 0, fmt.Errorf("test scheduler is disabled in configuration")
+}
+
 // newTestScheduler creates a new test scheduler
 func newTestScheduler(
 	cfg *happydns.Options,
@@ -169,6 +191,7 @@ func newTestScheduler(
 		queue:            newPriorityQueue(),
 		activeExecutions: make(map[string]*activeExecution),
 		workers:          make([]*worker, numWorkers),
+		runtimeEnabled:   true,
 	}
 
 	// Create workers
@@ -215,8 +238,27 @@ func (s *testScheduler) Run() {
 		return
 	}
 
+	s.mu.Lock()
+	s.running = true
+	s.mu.Unlock()
+
+	defer func() {
+		s.mu.Lock()
+		s.running = false
+		s.mu.Unlock()
+	}()
+
 	log.Printf("Starting test scheduler with %d workers...\n", len(s.workers))
 
+	// Reschedule overdue tests before starting workers so that tests missed
+	// during a server suspend or shutdown are spread into the near future
+	// instead of all firing at once.
+	if n, err := s.scheduleUsecase.RescheduleOverdueTests(); err != nil {
+		log.Printf("Warning: failed to reschedule overdue tests: %v\n", err)
+	} else if n > 0 {
+		log.Printf("Rescheduled %d overdue test(s) into the near future\n", n)
+	}
+
 	// Start workers
 	for _, w := range s.workers {
 		s.wg.Add(1)
@@ -226,9 +268,13 @@ func (s *testScheduler) Run() {
 	// Main scheduling loop
 	checkTicker := time.NewTicker(SchedulerCheckInterval)
 	cleanupTicker := time.NewTicker(SchedulerCleanupInterval)
+	discoveryTicker := time.NewTicker(SchedulerDiscoveryInterval)
 	defer checkTicker.Stop()
 	defer cleanupTicker.Stop()
+	defer discoveryTicker.Stop()
 
+	// Initial discovery: create default schedules for all existing targets
+	s.discoverAndEnsureSchedules()
 	// Initial check
 	s.checkSchedules()
 
@@ -240,6 +286,9 @@ func (s *testScheduler) Run() {
 		case <-cleanupTicker.C:
 			s.cleanup()
 
+		case <-discoveryTicker.C:
+			s.discoverAndEnsureSchedules()
+
 		case schedule := <-s.runNowChan:
 			s.queueOnDemandTest(schedule)
 
@@ -251,6 +300,13 @@ func (s *testScheduler) Run() {
 
 // checkSchedules checks for due tests and queues them
 func (s *testScheduler) checkSchedules() {
+	s.mu.RLock()
+	enabled := s.runtimeEnabled
+	s.mu.RUnlock()
+	if !enabled {
+		return
+	}
+
 	dueSchedules, err := s.scheduleUsecase.ListDueSchedules()
 	if err != nil {
 		log.Printf("Error listing due schedules: %v\n", err)
@@ -298,6 +354,80 @@ func (s *testScheduler) checkSchedules() {
 	}
 }
 
+// discoverAndEnsureSchedules creates default (enabled) schedules for all
+// (plugin, target) pairs that don't yet have an explicit schedule record.
+// This implements the opt-out model: tests run automatically unless a schedule
+// with Enabled=false has been explicitly saved.
+func (s *testScheduler) discoverAndEnsureSchedules() {
+	s.mu.RLock()
+	enabled := s.runtimeEnabled
+	s.mu.RUnlock()
+	if !enabled {
+		return
+	}
+
+	plugins, err := s.pluginUsecase.ListTestPlugins()
+	if err != nil {
+		log.Printf("Error listing test plugins for discovery: %v\n", err)
+		return
+	}
+
+	// Filter domain-level plugins
+	var domainPlugins []happydns.TestPlugin
+	for _, p := range plugins {
+		if p.Version().AvailableOn.ApplyToDomain {
+			domainPlugins = append(domainPlugins, p)
+		}
+	}
+
+	if len(domainPlugins) > 0 {
+		iter, err := s.store.ListAllDomains()
+		if err != nil {
+			log.Printf("Error listing domains for schedule discovery: %v\n", err)
+		} else {
+			defer iter.Close()
+			for iter.Next() {
+				domain := iter.Item()
+				if domain == nil {
+					continue
+				}
+				for _, plugin := range domainPlugins {
+					pluginName := plugin.Version().Name
+					schedules, err := s.scheduleUsecase.ListSchedulesByTarget(happydns.TestScopeDomain, domain.Id)
+					if err != nil {
+						continue
+					}
+
+					hasSchedule := false
+					for _, sched := range schedules {
+						if sched.PluginName == pluginName {
+							hasSchedule = true
+							break
+						}
+					}
+
+					if !hasSchedule {
+						if err := s.scheduleUsecase.CreateSchedule(&happydns.TestSchedule{
+							PluginName: pluginName,
+							OwnerId:    domain.Owner,
+							TargetType: happydns.TestScopeDomain,
+							TargetId:   domain.Id,
+							Enabled:    true,
+						}); err != nil {
+							log.Printf("Error auto-creating schedule for domain %s / plugin %s: %v\n",
+								domain.Id, pluginName, err)
+						}
+					}
+				}
+			}
+		}
+	}
+
+	// Service-level plugin discovery is deferred: services live inside zones
+	// and enumeration would require iterating all zones across all domains.
+	// Services get auto-scheduled on their first explicit interaction instead.
+}
+
 // queueOnDemandTest queues an on-demand test execution
 func (s *testScheduler) queueOnDemandTest(schedule *happydns.TestSchedule) {
 	execution := &happydns.TestExecution{
@@ -366,6 +496,41 @@ func (s *testScheduler) TriggerOnDemandTest(pluginName string, targetType happyd
 	return execution.Id, nil
 }
 
+// GetSchedulerStatus returns a snapshot of the current scheduler state
+func (s *testScheduler) GetSchedulerStatus() happydns.SchedulerStatus {
+	s.mu.RLock()
+	activeCount := len(s.activeExecutions)
+	running := s.running
+	runtimeEnabled := s.runtimeEnabled
+	s.mu.RUnlock()
+
+	nextSchedules, _ := s.scheduleUsecase.ListUpcomingSchedules(20)
+
+	return happydns.SchedulerStatus{
+		ConfigEnabled:  !s.cfg.DisableScheduler,
+		RuntimeEnabled: runtimeEnabled,
+		Running:        running,
+		WorkerCount:    len(s.workers),
+		QueueSize:      s.queue.Len(),
+		ActiveCount:    activeCount,
+		NextSchedules:  nextSchedules,
+	}
+}
+
+// SetEnabled enables or disables the scheduler at runtime
+func (s *testScheduler) SetEnabled(enabled bool) error {
+	s.mu.Lock()
+	s.runtimeEnabled = enabled
+	s.mu.Unlock()
+	return nil
+}
+
+// RescheduleUpcomingTests randomizes the next run time of all enabled schedules
+// within their respective intervals, delegating to the schedule usecase.
+func (s *testScheduler) RescheduleUpcomingTests() (int, error) {
+	return s.scheduleUsecase.RescheduleUpcomingTests()
+}
+
 // cleanup removes old execution records
 func (s *testScheduler) cleanup() {
 	// This is a placeholder for cleanup logic
@@ -409,10 +574,21 @@ func (w *worker) executeTest(item *queueItem) {
 	execution := item.execution
 	schedule := item.schedule
 
+	// Always update schedule NextRun after execution, whether it succeeds or fails.
+	// This prevents the schedule from being re-queued on the next tick if the test fails.
+	if item.execution.ScheduleId != nil {
+		defer func() {
+			if err := w.scheduler.scheduleUsecase.UpdateScheduleAfterRun(*item.execution.ScheduleId); err != nil {
+				log.Printf("Worker %d: Error updating schedule after run: %v\n", w.id, err)
+			}
+		}()
+	}
+
 	// Mark execution as running
 	execution.Status = happydns.TestExecutionRunning
 	if err := w.scheduler.resultUsecase.UpdateTestExecution(execution); err != nil {
 		log.Printf("Worker %d: Error updating execution status: %v\n", w.id, err)
+		_ = w.scheduler.resultUsecase.FailTestExecution(execution.Id, err.Error())
 		return
 	}
 
@@ -514,13 +690,6 @@ func (w *worker) executeTest(item *queueItem) {
 		return
 	}
 
-	// Update schedule if this was a scheduled test
+	log.Printf("Worker %d: Completed test %s for target %s (status: %d, duration: %v)\n",
-	if item.execution.ScheduleId != nil {
-		if err := w.scheduler.scheduleUsecase.UpdateScheduleAfterRun(*item.execution.ScheduleId); err != nil {
-			log.Printf("Worker %d: Error updating schedule: %v\n", w.id, err)
-		}
-	}
-
-	log.Printf("Worker %d: Completed test %s for target %s (status: %s, duration: %v)\n",
 		w.id, schedule.PluginName, schedule.TargetId, result.Status, duration)
 }

internal/usecase/testresult/testschedule_usecase.go

@@ -0,0 +1,367 @@
+// This file is part of the happyDomain (R) project.
+// Copyright (c) 2020-2026 happyDomain
+// Authors: Pierre-Olivier Mercier, et al.
+//
+// This program is offered under a commercial and under the AGPL license.
+// For commercial licensing, contact us at <contact@happydomain.org>.
+//
+// For AGPL licensing:
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package testresult
+
+import (
+	"fmt"
+	"math/rand"
+	"sort"
+	"time"
+
+	"git.happydns.org/happyDomain/model"
+)
+
+const (
+	// Default test intervals
+	DefaultUserTestInterval    = 4 * time.Hour   // 4 hours for domain tests
+	DefaultDomainTestInterval  = 24 * time.Hour  // 24 hours for domain tests
+	DefaultServiceTestInterval = 1 * time.Hour   // 1 hour for service tests
+	MinimumTestInterval        = 5 * time.Minute // Minimum interval allowed
+)
+
+// TestScheduleUsecase implements business logic for test schedules
+type TestScheduleUsecase struct {
+	storage TestResultStorage
+	options *happydns.Options
+}
+
+// NewTestScheduleUsecase creates a new test schedule usecase
+func NewTestScheduleUsecase(storage TestResultStorage, options *happydns.Options) *TestScheduleUsecase {
+	return &TestScheduleUsecase{
+		storage: storage,
+		options: options,
+	}
+}
+
+// ListUserSchedules retrieves all schedules for a specific user
+func (u *TestScheduleUsecase) ListUserSchedules(userId happydns.Identifier) ([]*happydns.TestSchedule, error) {
+	return u.storage.ListTestSchedulesByUser(userId)
+}
+
+// ListSchedulesByTarget retrieves all schedules for a specific target
+func (u *TestScheduleUsecase) ListSchedulesByTarget(targetType happydns.TestScopeType, targetId happydns.Identifier) ([]*happydns.TestSchedule, error) {
+	return u.storage.ListTestSchedulesByTarget(targetType, targetId)
+}
+
+// GetSchedule retrieves a specific schedule by ID
+func (u *TestScheduleUsecase) GetSchedule(scheduleId happydns.Identifier) (*happydns.TestSchedule, error) {
+	return u.storage.GetTestSchedule(scheduleId)
+}
+
+// CreateSchedule creates a new test schedule with validation
+func (u *TestScheduleUsecase) CreateSchedule(schedule *happydns.TestSchedule) error {
+	// Set default interval if not specified
+	if schedule.Interval == 0 {
+		schedule.Interval = u.getDefaultInterval(schedule.TargetType)
+	}
+
+	// Validate interval
+	if schedule.Interval < MinimumTestInterval {
+		return fmt.Errorf("test interval must be at least %v", MinimumTestInterval)
+	}
+
+	// Calculate next run time: pick a random offset within the interval
+	// to spread load evenly across all schedules
+	// TODO: Use a smarter load balance function in the future
+	if schedule.NextRun.IsZero() {
+		offset := time.Duration(rand.Int63n(int64(schedule.Interval)))
+		schedule.NextRun = time.Now().Add(offset)
+	}
+
+	return u.storage.CreateTestSchedule(schedule)
+}
+
+// UpdateSchedule updates an existing schedule
+func (u *TestScheduleUsecase) UpdateSchedule(schedule *happydns.TestSchedule) error {
+	// Validate interval
+	if schedule.Interval < MinimumTestInterval {
+		return fmt.Errorf("test interval must be at least %v", MinimumTestInterval)
+	}
+
+	// Get existing schedule to preserve certain fields
+	existing, err := u.storage.GetTestSchedule(schedule.Id)
+	if err != nil {
+		return err
+	}
+
+	// Preserve LastRun if not explicitly changed
+	if schedule.LastRun == nil {
+		schedule.LastRun = existing.LastRun
+	}
+
+	// Recalculate next run time if interval changed
+	if schedule.Interval != existing.Interval {
+		if schedule.LastRun != nil {
+			schedule.NextRun = schedule.LastRun.Add(schedule.Interval)
+		} else {
+			schedule.NextRun = time.Now().Add(schedule.Interval)
+		}
+	}
+
+	return u.storage.UpdateTestSchedule(schedule)
+}
+
+// DeleteSchedule removes a schedule
+func (u *TestScheduleUsecase) DeleteSchedule(scheduleId happydns.Identifier) error {
+	return u.storage.DeleteTestSchedule(scheduleId)
+}
+
+// EnableSchedule enables a schedule
+func (u *TestScheduleUsecase) EnableSchedule(scheduleId happydns.Identifier) error {
+	schedule, err := u.storage.GetTestSchedule(scheduleId)
+	if err != nil {
+		return err
+	}
+
+	schedule.Enabled = true
+
+	// Reset next run time if it's in the past
+	if schedule.NextRun.Before(time.Now()) {
+		schedule.NextRun = time.Now().Add(schedule.Interval)
+	}
+
+	return u.storage.UpdateTestSchedule(schedule)
+}
+
+// DisableSchedule disables a schedule
+func (u *TestScheduleUsecase) DisableSchedule(scheduleId happydns.Identifier) error {
+	schedule, err := u.storage.GetTestSchedule(scheduleId)
+	if err != nil {
+		return err
+	}
+
+	schedule.Enabled = false
+	return u.storage.UpdateTestSchedule(schedule)
+}
+
+// UpdateScheduleAfterRun updates a schedule after it has been executed
+func (u *TestScheduleUsecase) UpdateScheduleAfterRun(scheduleId happydns.Identifier) error {
+	schedule, err := u.storage.GetTestSchedule(scheduleId)
+	if err != nil {
+		return err
+	}
+
+	now := time.Now()
+	schedule.LastRun = &now
+	schedule.NextRun = now.Add(schedule.Interval)
+
+	return u.storage.UpdateTestSchedule(schedule)
+}
+
+// ListDueSchedules retrieves all enabled schedules that are due to run
+func (u *TestScheduleUsecase) ListDueSchedules() ([]*happydns.TestSchedule, error) {
+	schedules, err := u.storage.ListEnabledTestSchedules()
+	if err != nil {
+		return nil, err
+	}
+
+	now := time.Now()
+	var dueSchedules []*happydns.TestSchedule
+
+	for _, schedule := range schedules {
+		if schedule.NextRun.Before(now) {
+			dueSchedules = append(dueSchedules, schedule)
+		}
+	}
+
+	return dueSchedules, nil
+}
+
+// ListUpcomingSchedules retrieves the next `limit` enabled schedules sorted by NextRun ascending
+func (u *TestScheduleUsecase) ListUpcomingSchedules(limit int) ([]*happydns.TestSchedule, error) {
+	schedules, err := u.storage.ListEnabledTestSchedules()
+	if err != nil {
+		return nil, err
+	}
+
+	sort.Slice(schedules, func(i, j int) bool {
+		return schedules[i].NextRun.Before(schedules[j].NextRun)
+	})
+
+	if limit > 0 && len(schedules) > limit {
+		schedules = schedules[:limit]
+	}
+
+	return schedules, nil
+}
+
+// getDefaultInterval returns the default test interval based on target type
+func (u *TestScheduleUsecase) getDefaultInterval(targetType happydns.TestScopeType) time.Duration {
+	switch targetType {
+	case happydns.TestScopeUser:
+		return DefaultUserTestInterval
+	case happydns.TestScopeDomain:
+		return DefaultDomainTestInterval
+	case happydns.TestScopeService:
+		return DefaultServiceTestInterval
+	default:
+		return DefaultDomainTestInterval
+	}
+}
+
+// MergePluginOptions merges plugin options from different scopes
+// Priority: schedule options > domain options > user options > global options
+func (u *TestScheduleUsecase) MergePluginOptions(
+	globalOpts happydns.PluginOptions,
+	userOpts happydns.PluginOptions,
+	domainOpts happydns.PluginOptions,
+	scheduleOpts happydns.PluginOptions,
+) happydns.PluginOptions {
+	merged := make(happydns.PluginOptions)
+
+	// Start with global options
+	for k, v := range globalOpts {
+		merged[k] = v
+	}
+
+	// Override with user options
+	for k, v := range userOpts {
+		merged[k] = v
+	}
+
+	// Override with domain options
+	for k, v := range domainOpts {
+		merged[k] = v
+	}
+
+	// Override with schedule options (highest priority)
+	for k, v := range scheduleOpts {
+		merged[k] = v
+	}
+
+	return merged
+}
+
+// ValidateScheduleOwnership checks if a user owns a schedule
+func (u *TestScheduleUsecase) ValidateScheduleOwnership(scheduleId happydns.Identifier, userId happydns.Identifier) error {
+	schedule, err := u.storage.GetTestSchedule(scheduleId)
+	if err != nil {
+		return err
+	}
+
+	if !schedule.OwnerId.Equals(userId) {
+		return fmt.Errorf("user does not own this schedule")
+	}
+
+	return nil
+}
+
+// CreateDefaultSchedulesForTarget creates default schedules for a new target
+func (u *TestScheduleUsecase) CreateDefaultSchedulesForTarget(
+	pluginName string,
+	targetType happydns.TestScopeType,
+	targetId happydns.Identifier,
+	ownerId happydns.Identifier,
+	enabled bool,
+) error {
+	schedule := &happydns.TestSchedule{
+		PluginName: pluginName,
+		OwnerId:    ownerId,
+		TargetType: targetType,
+		TargetId:   targetId,
+		Interval:   u.getDefaultInterval(targetType),
+		Enabled:    enabled,
+		NextRun:    time.Now().Add(u.getDefaultInterval(targetType)),
+		Options:    make(happydns.PluginOptions),
+	}
+
+	return u.CreateSchedule(schedule)
+}
+
+// rescheduleTests reschedules each given schedule to a random time in [now, now+maxOffsetFn(schedule)].
+func (u *TestScheduleUsecase) rescheduleTests(schedules []*happydns.TestSchedule, maxOffsetFn func(*happydns.TestSchedule) time.Duration) (int, error) {
+	count := 0
+	now := time.Now()
+	for _, schedule := range schedules {
+		maxOffset := maxOffsetFn(schedule)
+		if maxOffset <= 0 {
+			maxOffset = time.Second
+		}
+		schedule.NextRun = now.Add(time.Duration(rand.Int63n(int64(maxOffset))))
+		if err := u.storage.UpdateTestSchedule(schedule); err != nil {
+			return count, err
+		}
+		count++
+	}
+	return count, nil
+}
+
+// RescheduleUpcomingTests randomizes the next run time of all enabled schedules
+// within their respective intervals to spread load evenly. Useful after a restart.
+func (u *TestScheduleUsecase) RescheduleUpcomingTests() (int, error) {
+	schedules, err := u.storage.ListEnabledTestSchedules()
+	if err != nil {
+		return 0, err
+	}
+	return u.rescheduleTests(schedules, func(s *happydns.TestSchedule) time.Duration {
+		return s.Interval
+	})
+}
+
+// RescheduleOverdueTests reschedules tests whose NextRun is in the past,
+// spreading them over a short window to avoid scheduler famine (e.g. after
+// a long machine suspend or server downtime).
+func (u *TestScheduleUsecase) RescheduleOverdueTests() (int, error) {
+	schedules, err := u.storage.ListEnabledTestSchedules()
+	if err != nil {
+		return 0, err
+	}
+
+	now := time.Now()
+	var overdue []*happydns.TestSchedule
+	for _, s := range schedules {
+		if s.NextRun.Before(now) {
+			overdue = append(overdue, s)
+		}
+	}
+
+	if len(overdue) == 0 {
+		return 0, nil
+	}
+
+	// Spread overdue tests over a small window proportional to their count,
+	// capped at MinimumTestInterval, to prevent all of them from running at once.
+	spreadWindow := time.Duration(len(overdue)) * 5 * time.Second
+	if spreadWindow > MinimumTestInterval {
+		spreadWindow = MinimumTestInterval
+	}
+
+	return u.rescheduleTests(overdue, func(s *happydns.TestSchedule) time.Duration {
+		return spreadWindow
+	})
+}
+
+// DeleteSchedulesForTarget removes all schedules for a target
+func (u *TestScheduleUsecase) DeleteSchedulesForTarget(targetType happydns.TestScopeType, targetId happydns.Identifier) error {
+	schedules, err := u.storage.ListTestSchedulesByTarget(targetType, targetId)
+	if err != nil {
+		return err
+	}
+
+	for _, schedule := range schedules {
+		if err := u.storage.DeleteTestSchedule(schedule.Id); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

model/test_result.go

@@ -191,6 +191,30 @@ type TestExecution struct {
 	Options PluginOptions `json:"options,omitempty"`
 }
 
+// SchedulerStatus holds a snapshot of the scheduler state for monitoring
+type SchedulerStatus struct {
+	// ConfigEnabled indicates if the scheduler is enabled in the configuration file
+	ConfigEnabled bool `json:"config_enabled"`
+
+	// RuntimeEnabled indicates if the scheduler is currently enabled at runtime
+	RuntimeEnabled bool `json:"runtime_enabled"`
+
+	// Running indicates if the scheduler goroutine is currently running
+	Running bool `json:"running"`
+
+	// WorkerCount is the number of worker goroutines
+	WorkerCount int `json:"worker_count"`
+
+	// QueueSize is the number of items currently waiting in the execution queue
+	QueueSize int `json:"queue_size"`
+
+	// ActiveCount is the number of tests currently being executed
+	ActiveCount int `json:"active_count"`
+
+	// NextSchedules contains the upcoming scheduled tests sorted by next run time
+	NextSchedules []*TestSchedule `json:"next_schedules"`
+}
+
 // TestResultUsecase defines business logic for test results
 type TestResultUsecase interface {
 	// ListTestResultsByTarget retrieves test results for a specific target

model/usecase.go

@@ -37,6 +37,7 @@ type UsecaseDependancies interface {
 	TestPluginUsecase() TestPluginUsecase
 	TestResultUsecase() TestResultUsecase
 	TestScheduleUsecase() TestScheduleUsecase
+	TestScheduler() AdminSchedulerUsecase
 	UserUsecase() UserUsecase
 	ZoneCorrectionApplierUsecase() ZoneCorrectionApplierUsecase
 	ZoneImporterUsecase() ZoneImporterUsecase

web-admin/src/routes/+layout.svelte

@@ -104,6 +104,9 @@
             <NavItem>
                 <NavLink href="/plugins" active={page && page.url.pathname.startsWith('/plugins')}>Plugins</NavLink>
             </NavItem>
+            <NavItem>
+                <NavLink href="/scheduler" active={page && page.url.pathname.startsWith('/scheduler')}>Scheduler</NavLink>
+            </NavItem>
         </Nav>
     </Collapse>
 </Navbar>

web-admin/src/routes/scheduler/+page.svelte

@@ -0,0 +1,334 @@
+<!--
+     This file is part of the happyDomain (R) project.
+     Copyright (c) 2022-2026 happyDomain
+     Authors: Pierre-Olivier Mercier, et al.
+
+     This program is offered under a commercial and under the AGPL license.
+     For commercial licensing, contact us at <contact@happydomain.org>.
+
+     For AGPL licensing:
+     This program is free software: you can redistribute it and/or modify
+     it under the terms of the GNU Affero General Public License as published by
+     the Free Software Foundation, either version 3 of the License, or
+     (at your option) any later version.
+
+     This program is distributed in the hope that it will be useful,
+     but WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+     GNU Affero General Public License for more details.
+
+     You should have received a copy of the GNU Affero General Public License
+     along with this program.  If not, see <https://www.gnu.org/licenses/>.
+-->
+
+<script lang="ts">
+    import { onMount } from "svelte";
+    import {
+        Badge,
+        Button,
+        Card,
+        CardBody,
+        CardHeader,
+        Col,
+        Container,
+        Icon,
+        Row,
+        Spinner,
+        Table,
+    } from "@sveltestrap/sveltestrap";
+
+    import { toasts } from "$lib/stores/toasts";
+    import { getScheduler, postSchedulerDisable, postSchedulerEnable, postSchedulerRescheduleUpcoming } from "$lib/api-admin/sdk.gen";
+
+    interface TestSchedule {
+        id: string;
+        plugin_name: string;
+        user_id: string;
+        target_type: number;
+        target_id: string;
+        interval: number;
+        enabled: boolean;
+        last_run?: string;
+        next_run: string;
+    }
+
+    interface SchedulerStatus {
+        config_enabled: boolean;
+        runtime_enabled: boolean;
+        running: boolean;
+        worker_count: number;
+        queue_size: number;
+        active_count: number;
+        next_schedules: TestSchedule[] | null;
+    }
+
+    let status = $state<SchedulerStatus | null>(null);
+    let loading = $state(true);
+    let actionInProgress = $state(false);
+    let rescheduleInProgress = $state(false);
+    let error = $state<string | null>(null);
+
+    async function fetchStatus() {
+        loading = true;
+        error = null;
+        try {
+            const { data, error: err } = await getScheduler();
+            if (err) throw new Error(String(err));
+            status = data as SchedulerStatus;
+        } catch (e: any) {
+            error = e.message ?? "Unknown error";
+        } finally {
+            loading = false;
+        }
+    }
+
+    async function setEnabled(enabled: boolean) {
+        actionInProgress = true;
+        const action = enabled ? "enable" : "disable";
+        try {
+            const { data, error: err } = await (enabled ? postSchedulerEnable() : postSchedulerDisable());
+            if (err) {
+                toasts.addErrorToast({ message: `Failed to ${action} scheduler: ${err}` });
+                return;
+            }
+            status = data as SchedulerStatus;
+            toasts.addToast({ message: `Scheduler ${action}d successfully`, color: "success" });
+        } catch (e: any) {
+            toasts.addErrorToast({ message: e.message ?? `Failed to ${action} scheduler` });
+        } finally {
+            actionInProgress = false;
+        }
+    }
+
+    async function rescheduleUpcoming() {
+        rescheduleInProgress = true;
+        try {
+            const { data, error: err } = await postSchedulerRescheduleUpcoming();
+            if (err) {
+                toasts.addErrorToast({ message: `Failed to reschedule: ${err}` });
+                return;
+            }
+            toasts.addToast({
+                message: `Rescheduled ${(data as any).rescheduled} schedule(s) successfully`,
+                color: "success",
+            });
+            await fetchStatus();
+        } catch (e: any) {
+            toasts.addErrorToast({ message: e.message ?? "Failed to reschedule upcoming tests" });
+        } finally {
+            rescheduleInProgress = false;
+        }
+    }
+
+    function formatDuration(ns: number): string {
+        const seconds = ns / 1e9;
+        if (seconds < 60) return `${Math.round(seconds)}s`;
+        const minutes = seconds / 60;
+        if (minutes < 60) return `${Math.round(minutes)}m`;
+        const hours = minutes / 60;
+        if (hours < 24) return `${Math.round(hours)}h`;
+        return `${Math.round(hours / 24)}d`;
+    }
+
+    function targetTypeName(t: number): string {
+        const names: Record<number, string> = {
+            0: "instance",
+            1: "user",
+            2: "domain",
+            3: "zone",
+            4: "service",
+            5: "ondemand",
+        };
+        return names[t] ?? "unknown";
+    }
+
+    onMount(fetchStatus);
+</script>
+
+<Container class="flex-fill my-5">
+    <Row class="mb-4">
+        <Col>
+            <h1 class="display-5">
+                <Icon name="clock-history"></Icon>
+                Test Scheduler
+            </h1>
+            <p class="text-muted lead">Monitor and control the background test scheduler</p>
+        </Col>
+    </Row>
+
+    {#if loading}
+        <div class="d-flex align-items-center gap-2">
+            <Spinner size="sm" />
+            <span>Loading scheduler status...</span>
+        </div>
+    {:else if error}
+        <Card color="danger" body>
+            <Icon name="exclamation-triangle-fill"></Icon>
+            Error loading scheduler status: {error}
+            <Button class="ms-3" size="sm" color="light" onclick={fetchStatus}>Retry</Button>
+        </Card>
+    {:else if status}
+        <!-- Status Card -->
+        <Card class="mb-4">
+            <CardHeader>
+                <div class="d-flex justify-content-between align-items-center">
+                    <span><Icon name="info-circle-fill"></Icon> Scheduler Status</span>
+                    <Button size="sm" color="secondary" outline onclick={fetchStatus}>
+                        <Icon name="arrow-clockwise"></Icon> Refresh
+                    </Button>
+                </div>
+            </CardHeader>
+            <CardBody>
+                <Row class="g-3 mb-3">
+                    <Col sm={6} md={4}>
+                        <div class="text-muted small">Config Enabled</div>
+                        {#if status.config_enabled}
+                            <Badge color="success">Yes</Badge>
+                        {:else}
+                            <Badge color="danger">No</Badge>
+                        {/if}
+                    </Col>
+                    <Col sm={6} md={4}>
+                        <div class="text-muted small">Runtime Enabled</div>
+                        {#if status.runtime_enabled}
+                            <Badge color="success">Yes</Badge>
+                        {:else}
+                            <Badge color="warning">Disabled</Badge>
+                        {/if}
+                    </Col>
+                    <Col sm={6} md={4}>
+                        <div class="text-muted small">Running</div>
+                        {#if status.running}
+                            <Badge color="success"><Icon name="play-fill"></Icon> Running</Badge>
+                        {:else}
+                            <Badge color="secondary"><Icon name="stop-fill"></Icon> Stopped</Badge>
+                        {/if}
+                    </Col>
+                    <Col sm={6} md={4}>
+                        <div class="text-muted small">Workers</div>
+                        <strong>{status.worker_count}</strong>
+                    </Col>
+                    <Col sm={6} md={4}>
+                        <div class="text-muted small">Queue Size</div>
+                        <strong>{status.queue_size}</strong>
+                    </Col>
+                    <Col sm={6} md={4}>
+                        <div class="text-muted small">Active Executions</div>
+                        <strong>{status.active_count}</strong>
+                    </Col>
+                </Row>
+
+                {#if status.config_enabled}
+                    <div class="d-flex gap-2">
+                        {#if status.runtime_enabled}
+                            <Button
+                                color="warning"
+                                disabled={actionInProgress}
+                                onclick={() => setEnabled(false)}
+                            >
+                                {#if actionInProgress}<Spinner size="sm" />{:else}<Icon
+                                        name="pause-fill"
+                                    ></Icon>{/if}
+                                Disable Scheduler
+                            </Button>
+                        {:else}
+                            <Button
+                                color="success"
+                                disabled={actionInProgress}
+                                onclick={() => setEnabled(true)}
+                            >
+                                {#if actionInProgress}<Spinner size="sm" />{:else}<Icon
+                                        name="play-fill"
+                                    ></Icon>{/if}
+                                Enable Scheduler
+                            </Button>
+                        {/if}
+                        <Button
+                            color="secondary"
+                            outline
+                            disabled={rescheduleInProgress}
+                            onclick={rescheduleUpcoming}
+                        >
+                            {#if rescheduleInProgress}<Spinner size="sm" />{:else}<Icon
+                                    name="shuffle"
+                                ></Icon>{/if}
+                            Spread Upcoming Tests
+                        </Button>
+                    </div>
+                {:else}
+                    <p class="text-muted mb-0">
+                        <Icon name="lock-fill"></Icon>
+                        The scheduler is disabled in the server configuration and cannot be enabled at
+                        runtime.
+                    </p>
+                {/if}
+            </CardBody>
+        </Card>
+
+        <!-- Upcoming Scheduled Tests -->
+        <Card>
+            <CardHeader>
+                <Icon name="calendar-event-fill"></Icon>
+                Upcoming Scheduled Tests
+                {#if status.next_schedules}
+                    <Badge color="secondary" class="ms-2">{status.next_schedules.length}</Badge>
+                {/if}
+            </CardHeader>
+            <CardBody class="p-0">
+                <div class="table-responsive">
+                    <Table hover class="mb-0">
+                        <thead>
+                            <tr>
+                                <th>Plugin</th>
+                                <th>Target Type</th>
+                                <th>Target ID</th>
+                                <th>Interval</th>
+                                <th>Last Run</th>
+                                <th>Next Run</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                            {#if !status.next_schedules || status.next_schedules.length === 0}
+                                <tr>
+                                    <td colspan="6" class="text-center text-muted py-3">
+                                        No scheduled tests
+                                    </td>
+                                </tr>
+                            {:else}
+                                {#each status.next_schedules as schedule}
+                                    <tr>
+                                        <td><strong>{schedule.plugin_name}</strong></td>
+                                        <td
+                                            ><Badge color="info"
+                                                >{targetTypeName(schedule.target_type)}</Badge
+                                            ></td
+                                        >
+                                        <td><code class="small">{schedule.target_id}</code></td>
+                                        <td>{formatDuration(schedule.interval)}</td>
+                                        <td>
+                                            {#if schedule.last_run}
+                                                {new Date(schedule.last_run).toLocaleString()}
+                                            {:else}
+                                                <span class="text-muted">Never</span>
+                                            {/if}
+                                        </td>
+                                        <td>
+                                            {#if new Date(schedule.next_run) < new Date()}
+                                                <span class="text-danger">
+                                                    <Icon name="exclamation-circle-fill"></Icon>
+                                                    {new Date(schedule.next_run).toLocaleString()}
+                                                </span>
+                                            {:else}
+                                                {new Date(schedule.next_run).toLocaleString()}
+                                            {/if}
+                                        </td>
+                                    </tr>
+                                {/each}
+                            {/if}
+                        </tbody>
+                    </Table>
+                </div>
+            </CardBody>
+        </Card>
+    {/if}
+</Container>