From c19cf16480b4435a2e8643e0984644f53930a333 Mon Sep 17 00:00:00 2001 From: Pierre-Olivier Mercier Date: Wed, 11 Feb 2026 11:21:20 +0700 Subject: [PATCH] Add admin API and frontend for scheduler management --- .../controller/scheduler_controller.go | 102 +++++ internal/api-admin/route/route.go | 1 + internal/api-admin/route/scheduler.go | 45 +++ .../api/controller/testresult_controller.go | 3 + internal/app/app.go | 11 +- internal/app/testscheduler.go | 193 ++++++++- .../testresult/testschedule_usecase.go | 367 ++++++++++++++++++ model/test_result.go | 24 ++ model/usecase.go | 1 + web-admin/src/routes/+layout.svelte | 3 + web-admin/src/routes/scheduler/+page.svelte | 334 ++++++++++++++++ 11 files changed, 1066 insertions(+), 18 deletions(-) create mode 100644 internal/api-admin/controller/scheduler_controller.go create mode 100644 internal/api-admin/route/scheduler.go create mode 100644 internal/usecase/testresult/testschedule_usecase.go create mode 100644 web-admin/src/routes/scheduler/+page.svelte diff --git a/internal/api-admin/controller/scheduler_controller.go b/internal/api-admin/controller/scheduler_controller.go new file mode 100644 index 00000000..e173ebd3 --- /dev/null +++ b/internal/api-admin/controller/scheduler_controller.go @@ -0,0 +1,102 @@ +// This file is part of the happyDomain (R) project. +// Copyright (c) 2020-2026 happyDomain +// Authors: Pierre-Olivier Mercier, et al. +// +// This program is offered under a commercial and under the AGPL license. +// For commercial licensing, contact us at . +// +// For AGPL licensing: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package controller + +import ( + "net/http" + + "github.com/gin-gonic/gin" + + "git.happydns.org/happyDomain/model" +) + +// AdminSchedulerController handles admin operations on the test scheduler +type AdminSchedulerController struct { + scheduler happydns.AdminSchedulerUsecase +} + +func NewAdminSchedulerController(scheduler happydns.AdminSchedulerUsecase) *AdminSchedulerController { + return &AdminSchedulerController{scheduler: scheduler} +} + +// GetSchedulerStatus returns the current scheduler state +// +// @Summary Get scheduler status +// @Description Returns the current state of the test scheduler including worker count, queue size, and upcoming schedules +// @Tags scheduler +// @Produce json +// @Success 200 {object} happydns.SchedulerStatus +// @Router /scheduler [get] +func (ctrl *AdminSchedulerController) GetSchedulerStatus(c *gin.Context) { + c.JSON(http.StatusOK, ctrl.scheduler.GetSchedulerStatus()) +} + +// EnableScheduler enables the test scheduler at runtime +// +// @Summary Enable scheduler +// @Description Enables the test scheduler at runtime without restarting the server +// @Tags scheduler +// @Success 200 {object} happydns.SchedulerStatus +// @Failure 500 {object} happydns.ErrorResponse +// @Router /scheduler/enable [post] +func (ctrl *AdminSchedulerController) EnableScheduler(c *gin.Context) { + if err := ctrl.scheduler.SetEnabled(true); err != nil { + c.JSON(http.StatusInternalServerError, happydns.ErrorResponse{Message: err.Error()}) + return + } + c.JSON(http.StatusOK, ctrl.scheduler.GetSchedulerStatus()) +} + +// DisableScheduler disables the test scheduler at runtime +// +// @Summary Disable scheduler +// @Description Disables the test scheduler at runtime without restarting the server +// @Tags scheduler +// @Success 200 {object} happydns.SchedulerStatus +// @Failure 500 {object} happydns.ErrorResponse +// @Router /scheduler/disable [post] +func (ctrl *AdminSchedulerController) DisableScheduler(c *gin.Context) { + if err := ctrl.scheduler.SetEnabled(false); err != nil { + c.JSON(http.StatusInternalServerError, happydns.ErrorResponse{Message: err.Error()}) + return + } + c.JSON(http.StatusOK, ctrl.scheduler.GetSchedulerStatus()) +} + +// RescheduleUpcoming randomizes the next run time of all enabled schedules +// within their respective intervals to spread load evenly. +// +// @Summary Reschedule upcoming tests +// @Description Randomizes the next run time of all enabled schedules within their intervals to spread load +// @Tags scheduler +// @Produce json +// @Success 200 {object} map[string]int +// @Failure 500 {object} happydns.ErrorResponse +// @Router /scheduler/reschedule-upcoming [post] +func (ctrl *AdminSchedulerController) RescheduleUpcoming(c *gin.Context) { + n, err := ctrl.scheduler.RescheduleUpcomingTests() + if err != nil { + c.JSON(http.StatusInternalServerError, happydns.ErrorResponse{Message: err.Error()}) + return + } + c.JSON(http.StatusOK, gin.H{"rescheduled": n}) +} diff --git a/internal/api-admin/route/route.go b/internal/api-admin/route/route.go index 9c8f168e..a3ae338e 100644 --- a/internal/api-admin/route/route.go +++ b/internal/api-admin/route/route.go @@ -36,6 +36,7 @@ func DeclareRoutes(cfg *happydns.Options, router *gin.Engine, s storage.Storage, declareDomainRoutes(apiRoutes, dependancies, s) declarePluginsRoutes(apiRoutes, dependancies, s) declareProviderRoutes(apiRoutes, dependancies, s) + declareSchedulerRoutes(apiRoutes, dependancies) declareSessionsRoutes(cfg, apiRoutes, s) declareUserAuthsRoutes(apiRoutes, dependancies, s) declareUsersRoutes(apiRoutes, dependancies, s) diff --git a/internal/api-admin/route/scheduler.go b/internal/api-admin/route/scheduler.go new file mode 100644 index 00000000..84258903 --- /dev/null +++ b/internal/api-admin/route/scheduler.go @@ -0,0 +1,45 @@ +// This file is part of the happyDomain (R) project. +// Copyright (c) 2020-2026 happyDomain +// Authors: Pierre-Olivier Mercier, et al. +// +// This program is offered under a commercial and under the AGPL license. +// For commercial licensing, contact us at . +// +// For AGPL licensing: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package route + +import ( + "github.com/gin-gonic/gin" + + "git.happydns.org/happyDomain/internal/api-admin/controller" + apicontroller "git.happydns.org/happyDomain/internal/api/controller" + "git.happydns.org/happyDomain/model" +) + +// schedulerDepsProvider is satisfied by App which exposes TestScheduler() +type schedulerDepsProvider interface { + TestScheduler() apicontroller.TestSchedulerInterface +} + +func declareSchedulerRoutes(router *gin.RouterGroup, dependancies happydns.UsecaseDependancies) { + ctrl := controller.NewAdminSchedulerController(dependancies.TestScheduler()) + + schedulerRoute := router.Group("/scheduler") + schedulerRoute.GET("", ctrl.GetSchedulerStatus) + schedulerRoute.POST("/enable", ctrl.EnableScheduler) + schedulerRoute.POST("/disable", ctrl.DisableScheduler) + schedulerRoute.POST("/reschedule-upcoming", ctrl.RescheduleUpcoming) +} diff --git a/internal/api/controller/testresult_controller.go b/internal/api/controller/testresult_controller.go index 251db5e3..7aad3bd5 100644 --- a/internal/api/controller/testresult_controller.go +++ b/internal/api/controller/testresult_controller.go @@ -45,6 +45,9 @@ type TestResultController struct { // TestSchedulerInterface defines the interface for triggering on-demand tests type TestSchedulerInterface interface { TriggerOnDemandTest(pluginName string, targetType happydns.TestScopeType, targetID happydns.Identifier, userID happydns.Identifier, options happydns.PluginOptions) (happydns.Identifier, error) + GetSchedulerStatus() happydns.SchedulerStatus + SetEnabled(enabled bool) error + RescheduleUpcomingTests() (int, error) } func NewTestResultController( diff --git a/internal/app/app.go b/internal/app/app.go index fff10247..315caaa7 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -31,7 +31,6 @@ import ( "github.com/gin-gonic/gin" api "git.happydns.org/happyDomain/internal/api/route" - "git.happydns.org/happyDomain/internal/api/controller" "git.happydns.org/happyDomain/internal/mailer" "git.happydns.org/happyDomain/internal/newsletter" "git.happydns.org/happyDomain/internal/session" @@ -83,7 +82,7 @@ type App struct { router *gin.Engine srv *http.Server insights *insightsCollector - testScheduler controller.TestSchedulerInterface + testScheduler happydns.AdminSchedulerUsecase plugins happydns.PluginManager store storage.Storage usecases Usecases @@ -157,6 +156,10 @@ func (a *App) TestScheduleUsecase() happydns.TestScheduleUsecase { return a.usecases.testSchedule } +func (a *App) TestScheduler() happydns.AdminSchedulerUsecase { + return a.testScheduler +} + func (a *App) UserUsecase() happydns.UserUsecase { return a.usecases.user } @@ -173,10 +176,6 @@ func (a *App) ZoneUsecase() happydns.ZoneUsecase { return a.usecases.zone } -func (a *App) TestScheduler() controller.TestSchedulerInterface { - return a.testScheduler -} - func (a *App) ZoneServiceUsecase() happydns.ZoneServiceUsecase { return a.usecases.zoneService } diff --git a/internal/app/testscheduler.go b/internal/app/testscheduler.go index a44811c6..e4205e7a 100644 --- a/internal/app/testscheduler.go +++ b/internal/app/testscheduler.go @@ -35,10 +35,11 @@ import ( ) const ( - SchedulerCheckInterval = 1 * time.Minute // How often to check for due tests - SchedulerCleanupInterval = 24 * time.Hour // How often to clean up old executions - TestExecutionTimeout = 5 * time.Minute // Max time for a single test - MaxRetries = 3 // Max retry attempts for failed tests + SchedulerCheckInterval = 1 * time.Minute // How often to check for due tests + SchedulerCleanupInterval = 24 * time.Hour // How often to clean up old executions + SchedulerDiscoveryInterval = 1 * time.Hour // How often to auto-discover new targets + TestExecutionTimeout = 5 * time.Minute // Max time for a single test + MaxRetries = 3 // Max retry attempts for failed tests ) // Priority levels for test execution queue @@ -62,6 +63,8 @@ type testScheduler struct { workers []*worker mu sync.RWMutex wg sync.WaitGroup + runtimeEnabled bool + running bool } // activeExecution tracks a running test execution @@ -147,6 +150,25 @@ func (d *disabledScheduler) TriggerOnDemandTest(pluginName string, targetType ha return happydns.Identifier{}, fmt.Errorf("test scheduler is disabled in configuration") } +// GetSchedulerStatus returns a status indicating the scheduler is disabled +func (d *disabledScheduler) GetSchedulerStatus() happydns.SchedulerStatus { + return happydns.SchedulerStatus{ + ConfigEnabled: false, + RuntimeEnabled: false, + Running: false, + } +} + +// SetEnabled returns an error since the scheduler is disabled in configuration +func (d *disabledScheduler) SetEnabled(enabled bool) error { + return fmt.Errorf("scheduler is disabled in configuration, cannot enable at runtime") +} + +// RescheduleUpcomingTests returns an error since the scheduler is disabled +func (d *disabledScheduler) RescheduleUpcomingTests() (int, error) { + return 0, fmt.Errorf("test scheduler is disabled in configuration") +} + // newTestScheduler creates a new test scheduler func newTestScheduler( cfg *happydns.Options, @@ -169,6 +191,7 @@ func newTestScheduler( queue: newPriorityQueue(), activeExecutions: make(map[string]*activeExecution), workers: make([]*worker, numWorkers), + runtimeEnabled: true, } // Create workers @@ -215,8 +238,27 @@ func (s *testScheduler) Run() { return } + s.mu.Lock() + s.running = true + s.mu.Unlock() + + defer func() { + s.mu.Lock() + s.running = false + s.mu.Unlock() + }() + log.Printf("Starting test scheduler with %d workers...\n", len(s.workers)) + // Reschedule overdue tests before starting workers so that tests missed + // during a server suspend or shutdown are spread into the near future + // instead of all firing at once. + if n, err := s.scheduleUsecase.RescheduleOverdueTests(); err != nil { + log.Printf("Warning: failed to reschedule overdue tests: %v\n", err) + } else if n > 0 { + log.Printf("Rescheduled %d overdue test(s) into the near future\n", n) + } + // Start workers for _, w := range s.workers { s.wg.Add(1) @@ -226,9 +268,13 @@ func (s *testScheduler) Run() { // Main scheduling loop checkTicker := time.NewTicker(SchedulerCheckInterval) cleanupTicker := time.NewTicker(SchedulerCleanupInterval) + discoveryTicker := time.NewTicker(SchedulerDiscoveryInterval) defer checkTicker.Stop() defer cleanupTicker.Stop() + defer discoveryTicker.Stop() + // Initial discovery: create default schedules for all existing targets + s.discoverAndEnsureSchedules() // Initial check s.checkSchedules() @@ -240,6 +286,9 @@ func (s *testScheduler) Run() { case <-cleanupTicker.C: s.cleanup() + case <-discoveryTicker.C: + s.discoverAndEnsureSchedules() + case schedule := <-s.runNowChan: s.queueOnDemandTest(schedule) @@ -251,6 +300,13 @@ func (s *testScheduler) Run() { // checkSchedules checks for due tests and queues them func (s *testScheduler) checkSchedules() { + s.mu.RLock() + enabled := s.runtimeEnabled + s.mu.RUnlock() + if !enabled { + return + } + dueSchedules, err := s.scheduleUsecase.ListDueSchedules() if err != nil { log.Printf("Error listing due schedules: %v\n", err) @@ -298,6 +354,80 @@ func (s *testScheduler) checkSchedules() { } } +// discoverAndEnsureSchedules creates default (enabled) schedules for all +// (plugin, target) pairs that don't yet have an explicit schedule record. +// This implements the opt-out model: tests run automatically unless a schedule +// with Enabled=false has been explicitly saved. +func (s *testScheduler) discoverAndEnsureSchedules() { + s.mu.RLock() + enabled := s.runtimeEnabled + s.mu.RUnlock() + if !enabled { + return + } + + plugins, err := s.pluginUsecase.ListTestPlugins() + if err != nil { + log.Printf("Error listing test plugins for discovery: %v\n", err) + return + } + + // Filter domain-level plugins + var domainPlugins []happydns.TestPlugin + for _, p := range plugins { + if p.Version().AvailableOn.ApplyToDomain { + domainPlugins = append(domainPlugins, p) + } + } + + if len(domainPlugins) > 0 { + iter, err := s.store.ListAllDomains() + if err != nil { + log.Printf("Error listing domains for schedule discovery: %v\n", err) + } else { + defer iter.Close() + for iter.Next() { + domain := iter.Item() + if domain == nil { + continue + } + for _, plugin := range domainPlugins { + pluginName := plugin.Version().Name + schedules, err := s.scheduleUsecase.ListSchedulesByTarget(happydns.TestScopeDomain, domain.Id) + if err != nil { + continue + } + + hasSchedule := false + for _, sched := range schedules { + if sched.PluginName == pluginName { + hasSchedule = true + break + } + } + + if !hasSchedule { + if err := s.scheduleUsecase.CreateSchedule(&happydns.TestSchedule{ + PluginName: pluginName, + OwnerId: domain.Owner, + TargetType: happydns.TestScopeDomain, + TargetId: domain.Id, + Enabled: true, + }); err != nil { + log.Printf("Error auto-creating schedule for domain %s / plugin %s: %v\n", + domain.Id, pluginName, err) + } + } + } + } + } + } + + // Service-level plugin discovery is deferred: services live inside zones + // and enumeration would require iterating all zones across all domains. + // Services get auto-scheduled on their first explicit interaction instead. +} + // queueOnDemandTest queues an on-demand test execution func (s *testScheduler) queueOnDemandTest(schedule *happydns.TestSchedule) { execution := &happydns.TestExecution{ @@ -366,6 +496,41 @@ func (s *testScheduler) TriggerOnDemandTest(pluginName string, targetType happyd return execution.Id, nil } +// GetSchedulerStatus returns a snapshot of the current scheduler state +func (s *testScheduler) GetSchedulerStatus() happydns.SchedulerStatus { + s.mu.RLock() + activeCount := len(s.activeExecutions) + running := s.running + runtimeEnabled := s.runtimeEnabled + s.mu.RUnlock() + + nextSchedules, _ := s.scheduleUsecase.ListUpcomingSchedules(20) + + return happydns.SchedulerStatus{ + ConfigEnabled: !s.cfg.DisableScheduler, + RuntimeEnabled: runtimeEnabled, + Running: running, + WorkerCount: len(s.workers), + QueueSize: s.queue.Len(), + ActiveCount: activeCount, + NextSchedules: nextSchedules, + } +} + +// SetEnabled enables or disables the scheduler at runtime +func (s *testScheduler) SetEnabled(enabled bool) error { + s.mu.Lock() + s.runtimeEnabled = enabled + s.mu.Unlock() + return nil +} + +// RescheduleUpcomingTests randomizes the next run time of all enabled schedules +// within their respective intervals, delegating to the schedule usecase. +func (s *testScheduler) RescheduleUpcomingTests() (int, error) { + return s.scheduleUsecase.RescheduleUpcomingTests() +} + // cleanup removes old execution records func (s *testScheduler) cleanup() { // This is a placeholder for cleanup logic @@ -409,10 +574,21 @@ func (w *worker) executeTest(item *queueItem) { execution := item.execution schedule := item.schedule + // Always update schedule NextRun after execution, whether it succeeds or fails. + // This prevents the schedule from being re-queued on the next tick if the test fails. + if item.execution.ScheduleId != nil { + defer func() { + if err := w.scheduler.scheduleUsecase.UpdateScheduleAfterRun(*item.execution.ScheduleId); err != nil { + log.Printf("Worker %d: Error updating schedule after run: %v\n", w.id, err) + } + }() + } + // Mark execution as running execution.Status = happydns.TestExecutionRunning if err := w.scheduler.resultUsecase.UpdateTestExecution(execution); err != nil { log.Printf("Worker %d: Error updating execution status: %v\n", w.id, err) + _ = w.scheduler.resultUsecase.FailTestExecution(execution.Id, err.Error()) return } @@ -514,13 +690,6 @@ func (w *worker) executeTest(item *queueItem) { return } - // Update schedule if this was a scheduled test - if item.execution.ScheduleId != nil { - if err := w.scheduler.scheduleUsecase.UpdateScheduleAfterRun(*item.execution.ScheduleId); err != nil { - log.Printf("Worker %d: Error updating schedule: %v\n", w.id, err) - } - } - - log.Printf("Worker %d: Completed test %s for target %s (status: %s, duration: %v)\n", + log.Printf("Worker %d: Completed test %s for target %s (status: %d, duration: %v)\n", w.id, schedule.PluginName, schedule.TargetId, result.Status, duration) } diff --git a/internal/usecase/testresult/testschedule_usecase.go b/internal/usecase/testresult/testschedule_usecase.go new file mode 100644 index 00000000..bb3aec1c --- /dev/null +++ b/internal/usecase/testresult/testschedule_usecase.go @@ -0,0 +1,367 @@ +// This file is part of the happyDomain (R) project. +// Copyright (c) 2020-2026 happyDomain +// Authors: Pierre-Olivier Mercier, et al. +// +// This program is offered under a commercial and under the AGPL license. +// For commercial licensing, contact us at . +// +// For AGPL licensing: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package testresult + +import ( + "fmt" + "math/rand" + "sort" + "time" + + "git.happydns.org/happyDomain/model" +) + +const ( + // Default test intervals + DefaultUserTestInterval = 4 * time.Hour // 4 hours for domain tests + DefaultDomainTestInterval = 24 * time.Hour // 24 hours for domain tests + DefaultServiceTestInterval = 1 * time.Hour // 1 hour for service tests + MinimumTestInterval = 5 * time.Minute // Minimum interval allowed +) + +// TestScheduleUsecase implements business logic for test schedules +type TestScheduleUsecase struct { + storage TestResultStorage + options *happydns.Options +} + +// NewTestScheduleUsecase creates a new test schedule usecase +func NewTestScheduleUsecase(storage TestResultStorage, options *happydns.Options) *TestScheduleUsecase { + return &TestScheduleUsecase{ + storage: storage, + options: options, + } +} + +// ListUserSchedules retrieves all schedules for a specific user +func (u *TestScheduleUsecase) ListUserSchedules(userId happydns.Identifier) ([]*happydns.TestSchedule, error) { + return u.storage.ListTestSchedulesByUser(userId) +} + +// ListSchedulesByTarget retrieves all schedules for a specific target +func (u *TestScheduleUsecase) ListSchedulesByTarget(targetType happydns.TestScopeType, targetId happydns.Identifier) ([]*happydns.TestSchedule, error) { + return u.storage.ListTestSchedulesByTarget(targetType, targetId) +} + +// GetSchedule retrieves a specific schedule by ID +func (u *TestScheduleUsecase) GetSchedule(scheduleId happydns.Identifier) (*happydns.TestSchedule, error) { + return u.storage.GetTestSchedule(scheduleId) +} + +// CreateSchedule creates a new test schedule with validation +func (u *TestScheduleUsecase) CreateSchedule(schedule *happydns.TestSchedule) error { + // Set default interval if not specified + if schedule.Interval == 0 { + schedule.Interval = u.getDefaultInterval(schedule.TargetType) + } + + // Validate interval + if schedule.Interval < MinimumTestInterval { + return fmt.Errorf("test interval must be at least %v", MinimumTestInterval) + } + + // Calculate next run time: pick a random offset within the interval + // to spread load evenly across all schedules + // TODO: Use a smarter load balance function in the future + if schedule.NextRun.IsZero() { + offset := time.Duration(rand.Int63n(int64(schedule.Interval))) + schedule.NextRun = time.Now().Add(offset) + } + + return u.storage.CreateTestSchedule(schedule) +} + +// UpdateSchedule updates an existing schedule +func (u *TestScheduleUsecase) UpdateSchedule(schedule *happydns.TestSchedule) error { + // Validate interval + if schedule.Interval < MinimumTestInterval { + return fmt.Errorf("test interval must be at least %v", MinimumTestInterval) + } + + // Get existing schedule to preserve certain fields + existing, err := u.storage.GetTestSchedule(schedule.Id) + if err != nil { + return err + } + + // Preserve LastRun if not explicitly changed + if schedule.LastRun == nil { + schedule.LastRun = existing.LastRun + } + + // Recalculate next run time if interval changed + if schedule.Interval != existing.Interval { + if schedule.LastRun != nil { + schedule.NextRun = schedule.LastRun.Add(schedule.Interval) + } else { + schedule.NextRun = time.Now().Add(schedule.Interval) + } + } + + return u.storage.UpdateTestSchedule(schedule) +} + +// DeleteSchedule removes a schedule +func (u *TestScheduleUsecase) DeleteSchedule(scheduleId happydns.Identifier) error { + return u.storage.DeleteTestSchedule(scheduleId) +} + +// EnableSchedule enables a schedule +func (u *TestScheduleUsecase) EnableSchedule(scheduleId happydns.Identifier) error { + schedule, err := u.storage.GetTestSchedule(scheduleId) + if err != nil { + return err + } + + schedule.Enabled = true + + // Reset next run time if it's in the past + if schedule.NextRun.Before(time.Now()) { + schedule.NextRun = time.Now().Add(schedule.Interval) + } + + return u.storage.UpdateTestSchedule(schedule) +} + +// DisableSchedule disables a schedule +func (u *TestScheduleUsecase) DisableSchedule(scheduleId happydns.Identifier) error { + schedule, err := u.storage.GetTestSchedule(scheduleId) + if err != nil { + return err + } + + schedule.Enabled = false + return u.storage.UpdateTestSchedule(schedule) +} + +// UpdateScheduleAfterRun updates a schedule after it has been executed +func (u *TestScheduleUsecase) UpdateScheduleAfterRun(scheduleId happydns.Identifier) error { + schedule, err := u.storage.GetTestSchedule(scheduleId) + if err != nil { + return err + } + + now := time.Now() + schedule.LastRun = &now + schedule.NextRun = now.Add(schedule.Interval) + + return u.storage.UpdateTestSchedule(schedule) +} + +// ListDueSchedules retrieves all enabled schedules that are due to run +func (u *TestScheduleUsecase) ListDueSchedules() ([]*happydns.TestSchedule, error) { + schedules, err := u.storage.ListEnabledTestSchedules() + if err != nil { + return nil, err + } + + now := time.Now() + var dueSchedules []*happydns.TestSchedule + + for _, schedule := range schedules { + if schedule.NextRun.Before(now) { + dueSchedules = append(dueSchedules, schedule) + } + } + + return dueSchedules, nil +} + +// ListUpcomingSchedules retrieves the next `limit` enabled schedules sorted by NextRun ascending +func (u *TestScheduleUsecase) ListUpcomingSchedules(limit int) ([]*happydns.TestSchedule, error) { + schedules, err := u.storage.ListEnabledTestSchedules() + if err != nil { + return nil, err + } + + sort.Slice(schedules, func(i, j int) bool { + return schedules[i].NextRun.Before(schedules[j].NextRun) + }) + + if limit > 0 && len(schedules) > limit { + schedules = schedules[:limit] + } + + return schedules, nil +} + +// getDefaultInterval returns the default test interval based on target type +func (u *TestScheduleUsecase) getDefaultInterval(targetType happydns.TestScopeType) time.Duration { + switch targetType { + case happydns.TestScopeUser: + return DefaultUserTestInterval + case happydns.TestScopeDomain: + return DefaultDomainTestInterval + case happydns.TestScopeService: + return DefaultServiceTestInterval + default: + return DefaultDomainTestInterval + } +} + +// MergePluginOptions merges plugin options from different scopes +// Priority: schedule options > domain options > user options > global options +func (u *TestScheduleUsecase) MergePluginOptions( + globalOpts happydns.PluginOptions, + userOpts happydns.PluginOptions, + domainOpts happydns.PluginOptions, + scheduleOpts happydns.PluginOptions, +) happydns.PluginOptions { + merged := make(happydns.PluginOptions) + + // Start with global options + for k, v := range globalOpts { + merged[k] = v + } + + // Override with user options + for k, v := range userOpts { + merged[k] = v + } + + // Override with domain options + for k, v := range domainOpts { + merged[k] = v + } + + // Override with schedule options (highest priority) + for k, v := range scheduleOpts { + merged[k] = v + } + + return merged +} + +// ValidateScheduleOwnership checks if a user owns a schedule +func (u *TestScheduleUsecase) ValidateScheduleOwnership(scheduleId happydns.Identifier, userId happydns.Identifier) error { + schedule, err := u.storage.GetTestSchedule(scheduleId) + if err != nil { + return err + } + + if !schedule.OwnerId.Equals(userId) { + return fmt.Errorf("user does not own this schedule") + } + + return nil +} + +// CreateDefaultSchedulesForTarget creates default schedules for a new target +func (u *TestScheduleUsecase) CreateDefaultSchedulesForTarget( + pluginName string, + targetType happydns.TestScopeType, + targetId happydns.Identifier, + ownerId happydns.Identifier, + enabled bool, +) error { + schedule := &happydns.TestSchedule{ + PluginName: pluginName, + OwnerId: ownerId, + TargetType: targetType, + TargetId: targetId, + Interval: u.getDefaultInterval(targetType), + Enabled: enabled, + NextRun: time.Now().Add(u.getDefaultInterval(targetType)), + Options: make(happydns.PluginOptions), + } + + return u.CreateSchedule(schedule) +} + +// rescheduleTests reschedules each given schedule to a random time in [now, now+maxOffsetFn(schedule)]. +func (u *TestScheduleUsecase) rescheduleTests(schedules []*happydns.TestSchedule, maxOffsetFn func(*happydns.TestSchedule) time.Duration) (int, error) { + count := 0 + now := time.Now() + for _, schedule := range schedules { + maxOffset := maxOffsetFn(schedule) + if maxOffset <= 0 { + maxOffset = time.Second + } + schedule.NextRun = now.Add(time.Duration(rand.Int63n(int64(maxOffset)))) + if err := u.storage.UpdateTestSchedule(schedule); err != nil { + return count, err + } + count++ + } + return count, nil +} + +// RescheduleUpcomingTests randomizes the next run time of all enabled schedules +// within their respective intervals to spread load evenly. Useful after a restart. +func (u *TestScheduleUsecase) RescheduleUpcomingTests() (int, error) { + schedules, err := u.storage.ListEnabledTestSchedules() + if err != nil { + return 0, err + } + return u.rescheduleTests(schedules, func(s *happydns.TestSchedule) time.Duration { + return s.Interval + }) +} + +// RescheduleOverdueTests reschedules tests whose NextRun is in the past, +// spreading them over a short window to avoid scheduler famine (e.g. after +// a long machine suspend or server downtime). +func (u *TestScheduleUsecase) RescheduleOverdueTests() (int, error) { + schedules, err := u.storage.ListEnabledTestSchedules() + if err != nil { + return 0, err + } + + now := time.Now() + var overdue []*happydns.TestSchedule + for _, s := range schedules { + if s.NextRun.Before(now) { + overdue = append(overdue, s) + } + } + + if len(overdue) == 0 { + return 0, nil + } + + // Spread overdue tests over a small window proportional to their count, + // capped at MinimumTestInterval, to prevent all of them from running at once. + spreadWindow := time.Duration(len(overdue)) * 5 * time.Second + if spreadWindow > MinimumTestInterval { + spreadWindow = MinimumTestInterval + } + + return u.rescheduleTests(overdue, func(s *happydns.TestSchedule) time.Duration { + return spreadWindow + }) +} + +// DeleteSchedulesForTarget removes all schedules for a target +func (u *TestScheduleUsecase) DeleteSchedulesForTarget(targetType happydns.TestScopeType, targetId happydns.Identifier) error { + schedules, err := u.storage.ListTestSchedulesByTarget(targetType, targetId) + if err != nil { + return err + } + + for _, schedule := range schedules { + if err := u.storage.DeleteTestSchedule(schedule.Id); err != nil { + return err + } + } + + return nil +} diff --git a/model/test_result.go b/model/test_result.go index 8b9af544..9f046cae 100644 --- a/model/test_result.go +++ b/model/test_result.go @@ -191,6 +191,30 @@ type TestExecution struct { Options PluginOptions `json:"options,omitempty"` } +// SchedulerStatus holds a snapshot of the scheduler state for monitoring +type SchedulerStatus struct { + // ConfigEnabled indicates if the scheduler is enabled in the configuration file + ConfigEnabled bool `json:"config_enabled"` + + // RuntimeEnabled indicates if the scheduler is currently enabled at runtime + RuntimeEnabled bool `json:"runtime_enabled"` + + // Running indicates if the scheduler goroutine is currently running + Running bool `json:"running"` + + // WorkerCount is the number of worker goroutines + WorkerCount int `json:"worker_count"` + + // QueueSize is the number of items currently waiting in the execution queue + QueueSize int `json:"queue_size"` + + // ActiveCount is the number of tests currently being executed + ActiveCount int `json:"active_count"` + + // NextSchedules contains the upcoming scheduled tests sorted by next run time + NextSchedules []*TestSchedule `json:"next_schedules"` +} + // TestResultUsecase defines business logic for test results type TestResultUsecase interface { // ListTestResultsByTarget retrieves test results for a specific target diff --git a/model/usecase.go b/model/usecase.go index 0a837186..326c5a75 100644 --- a/model/usecase.go +++ b/model/usecase.go @@ -37,6 +37,7 @@ type UsecaseDependancies interface { TestPluginUsecase() TestPluginUsecase TestResultUsecase() TestResultUsecase TestScheduleUsecase() TestScheduleUsecase + TestScheduler() AdminSchedulerUsecase UserUsecase() UserUsecase ZoneCorrectionApplierUsecase() ZoneCorrectionApplierUsecase ZoneImporterUsecase() ZoneImporterUsecase diff --git a/web-admin/src/routes/+layout.svelte b/web-admin/src/routes/+layout.svelte index 9230b2b0..3fbd65bf 100644 --- a/web-admin/src/routes/+layout.svelte +++ b/web-admin/src/routes/+layout.svelte @@ -104,6 +104,9 @@ Plugins + + Scheduler + diff --git a/web-admin/src/routes/scheduler/+page.svelte b/web-admin/src/routes/scheduler/+page.svelte new file mode 100644 index 00000000..72049086 --- /dev/null +++ b/web-admin/src/routes/scheduler/+page.svelte @@ -0,0 +1,334 @@ + + + + + + + +

+ + Test Scheduler +

+

Monitor and control the background test scheduler

+ + + + {#if loading} +
+ + Loading scheduler status... +
+ {:else if error} + + + Error loading scheduler status: {error} + + + {:else if status} + + + +
+ Scheduler Status + +
+ + + + +
Config Enabled
+ {#if status.config_enabled} + Yes + {:else} + No + {/if} + + +
Runtime Enabled
+ {#if status.runtime_enabled} + Yes + {:else} + Disabled + {/if} + + +
Running
+ {#if status.running} + Running + {:else} + Stopped + {/if} + + +
Workers
+ {status.worker_count} + + +
Queue Size
+ {status.queue_size} + + +
Active Executions
+ {status.active_count} + + + + {#if status.config_enabled} +
+ {#if status.runtime_enabled} + + {:else} + + {/if} + +
+ {:else} +

+ + The scheduler is disabled in the server configuration and cannot be enabled at + runtime. +

+ {/if} + + + + + + + + Upcoming Scheduled Tests + {#if status.next_schedules} + {status.next_schedules.length} + {/if} + + +
+ + + + + + + + + + + + + {#if !status.next_schedules || status.next_schedules.length === 0} + + + + {:else} + {#each status.next_schedules as schedule} + + + + + + + + + {/each} + {/if} + +
PluginTarget TypeTarget IDIntervalLast RunNext Run
+ No scheduled tests +
{schedule.plugin_name}{targetTypeName(schedule.target_type)}{schedule.target_id}{formatDuration(schedule.interval)} + {#if schedule.last_run} + {new Date(schedule.last_run).toLocaleString()} + {:else} + Never + {/if} + + {#if new Date(schedule.next_run) < new Date()} + + + {new Date(schedule.next_run).toLocaleString()} + + {:else} + {new Date(schedule.next_run).toLocaleString()} + {/if} +
+
+ + + {/if} +