package sync import ( "encoding/json" "io" "log" "os" "sync" "time" "go.uber.org/multierr" "srs.epita.fr/fic-server/admin/generation" "srs.epita.fr/fic-server/libfic" ) // DeepReportPath stores the path to the report generated during full recursive import. var DeepReportPath = "full_import_report.json" // oneDeepSync ensure there is no more than one running deep sync. var oneDeepSync sync.Mutex var oneThemeDeepSync sync.Mutex // DeepSyncProgress expose the progression of the depp synchronization (0 = 0%, 255 = 100%). var DeepSyncProgress uint8 type SyncReport struct { DateStart time.Time `json:"_started"` DateEnd time.Time `json:"_ended"` DateUpdated []time.Time `json:"_updated"` Regeneration []string `json:"_regeneration"` SyncId string `json:"_id,omitempty"` ThemesSync []string `json:"_themes,omitempty"` Themes map[string][]string `json:"themes"` Exercices []string `json:"exercices,omitempty"` } // SpeedySyncDeep performs a recursive synchronisation without importing files. func SpeedySyncDeep(i Importer) (errs SyncReport) { oneDeepSync.Lock() defer func() { oneDeepSync.Unlock() if DeepSyncProgress != 255 { log.Printf("Speedy synchronization terminated at step %d/255", DeepSyncProgress) } }() DeepSyncProgress = 1 errs.Themes = map[string][]string{} startTime := time.Now() errs.DateStart = startTime exceptions, sterrs := SyncThemes(i) for _, sterr := range multierr.Errors(sterrs) { errs.ThemesSync = append(errs.ThemesSync, sterr.Error()) } if themes, err := fic.GetThemes(); err == nil { DeepSyncProgress = 2 if i.Exists(StandaloneExercicesDirectory) { themes = append(themes, &fic.Theme{Path: StandaloneExercicesDirectory}) } var themeStep uint8 = uint8(250) / uint8(len(themes)) for tid, theme := range themes { DeepSyncProgress = 3 + uint8(tid)*themeStep ex_exceptions, seerrs := SyncExercices(i, theme, exceptions[theme.Path]) for _, seerr := range multierr.Errors(seerrs) { errs.Themes[theme.Name] = append(errs.Themes[theme.Name], seerr.Error()) } if exercices, err := theme.GetExercices(); err == nil { if len(exercices) == 0 { continue } var exerciceStep uint8 = themeStep / uint8(len(exercices)) for eid, exercice := range exercices { log.Printf("Speedy synchronization in progress: %d/255 - doing Theme %q, Exercice %q: %q\n", DeepSyncProgress, theme.Name, exercice.Title, exercice.Path) DeepSyncProgress = 3 + uint8(tid)*themeStep + uint8(eid)*exerciceStep flagsBindings, ferrs := SyncExerciceFlags(i, exercice, ex_exceptions[eid]) for _, ferr := range multierr.Errors(ferrs) { errs.Themes[theme.Name] = append(errs.Themes[theme.Name], ferr.Error()) } DeepSyncProgress += exerciceStep / 2 _, herrs := SyncExerciceHints(i, exercice, flagsBindings, ex_exceptions[eid]) for _, herr := range multierr.Errors(herrs) { errs.Themes[theme.Name] = append(errs.Themes[theme.Name], herr.Error()) } } } } } DeepSyncProgress = 254 errs.DateEnd = time.Now() DeepSyncProgress = 255 log.Println("Speedy synchronization done in", time.Since(startTime)) return } // SyncDeep performs a recursive synchronisation: from themes to challenge items. func SyncDeep(i Importer) (errs SyncReport) { oneDeepSync.Lock() defer func() { oneDeepSync.Unlock() if DeepSyncProgress != 255 { log.Printf("Full synchronization terminated at step %d/255", DeepSyncProgress) } }() DeepSyncProgress = 1 errs.Themes = map[string][]string{} startTime := time.Now() // Import all themes errs.DateStart = startTime exceptions, sterrs := SyncThemes(i) for _, sterr := range multierr.Errors(sterrs) { errs.ThemesSync = append(errs.ThemesSync, sterr.Error()) } // Synchronize themes if themes, err := fic.GetThemes(); err == nil { DeepSyncProgress = 2 // Also synchronize standalone exercices if i.Exists(StandaloneExercicesDirectory) { themes = append(themes, &fic.Theme{Path: StandaloneExercicesDirectory}) } var themeStep uint8 = uint8(250) / uint8(len(themes)) for tid, theme := range themes { stderrs := SyncThemeDeep(i, theme, tid, themeStep, exceptions[theme.Path]) for _, stderr := range multierr.Errors(stderrs) { errs.Themes[theme.Name] = append(errs.Themes[theme.Name], stderr.Error()) } } } DeepSyncProgress = 254 EditDeepReport(&errs, true) resp, err := generation.FullGeneration() if err != nil { errs.Regeneration = append(errs.Regeneration, err.Error()) } else { defer resp.Body.Close() v, _ := io.ReadAll(resp.Body) errs.Regeneration = append(errs.Regeneration, string(v)) } DeepSyncProgress = 255 log.Println("Full synchronization done in", time.Since(startTime)) return } func readDeepReport() (ret *SyncReport, err error) { if fdfrom, err := os.Open(DeepReportPath); err == nil { defer fdfrom.Close() jdec := json.NewDecoder(fdfrom) if err := jdec.Decode(&ret); err != nil { return nil, err } } else { return nil, err } return } func EditDeepReport(errs *SyncReport, erase bool) { errs.Regeneration = []string{} if !erase { if in, err := readDeepReport(); err != nil { errs.Regeneration = append(errs.Regeneration, err.Error()) log.Println(err) } else { for k, v := range errs.Themes { in.Themes[k] = v } errs = in } } errs.DateUpdated = append(errs.DateUpdated, time.Now()) if fdto, err := os.Create(DeepReportPath); err == nil { defer fdto.Close() if out, err := json.Marshal(errs); err == nil { fdto.Write(out) } else { errs.Regeneration = append(errs.Regeneration, err.Error()) log.Println(err) } } else { errs.Regeneration = append(errs.Regeneration, err.Error()) log.Println(err) } } // SyncThemeDeep performs a recursive synchronisation: from challenges to challenge items. func SyncThemeDeep(i Importer, theme *fic.Theme, tid int, themeStep uint8, exceptions *CheckExceptions) (errs error) { var ex_exceptions map[int]*CheckExceptions oneThemeDeepSync.Lock() defer oneThemeDeepSync.Unlock() DeepSyncProgress = 3 + uint8(tid)*themeStep ex_exceptions, errs = SyncExercices(i, theme, exceptions) if exercices, err := theme.GetExercices(); err == nil && len(exercices) > 0 { var exerciceStep uint8 = themeStep / uint8(len(exercices)) for eid, exercice := range exercices { log.Printf("Deep synchronization in progress: %d/255 - doing Theme %q, Exercice %q: %q\n", DeepSyncProgress, theme.Name, exercice.Title, exercice.Path) DeepSyncProgress = 3 + uint8(tid)*themeStep + uint8(eid)*exerciceStep errs = multierr.Append(errs, SyncExerciceFiles(i, exercice, ex_exceptions[eid])) DeepSyncProgress += exerciceStep / 3 flagsBindings, ferrs := SyncExerciceFlags(i, exercice, ex_exceptions[eid]) errs = multierr.Append(errs, ferrs) DeepSyncProgress += exerciceStep / 3 _, herrs := SyncExerciceHints(i, exercice, flagsBindings, ex_exceptions[eid]) errs = multierr.Append(errs, herrs) } } return }