fic/server
Archived
1
0
Fork 0
Code Issues Pull requests 10 Releases Wiki Activity

admin: introducing speedy deep sync and themed deep sync

Browse source
This commit is contained in:
nemunaire 2020-01-20 08:41:36 +01:00
parent f7762c0828
commit a4c87b92a5
5 changed files with 170 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

147
admin/sync/full.go
View file

@ -17,10 +17,60 @@ var DeepReportPath = "full_import_report.json"
// oneDeepSync ensure there is no more than one running deep sync.
var oneDeepSync sync.Mutex
var oneThemeDeepSync sync.Mutex
// DeepSyncProgress expose the progression of the depp synchronization (0 = 0%, 255 = 100%).
var DeepSyncProgress uint8
// SpeedySyncDeep performs a recursive synchronisation without importing files.
func SpeedySyncDeep(i Importer) (errs map[string][]string) {
oneDeepSync.Lock()
defer func(){
oneDeepSync.Unlock()
if DeepSyncProgress != 255 {
log.Printf("Speedy synchronization terminated at step %d/255", DeepSyncProgress)
}
}()
DeepSyncProgress = 1
errs = map[string][]string{}
startTime := time.Now()
errs["_date"] = []string{fmt.Sprintf("%v", startTime)}
errs["_themes"] = SyncThemes(i)
if themes, err := fic.GetThemes(); err == nil {
DeepSyncProgress = 2
var themeStep uint8 = uint8(250) / uint8(len(themes))
for tid, theme := range themes {
DeepSyncProgress = 3 + uint8(tid) * themeStep
errs[theme.Name] = SyncExercices(i, theme)
if exercices, err := theme.GetExercices(); err == nil {
var exerciceStep uint8 = themeStep / uint8(len(exercices))
for eid, exercice := range exercices {
log.Printf("Deep synchronization in progress: %d/255 - doing Theme %q, Exercice %q: %q\n", DeepSyncProgress, theme.Name, exercice.Title, exercice.Path)
DeepSyncProgress = 3 + uint8(tid) * themeStep + uint8(eid) * exerciceStep
errs[theme.Name] = append(errs[theme.Name], SyncExerciceFiles(i, exercice)...)
DeepSyncProgress += exerciceStep / 2
_, ferrs := SyncExerciceFlags(i, exercice)
errs[theme.Name] = append(errs[theme.Name], ferrs...)
}
}
}
}
DeepSyncProgress = 254
errs["_date"] = append(errs["_date"], fmt.Sprintf("%v", time.Now()))
DeepSyncProgress = 255
log.Println("Speedy synchronization done in", time.Since(startTime))
return
}
// SyncDeep performs a recursive synchronisation: from themes to challenge items.
func SyncDeep(i Importer) (errs map[string][]string) {
oneDeepSync.Lock()
@ -43,34 +93,59 @@ func SyncDeep(i Importer) (errs map[string][]string) {
var themeStep uint8 = uint8(250) / uint8(len(themes))
for tid, theme := range themes {
DeepSyncProgress = 3 + uint8(tid) * themeStep
errs[theme.Name] = SyncExercices(i, theme)
if exercices, err := theme.GetExercices(); err == nil {
var exerciceStep uint8 = themeStep / uint8(len(exercices))
for eid, exercice := range exercices {
log.Printf("Full synchronization in progress: %d/255 - doing Theme %q, Exercice %q: %q\n", DeepSyncProgress, theme.Name, exercice.Title, exercice.Path)
DeepSyncProgress = 3 + uint8(tid) * themeStep + uint8(eid) * exerciceStep
errs[theme.Name] = append(errs[theme.Name], SyncExerciceFiles(i, exercice)...)
DeepSyncProgress += exerciceStep / 3
flagsBindings, ferrs := SyncExerciceFlags(i, exercice)
errs[theme.Name] = append(errs[theme.Name], ferrs...)
DeepSyncProgress += exerciceStep / 3
_, herrs := SyncExerciceHints(i, exercice, flagsBindings)
errs[theme.Name] = append(errs[theme.Name], herrs...)
}
}
errs[theme.Name] = SyncThemeDeep(i, theme, tid, themeStep)
}
}
DeepSyncProgress = 254
errs["_date"] = append(errs["_date"], fmt.Sprintf("%v", time.Now()))
EditDeepReport(errs, true)
if err := settings.ForceRegeneration(); err != nil {
errs["_regeneration"] = append(errs["_regeneration"], err.Error())
}
DeepSyncProgress = 255
log.Println("Full synchronization done in", time.Since(startTime))
return
}
func readDeepReport() (ret map[string][]string, err error) {
if fdfrom, err := os.Open(DeepReportPath); err == nil {
defer fdfrom.Close()
jdec := json.NewDecoder(fdfrom)
if err := jdec.Decode(&ret); err != nil {
return nil, err
}
} else {
return nil, err
}
return
}
func EditDeepReport(errs map[string][]string, erase bool) {
errs["_regeneration"] = []string{}
if !erase {
if in, err := readDeepReport(); err != nil {
errs["_regeneration"] = append(errs["_regeneration"], err.Error())
log.Println(err)
} else {
for k, v := range errs {
in[k] = v
}
errs = in
}
}
if _, ok := errs["_date"]; ok {
errs["_date"] = append(errs["_date"], fmt.Sprintf("%v", time.Now()))
}
if fdto, err := os.Create(DeepReportPath); err == nil {
defer fdto.Close()
@ -85,11 +160,33 @@ func SyncDeep(i Importer) (errs map[string][]string) {
log.Println(err)
}
if err := settings.ForceRegeneration(); err != nil {
errs["_regeneration"] = append(errs["_regeneration"], err.Error())
}
// SyncThemeDeep performs a recursive synchronisation: from challenges to challenge items.
func SyncThemeDeep(i Importer, theme fic.Theme, tid int, themeStep uint8) (errs []string) {
oneThemeDeepSync.Lock()
defer oneThemeDeepSync.Unlock()
DeepSyncProgress = 3 + uint8(tid) * themeStep
errs = SyncExercices(i, theme)
if exercices, err := theme.GetExercices(); err == nil && len(exercices) > 0 {
var exerciceStep uint8 = themeStep / uint8(len(exercices))
for eid, exercice := range exercices {
log.Printf("Deep synchronization in progress: %d/255 - doing Theme %q, Exercice %q: %q\n", DeepSyncProgress, theme.Name, exercice.Title, exercice.Path)
DeepSyncProgress = 3 + uint8(tid) * themeStep + uint8(eid) * exerciceStep
errs = append(errs, SyncExerciceFiles(i, exercice)...)
DeepSyncProgress += exerciceStep / 3
flagsBindings, ferrs := SyncExerciceFlags(i, exercice)
errs = append(errs, ferrs...)
DeepSyncProgress += exerciceStep / 3
_, herrs := SyncExerciceHints(i, exercice, flagsBindings)
errs = append(errs, herrs...)
}
}
DeepSyncProgress = 255
log.Println("Full synchronization done in", time.Since(startTime))
return
}