sync: Split SyncFiles function into import and files sync

admin/api/sync.go

@@ -241,7 +241,7 @@ func declareSyncExercicesRoutes(router *gin.RouterGroup) {
 
 		exceptions := sync.LoadExerciceException(sync.GlobalImporter, theme, exercice, nil)
 
-		c.JSON(http.StatusOK, flatifySyncErrors(sync.SyncExerciceFiles(sync.GlobalImporter, exercice, exceptions)))
+		c.JSON(http.StatusOK, flatifySyncErrors(sync.ImportExerciceFiles(sync.GlobalImporter, exercice, exceptions)))
 	})
 	apiSyncExercicesRoutes.POST("/fixurlid", func(c *gin.Context) {
 		exercice := c.MustGet("exercice").(*fic.Exercice)

admin/sync/exercice_files.go

@@ -315,9 +315,57 @@ func DownloadExerciceFile(pf ExerciceFile, dest string, exercice *fic.Exercice,
 	return
 }
 
-// SyncExerciceFiles reads the content of files/ directory and import it as EFile for the given challenge.
+type importedFile struct {
+	file interface{}
+	Name string
+}
+
+func SyncExerciceFiles(i Importer, exercice *fic.Exercice, paramsFiles map[string]ExerciceFile, actionAfterImport func(fname string, digests map[string][]byte, filePath, origin string) (interface{}, error)) (ret []*importedFile, errs error) {
+	files, digests, berrs := BuildFilesListInto(i, exercice, "files")
+	errs = multierr.Append(errs, berrs)
+
+	// Import standard files
+	for _, fname := range files {
+		var f interface{}
+		var err error
+
+		if pf, exists := paramsFiles[fname]; exists && pf.URL != "" && !i.Exists(path.Join(exercice.Path, "files", fname)) {
+			dest := GetDestinationFilePath(pf.URL, &pf.Filename)
+
+			if _, err := os.Stat(dest); !os.IsNotExist(err) {
+				if d, err := actionAfterImport(fname, digests, dest, pf.URL); err == nil {
+					f = d
+				}
+			}
+
+			if f == nil {
+				errs = multierr.Append(errs, DownloadExerciceFile(paramsFiles[fname], dest, exercice, false))
+
+				f, err = actionAfterImport(fname, digests, dest, pf.URL)
+			}
+		} else {
+			f, err = i.importFile(path.Join(exercice.Path, "files", fname), func(filePath, origin string) (interface{}, error) {
+				return actionAfterImport(fname, digests, filePath, origin)
+			})
+		}
+
+		if err != nil {
+			errs = multierr.Append(errs, NewFileError(exercice, fname, err))
+			continue
+		}
+
+		ret = append(ret, &importedFile{
+			f,
+			fname,
+		})
+	}
+
+	return
+}
+
+// ImportExerciceFiles reads the content of files/ directory and import it as EFile for the given challenge.
 // It takes care of DIGESTS.txt and ensure imported files match.
-func SyncExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExceptions) (errs error) {
+func ImportExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExceptions) (errs error) {
 	if _, err := exercice.WipeFiles(); err != nil {
 		errs = multierr.Append(errs, err)
 	}
@@ -328,63 +376,41 @@ func SyncExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExce
 		return
 	}
 
-	files, digests, berrs := BuildFilesListInto(i, exercice, "files")
+	actionAfterImport := func(fname string, digests map[string][]byte, filePath, origin string) (interface{}, error) {
+		var digest_shown []byte
+		if strings.HasSuffix(fname, ".gz") {
+			if d, exists := digests[strings.TrimSuffix(fname, ".gz")]; exists {
+				digest_shown = d
+			}
+		}
+
+		published := true
+		disclaimer := ""
+		if f, exists := paramsFiles[fname]; exists {
+			published = !f.Hidden
+
+			// Call checks hooks
+			for _, hk := range hooks.mdTextHooks {
+				for _, err := range multierr.Errors(hk(f.Disclaimer, exercice.Language, exceptions)) {
+					errs = multierr.Append(errs, NewFileError(exercice, fname, err))
+				}
+			}
+
+			if disclaimer, err = ProcessMarkdown(i, fixnbsp(f.Disclaimer), exercice.Path); err != nil {
+				errs = multierr.Append(errs, NewFileError(exercice, fname, fmt.Errorf("error during markdown formating of disclaimer: %w", err)))
+			}
+		}
+
+		return exercice.ImportFile(filePath, origin, digests[fname], digest_shown, disclaimer, published)
+	}
+
+	files, berrs := SyncExerciceFiles(i, exercice, paramsFiles, actionAfterImport)
 	errs = multierr.Append(errs, berrs)
 
-	// Import standard files
+	// Import files in db
-	for _, fname := range files {
+	for _, file := range files {
-		actionAfterImport := func(filePath string, origin string) (interface{}, error) {
+		fname := file.Name
-			var digest_shown []byte
+		f := file.file
-			if strings.HasSuffix(fname, ".gz") {
-				if d, exists := digests[strings.TrimSuffix(fname, ".gz")]; exists {
-					digest_shown = d
-				}
-			}
-
-			published := true
-			disclaimer := ""
-			if f, exists := paramsFiles[fname]; exists {
-				published = !f.Hidden
-
-				// Call checks hooks
-				for _, hk := range hooks.mdTextHooks {
-					for _, err := range multierr.Errors(hk(f.Disclaimer, exercice.Language, exceptions)) {
-						errs = multierr.Append(errs, NewFileError(exercice, fname, err))
-					}
-				}
-
-				if disclaimer, err = ProcessMarkdown(i, fixnbsp(f.Disclaimer), exercice.Path); err != nil {
-					errs = multierr.Append(errs, NewFileError(exercice, fname, fmt.Errorf("error during markdown formating of disclaimer: %w", err)))
-				}
-			}
-
-			return exercice.ImportFile(filePath, origin, digests[fname], digest_shown, disclaimer, published)
-		}
-
-		var f interface{}
-
-		if pf, exists := paramsFiles[fname]; exists && pf.URL != "" && !i.Exists(path.Join(exercice.Path, "files", fname)) {
-			dest := GetDestinationFilePath(pf.URL, &pf.Filename)
-
-			if _, err := os.Stat(dest); !os.IsNotExist(err) {
-				if d, err := actionAfterImport(dest, pf.URL); err == nil {
-					f = d
-				}
-			}
-
-			if f == nil {
-				errs = multierr.Append(errs, DownloadExerciceFile(paramsFiles[fname], dest, exercice, false))
-
-				f, err = actionAfterImport(dest, pf.URL)
-			}
-		} else {
-			f, err = i.importFile(path.Join(exercice.Path, "files", fname), actionAfterImport)
-		}
-
-		if err != nil {
-			errs = multierr.Append(errs, NewFileError(exercice, fname, err))
-			continue
-		}
 
 		if f.(*fic.EFile).Size == 0 {
 			errs = multierr.Append(errs, NewFileError(exercice, fname, fmt.Errorf("imported file is empty!")))

admin/sync/full.go

@@ -246,7 +246,7 @@ func SyncThemeDeep(i Importer, theme *fic.Theme, tid int, themeStep uint8, excep
 			log.Printf("Deep synchronization in progress: %d/255 - doing Theme %q, Exercice %q: %q\n", DeepSyncProgress, theme.Name, exercice.Title, exercice.Path)
 
 			DeepSyncProgress = 3 + uint8(tid)*themeStep + uint8(eid)*exerciceStep
-			errs = multierr.Append(errs, SyncExerciceFiles(i, exercice, ex_exceptions[eid]))
+			errs = multierr.Append(errs, ImportExerciceFiles(i, exercice, ex_exceptions[eid]))
 
 			DeepSyncProgress += exerciceStep / 3
 			flagsBindings, ferrs := SyncExerciceFlags(i, exercice, ex_exceptions[eid])

admin/sync/themes.go

@@ -273,6 +273,36 @@ func BuildTheme(i Importer, tdir string) (th *fic.Theme, exceptions *CheckExcept
 	return
 }
 
+// SyncThemeFiles import all theme's related files
+func SyncThemeFiles(i Importer, btheme *fic.Theme) (errs error) {
+	if len(btheme.Image) > 0 {
+		if _, err := i.importFile(btheme.Image,
+			func(filePath string, origin string) (interface{}, error) {
+				if err := resizePicture(origin, filePath, image.Rect(0, 0, 500, 300)); err != nil {
+					return nil, err
+				}
+
+				btheme.Image = strings.TrimPrefix(filePath, fic.FilesDir)
+				btheme.BackgroundColor, _ = getBackgroundColor(filePath)
+				return nil, nil
+			}); err != nil {
+			errs = multierr.Append(errs, NewThemeError(btheme, fmt.Errorf("unable to import heading image: %w", err)))
+		}
+	}
+
+	if len(btheme.PartnerImage) > 0 {
+		if _, err := i.importFile(btheme.PartnerImage,
+			func(filePath string, origin string) (interface{}, error) {
+				btheme.PartnerImage = strings.TrimPrefix(filePath, fic.FilesDir)
+				return nil, nil
+			}); err != nil {
+			errs = multierr.Append(errs, NewThemeError(btheme, fmt.Errorf("unable to import partner image: %w", err)))
+		}
+	}
+
+	return
+}
+
 // SyncThemes imports new or updates existing themes.
 func SyncThemes(i Importer) (exceptions map[string]*CheckExceptions, errs error) {
 	if themes, err := GetThemes(i); err != nil {
@@ -294,29 +324,9 @@ func SyncThemes(i Importer) (exceptions map[string]*CheckExceptions, errs error)
 
 			exceptions[tdir] = excepts
 
-			if len(btheme.Image) > 0 {
+			err = SyncThemeFiles(i, btheme)
-				if _, err := i.importFile(btheme.Image,
+			if err != nil {
-					func(filePath string, origin string) (interface{}, error) {
+				errs = multierr.Append(errs, NewThemeError(btheme, fmt.Errorf("unable to import heading image: %w", err)))
-						if err := resizePicture(filePath, image.Rect(0, 0, 500, 300)); err != nil {
-							return nil, err
-						}
-
-						btheme.Image = strings.TrimPrefix(filePath, fic.FilesDir)
-						btheme.BackgroundColor, _ = getBackgroundColor(filePath)
-						return nil, nil
-					}); err != nil {
-					errs = multierr.Append(errs, NewThemeError(btheme, fmt.Errorf("unable to import heading image: %w", err)))
-				}
-			}
-
-			if len(btheme.PartnerImage) > 0 {
-				if _, err := i.importFile(btheme.PartnerImage,
-					func(filePath string, origin string) (interface{}, error) {
-						btheme.PartnerImage = strings.TrimPrefix(filePath, fic.FilesDir)
-						return nil, nil
-					}); err != nil {
-					errs = multierr.Append(errs, NewThemeError(btheme, fmt.Errorf("unable to import partner image: %w", err)))
-				}
 			}
 
 			var theme *fic.Theme