sync: Refactor how and when remote files are downloaded

admin/sync/exercice_files.go

@@ -181,8 +181,23 @@ func CheckExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExc
 	flist, digests, berrs := BuildFilesListInto(i, exercice, "files")
 	errs = append(errs, berrs...)
 
+	paramsFiles, err := GetExerciceFilesParams(i, exercice)
+	if err != nil {
+		errs = append(errs, NewChallengeTxtError(exercice, 0, err))
+	}
+
 	for _, fname := range flist {
-		if fd, closer, err := GetFile(i, path.Join(exercice.Path, "files", fname)); err != nil {
+		dest := path.Join(exercice.Path, "files", fname)
+
+		if pf, exists := paramsFiles[fname]; exists && pf.URL != "" {
+			if li, ok := i.(LocalImporter); ok {
+				errs = append(errs, downloadExerciceFile(paramsFiles[fname], li.GetLocalPath(dest), exercice, false)...)
+			} else {
+				errs = append(errs, downloadExerciceFile(paramsFiles[fname], dest, exercice, false)...)
+			}
+		}
+
+		if fd, closer, err := GetFile(i, dest); err != nil {
 			errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to read file: %w", err)))
 			continue
 		} else {
@@ -221,12 +236,6 @@ func CheckExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExc
 					}
 				}
 
-				paramsFiles, err := GetExerciceFilesParams(i, exercice)
-				if err != nil {
-					errs = append(errs, NewChallengeTxtError(exercice, 0, err))
-					return
-				}
-
 				disclaimer := ""
 				if f, exists := paramsFiles[fname]; exists {
 					// Call checks hooks
@@ -257,6 +266,50 @@ func CheckExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExc
 	return
 }
 
+// downloadExerciceFile is responsible to fetch remote files.
+func downloadExerciceFile(pf ExerciceFile, dest string, exercice *fic.Exercice, force bool) (errs []error) {
+	if st, err := os.Stat(dest); !force && !os.IsNotExist(err) {
+		resp, err := http.Head(pf.URL)
+		if err == nil && resp.ContentLength == st.Size() {
+			return
+		}
+	}
+
+	if !isURLAllowed(pf.URL) {
+		errs = append(errs, NewFileError(exercice, path.Base(dest), fmt.Errorf("URL hostname is not whitelisted")))
+		return
+	}
+
+	resp, err := http.Get(pf.URL)
+	if err != nil {
+		errs = append(errs, NewFileError(exercice, path.Base(dest), err))
+		return
+	}
+	defer resp.Body.Close()
+
+	if err = os.MkdirAll(path.Dir(dest), 0751); err != nil {
+		errs = append(errs, NewFileError(exercice, path.Base(dest), err))
+		return
+	}
+
+	// Write file
+	var fdto *os.File
+	if fdto, err = os.Create(dest); err != nil {
+		errs = append(errs, NewFileError(exercice, path.Base(dest), err))
+		return
+	} else {
+		defer fdto.Close()
+
+		_, err = io.Copy(fdto, resp.Body)
+		if err != nil {
+			errs = append(errs, NewFileError(exercice, path.Base(dest), err))
+			return
+		}
+	}
+
+	return
+}
+
 // SyncExerciceFiles reads the content of files/ directory and import it as EFile for the given challenge.
 // It takes care of DIGESTS.txt and ensure imported files match.
 func SyncExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExceptions) (errs []error) {
@@ -314,39 +367,8 @@ func SyncExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExce
 				}
 			}
 
-			if !isURLAllowed(pf.URL) {
-				errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("URL hostname is not whitelisted")))
-				continue
-			}
-
 			if f == nil {
-				var resp *http.Response
+				errs = append(errs, downloadExerciceFile(paramsFiles[fname], dest, exercice, false)...)
-				resp, err = http.Get(pf.URL)
-				if err != nil {
-					errs = append(errs, NewFileError(exercice, fname, err))
-					continue
-				}
-				defer resp.Body.Close()
-
-				if err = os.MkdirAll(path.Dir(dest), 0751); err != nil {
-					errs = append(errs, NewFileError(exercice, fname, err))
-					continue
-				}
-
-				// Write file
-				var fdto *os.File
-				if fdto, err = os.Create(dest); err != nil {
-					errs = append(errs, NewFileError(exercice, fname, err))
-					continue
-				} else {
-					defer fdto.Close()
-
-					_, err = io.Copy(fdto, resp.Body)
-					if err != nil {
-						errs = append(errs, NewFileError(exercice, fname, err))
-						continue
-					}
-				}
 
 				f, err = actionAfterImport(dest, pf.URL)
 			}