fic/server
2
0
Fork 0
Code Issues Pull Requests 6 Releases Wiki Activity

sync: Refactor how and when remote files are downloaded

Browse Source
This commit is contained in:
nemunaire 2023-10-18 11:42:48 +02:00
parent 46db6bbf20
commit 49aa1682d7
1 changed files with 61 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
admin/sync/exercice_files.go
View File

@ -181,8 +181,23 @@ func CheckExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExc
flist, digests, berrs := BuildFilesListInto(i, exercice, "files")
errs = append(errs, berrs...)
paramsFiles, err := GetExerciceFilesParams(i, exercice)
if err != nil {
errs = append(errs, NewChallengeTxtError(exercice, 0, err))
}
for _, fname := range flist {
if fd, closer, err := GetFile(i, path.Join(exercice.Path, "files", fname)); err != nil {
dest := path.Join(exercice.Path, "files", fname)
if pf, exists := paramsFiles[fname]; exists && pf.URL != "" {
if li, ok := i.(LocalImporter); ok {
errs = append(errs, downloadExerciceFile(paramsFiles[fname], li.GetLocalPath(dest), exercice, false)...)
} else {
errs = append(errs, downloadExerciceFile(paramsFiles[fname], dest, exercice, false)...)
}
}
if fd, closer, err := GetFile(i, dest); err != nil {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to read file: %w", err)))
continue
} else {
@ -221,12 +236,6 @@ func CheckExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExc
}
}
paramsFiles, err := GetExerciceFilesParams(i, exercice)
if err != nil {
errs = append(errs, NewChallengeTxtError(exercice, 0, err))
return
}
disclaimer := ""
if f, exists := paramsFiles[fname]; exists {
// Call checks hooks
@ -257,6 +266,50 @@ func CheckExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExc
return
}
// downloadExerciceFile is responsible to fetch remote files.
func downloadExerciceFile(pf ExerciceFile, dest string, exercice *fic.Exercice, force bool) (errs []error) {
if st, err := os.Stat(dest); !force && !os.IsNotExist(err) {
resp, err := http.Head(pf.URL)
if err == nil && resp.ContentLength == st.Size() {
return
}
}
if !isURLAllowed(pf.URL) {
errs = append(errs, NewFileError(exercice, path.Base(dest), fmt.Errorf("URL hostname is not whitelisted")))
return
}
resp, err := http.Get(pf.URL)
if err != nil {
errs = append(errs, NewFileError(exercice, path.Base(dest), err))
return
}
defer resp.Body.Close()
if err = os.MkdirAll(path.Dir(dest), 0751); err != nil {
errs = append(errs, NewFileError(exercice, path.Base(dest), err))
return
}
// Write file
var fdto *os.File
if fdto, err = os.Create(dest); err != nil {
errs = append(errs, NewFileError(exercice, path.Base(dest), err))
return
} else {
defer fdto.Close()
_, err = io.Copy(fdto, resp.Body)
if err != nil {
errs = append(errs, NewFileError(exercice, path.Base(dest), err))
return
}
}
return
}
// SyncExerciceFiles reads the content of files/ directory and import it as EFile for the given challenge.
// It takes care of DIGESTS.txt and ensure imported files match.
func SyncExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExceptions) (errs []error) {
@ -314,39 +367,8 @@ func SyncExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExce
}
}
if !isURLAllowed(pf.URL) {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("URL hostname is not whitelisted")))
continue
}
if f == nil {
var resp *http.Response
resp, err = http.Get(pf.URL)
if err != nil {
errs = append(errs, NewFileError(exercice, fname, err))
continue
}
defer resp.Body.Close()
if err = os.MkdirAll(path.Dir(dest), 0751); err != nil {
errs = append(errs, NewFileError(exercice, fname, err))
continue
}
// Write file
var fdto *os.File
if fdto, err = os.Create(dest); err != nil {
errs = append(errs, NewFileError(exercice, fname, err))
continue
} else {
defer fdto.Close()
_, err = io.Copy(fdto, resp.Body)
if err != nil {
errs = append(errs, NewFileError(exercice, fname, err))
continue
}
}
errs = append(errs, downloadExerciceFile(paramsFiles[fname], dest, exercice, false)...)
f, err = actionAfterImport(dest, pf.URL)
}