fic/server
Archived
1
0
Fork 0
Code Issues Pull requests 10 Releases Wiki Activity

sync: Replace []error by go.uber.org/multierr

Browse source
This commit is contained in:
nemunaire 2023-11-22 12:16:53 +01:00
parent 9f49a689fd
commit b6966d47ce
25 changed files with 380 additions and 348 deletions
Download patch file Download diff file Expand all files Collapse all files

97
admin/sync/exercice_files.go
View file

@ -14,6 +14,7 @@ import (
"unicode"
"github.com/gin-gonic/gin"
"go.uber.org/multierr"
"srs.epita.fr/fic-server/libfic"
)
@ -50,7 +51,7 @@ func isURLAllowed(in string) bool {
return false
}
func BuildFilesListInto(i Importer, exercice *fic.Exercice, into string) (files []string, digests map[string][]byte, errs []error) {
func BuildFilesListInto(i Importer, exercice *fic.Exercice, into string) (files []string, digests map[string][]byte, errs error) {
// If no files directory, don't display error
if !i.Exists(path.Join(exercice.Path, into)) {
return
@ -58,15 +59,15 @@ func BuildFilesListInto(i Importer, exercice *fic.Exercice, into string) (files
// Parse DIGESTS.txt
if digs, err := GetFileContent(i, path.Join(exercice.Path, into, "DIGESTS.txt")); err != nil {
errs = append(errs, NewExerciceError(exercice, fmt.Errorf("unable to read %s: %w", path.Join(into, "DIGESTS.txt"), err)))
errs = multierr.Append(errs, NewExerciceError(exercice, fmt.Errorf("unable to read %s: %w", path.Join(into, "DIGESTS.txt"), err)))
} else {
digests = map[string][]byte{}
for nline, d := range strings.Split(digs, "\n") {
if dsplt := strings.SplitN(d, " ", 2); len(dsplt) < 2 {
errs = append(errs, NewExerciceError(exercice, fmt.Errorf("unable to parse %s line %d: invalid format", path.Join(into, "DIGESTS.txt"), nline+1)))
errs = multierr.Append(errs, NewExerciceError(exercice, fmt.Errorf("unable to parse %s line %d: invalid format", path.Join(into, "DIGESTS.txt"), nline+1)))
continue
} else if hash, err := hex.DecodeString(dsplt[0]); err != nil {
errs = append(errs, NewExerciceError(exercice, fmt.Errorf("unable to parse %s line %d: %w", path.Join(into, "DIGESTS.txt"), nline+1, err)))
errs = multierr.Append(errs, NewExerciceError(exercice, fmt.Errorf("unable to parse %s line %d: %w", path.Join(into, "DIGESTS.txt"), nline+1, err)))
continue
} else {
digests[strings.TrimFunc(dsplt[1], unicode.IsSpace)] = hash
@ -76,7 +77,7 @@ func BuildFilesListInto(i Importer, exercice *fic.Exercice, into string) (files
// Read file list
if flist, err := i.listDir(path.Join(exercice.Path, into)); err != nil {
errs = append(errs, NewExerciceError(exercice, err))
errs = multierr.Append(errs, NewExerciceError(exercice, err))
} else {
for _, fname := range flist {
if fname == "DIGESTS.txt" || fname == ".gitattributes" {
@ -128,9 +129,9 @@ func BuildFilesListInto(i Importer, exercice *fic.Exercice, into string) (files
}
// CheckExerciceFilesPresence limits remote checks to presence, don't get it to check digest.
func CheckExerciceFilesPresence(i Importer, exercice *fic.Exercice) (files []string, errs []error) {
func CheckExerciceFilesPresence(i Importer, exercice *fic.Exercice) (files []string, errs error) {
flist, digests, berrs := BuildFilesListInto(i, exercice, "files")
errs = append(errs, berrs...)
errs = multierr.Append(errs, berrs)
paramsFiles, _ := GetExerciceFilesParams(i, exercice)
@ -138,28 +139,28 @@ func CheckExerciceFilesPresence(i Importer, exercice *fic.Exercice) (files []str
if !i.Exists(path.Join(exercice.Path, "files", fname)) && !i.Exists(path.Join(exercice.Path, "files", fname+".00")) {
// File not found locally, is this a remote file?
if pf, exists := paramsFiles[fname]; !exists || pf.URL == "" {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("No such file or directory")))
errs = multierr.Append(errs, NewFileError(exercice, fname, fmt.Errorf("No such file or directory")))
continue
} else if !isURLAllowed(pf.URL) {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("URL hostname is not whitelisted")))
errs = multierr.Append(errs, NewFileError(exercice, fname, fmt.Errorf("URL hostname is not whitelisted")))
continue
} else {
resp, err := http.Head(pf.URL)
if err != nil {
errs = append(errs, NewFileError(exercice, fname, err))
errs = multierr.Append(errs, NewFileError(exercice, fname, err))
continue
}
defer resp.Body.Close()
if resp.StatusCode >= 300 {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("Unexpected status code for the HTTP response: %d %s", resp.StatusCode, resp.Status)))
errs = multierr.Append(errs, NewFileError(exercice, fname, fmt.Errorf("Unexpected status code for the HTTP response: %d %s", resp.StatusCode, resp.Status)))
continue
}
}
}
if _, ok := digests[fname]; !ok {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to import file: No digest given")))
errs = multierr.Append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to import file: No digest given")))
} else {
files = append(files, fname)
}
@ -169,7 +170,7 @@ func CheckExerciceFilesPresence(i Importer, exercice *fic.Exercice) (files []str
if !i.Exists(path.Join(exercice.Path, "files", fname)) && !i.Exists(path.Join(exercice.Path, "files", fname+".gz")) && !i.Exists(path.Join(exercice.Path, "files", fname+".00")) && !i.Exists(path.Join(exercice.Path, "files", fname+".gz.00")) {
if pf, exists := paramsFiles[fname]; !exists || pf.URL == "" {
if pf, exists := paramsFiles[fname+".gz"]; !exists || pf.URL == "" {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to read file: No such file or directory. Check your DIGESTS.txt for legacy entries.")))
errs = multierr.Append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to read file: No such file or directory. Check your DIGESTS.txt for legacy entries.")))
}
}
}
@ -179,13 +180,13 @@ func CheckExerciceFilesPresence(i Importer, exercice *fic.Exercice) (files []str
}
// CheckExerciceFiles checks that remote files have the right digest.
func CheckExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExceptions) (files []string, errs []error) {
func CheckExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExceptions) (files []string, errs error) {
flist, digests, berrs := BuildFilesListInto(i, exercice, "files")
errs = append(errs, berrs...)
errs = multierr.Append(errs, berrs)
paramsFiles, err := GetExerciceFilesParams(i, exercice)
if err != nil {
errs = append(errs, NewChallengeTxtError(exercice, 0, err))
errs = multierr.Append(errs, NewChallengeTxtError(exercice, 0, err))
}
for _, fname := range flist {
@ -193,14 +194,14 @@ func CheckExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExc
if pf, exists := paramsFiles[fname]; exists && pf.URL != "" {
if li, ok := i.(LocalImporter); ok {
errs = append(errs, downloadExerciceFile(paramsFiles[fname], li.GetLocalPath(dest), exercice, false)...)
errs = multierr.Append(errs, downloadExerciceFile(paramsFiles[fname], li.GetLocalPath(dest), exercice, false))
} else {
errs = append(errs, downloadExerciceFile(paramsFiles[fname], dest, exercice, false)...)
errs = multierr.Append(errs, downloadExerciceFile(paramsFiles[fname], dest, exercice, false))
}
}
if fd, closer, err := GetFile(i, dest); err != nil {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to read file: %w", err)))
errs = multierr.Append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to read file: %w", err)))
continue
} else {
defer closer()
@ -208,9 +209,9 @@ func CheckExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExc
hash160, hash512 := fic.CreateHashBuffers(fd)
if _, err := fic.CheckBufferHash(hash160, hash512, digests[fname]); err != nil {
errs = append(errs, NewFileError(exercice, fname, err))
errs = multierr.Append(errs, NewFileError(exercice, fname, err))
} else if size, err := GetFileSize(i, path.Join(exercice.Path, "files", fname)); err != nil {
errs = append(errs, NewFileError(exercice, fname, err))
errs = multierr.Append(errs, NewFileError(exercice, fname, err))
} else {
var digest_shown []byte
if strings.HasSuffix(fname, ".gz") {
@ -219,11 +220,11 @@ func CheckExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExc
// Check that gunzipped file digest is correct
if fd, closer, err := GetFile(i, path.Join(exercice.Path, "files", fname)); err != nil {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to read file: %w", err)))
errs = multierr.Append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to read file: %w", err)))
continue
} else if gunzipfd, err := gzip.NewReader(fd); err != nil {
closer()
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to gunzip file: %w", err)))
errs = multierr.Append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to gunzip file: %w", err)))
continue
} else {
defer gunzipfd.Close()
@ -232,7 +233,7 @@ func CheckExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExc
hash160_inflate, hash512_inflate := fic.CreateHashBuffers(gunzipfd)
if _, err := fic.CheckBufferHash(hash160_inflate, hash512_inflate, digest_shown); err != nil {
errs = append(errs, NewFileError(exercice, strings.TrimSuffix(fname, ".gz"), err))
errs = multierr.Append(errs, NewFileError(exercice, strings.TrimSuffix(fname, ".gz"), err))
}
}
}
@ -242,13 +243,13 @@ func CheckExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExc
if f, exists := paramsFiles[fname]; exists {
// Call checks hooks
for _, hk := range hooks.mdTextHooks {
for _, err := range hk(f.Disclaimer, exercice.Language, exceptions) {
errs = append(errs, NewFileError(exercice, fname, err))
for _, err := range multierr.Errors(hk(f.Disclaimer, exercice.Language, exceptions)) {
errs = multierr.Append(errs, NewFileError(exercice, fname, err))
}
}
if disclaimer, err = ProcessMarkdown(i, fixnbsp(f.Disclaimer), exercice.Path); err != nil {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("error during markdown formating of disclaimer: %w", err)))
errs = multierr.Append(errs, NewFileError(exercice, fname, fmt.Errorf("error during markdown formating of disclaimer: %w", err)))
}
}
@ -256,8 +257,8 @@ func CheckExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExc
// Call checks hooks
for _, h := range hooks.fileHooks {
for _, e := range h(file, exercice, exceptions) {
errs = append(errs, NewFileError(exercice, fname, e))
for _, e := range multierr.Errors(h(file, exercice, exceptions)) {
errs = multierr.Append(errs, NewFileError(exercice, fname, e))
}
}
}
@ -269,7 +270,7 @@ func CheckExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExc
}
// downloadExerciceFile is responsible to fetch remote files.
func downloadExerciceFile(pf ExerciceFile, dest string, exercice *fic.Exercice, force bool) (errs []error) {
func downloadExerciceFile(pf ExerciceFile, dest string, exercice *fic.Exercice, force bool) (errs error) {
if st, err := os.Stat(dest); !force && !os.IsNotExist(err) {
resp, err := http.Head(pf.URL)
if err == nil && resp.ContentLength == st.Size() {
@ -278,33 +279,35 @@ func downloadExerciceFile(pf ExerciceFile, dest string, exercice *fic.Exercice,
}
if !isURLAllowed(pf.URL) {
errs = append(errs, NewFileError(exercice, path.Base(dest), fmt.Errorf("URL hostname is not whitelisted")))
errs = multierr.Append(errs, NewFileError(exercice, path.Base(dest), fmt.Errorf("URL hostname is not whitelisted")))
return
}
log.Println("Download exercice file: ", pf.URL)
resp, err := http.Get(pf.URL)
if err != nil {
errs = append(errs, NewFileError(exercice, path.Base(dest), err))
errs = multierr.Append(errs, NewFileError(exercice, path.Base(dest), err))
return
}
defer resp.Body.Close()
if err = os.MkdirAll(path.Dir(dest), 0751); err != nil {
errs = append(errs, NewFileError(exercice, path.Base(dest), err))
errs = multierr.Append(errs, NewFileError(exercice, path.Base(dest), err))
return
}
// Write file
var fdto *os.File
if fdto, err = os.Create(dest); err != nil {
errs = append(errs, NewFileError(exercice, path.Base(dest), err))
errs = multierr.Append(errs, NewFileError(exercice, path.Base(dest), err))
return
} else {
defer fdto.Close()
_, err = io.Copy(fdto, resp.Body)
if err != nil {
errs = append(errs, NewFileError(exercice, path.Base(dest), err))
errs = multierr.Append(errs, NewFileError(exercice, path.Base(dest), err))
return
}
}
@ -314,19 +317,19 @@ func downloadExerciceFile(pf ExerciceFile, dest string, exercice *fic.Exercice,
// SyncExerciceFiles reads the content of files/ directory and import it as EFile for the given challenge.
// It takes care of DIGESTS.txt and ensure imported files match.
func SyncExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExceptions) (errs []error) {
func SyncExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExceptions) (errs error) {
if _, err := exercice.WipeFiles(); err != nil {
errs = append(errs, err)
errs = multierr.Append(errs, err)
}
paramsFiles, err := GetExerciceFilesParams(i, exercice)
if err != nil {
errs = append(errs, NewChallengeTxtError(exercice, 0, err))
errs = multierr.Append(errs, NewChallengeTxtError(exercice, 0, err))
return
}
files, digests, berrs := BuildFilesListInto(i, exercice, "files")
errs = append(errs, berrs...)
errs = multierr.Append(errs, berrs)
// Import standard files
for _, fname := range files {
@ -345,13 +348,13 @@ func SyncExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExce
// Call checks hooks
for _, hk := range hooks.mdTextHooks {
for _, err := range hk(f.Disclaimer, exercice.Language, exceptions) {
errs = append(errs, NewFileError(exercice, fname, err))
for _, err := range multierr.Errors(hk(f.Disclaimer, exercice.Language, exceptions)) {
errs = multierr.Append(errs, NewFileError(exercice, fname, err))
}
}
if disclaimer, err = ProcessMarkdown(i, fixnbsp(f.Disclaimer), exercice.Path); err != nil {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("error during markdown formating of disclaimer: %w", err)))
errs = multierr.Append(errs, NewFileError(exercice, fname, fmt.Errorf("error during markdown formating of disclaimer: %w", err)))
}
}
@ -370,7 +373,7 @@ func SyncExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExce
}
if f == nil {
errs = append(errs, downloadExerciceFile(paramsFiles[fname], dest, exercice, false)...)
errs = multierr.Append(errs, downloadExerciceFile(paramsFiles[fname], dest, exercice, false))
f, err = actionAfterImport(dest, pf.URL)
}
@ -379,19 +382,19 @@ func SyncExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExce
}
if err != nil {
errs = append(errs, NewFileError(exercice, fname, err))
errs = multierr.Append(errs, NewFileError(exercice, fname, err))
continue
}
if f.(*fic.EFile).Size == 0 {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("imported file is empty!")))
errs = multierr.Append(errs, NewFileError(exercice, fname, fmt.Errorf("imported file is empty!")))
} else {
file := f.(*fic.EFile)
// Call checks hooks
for _, h := range hooks.fileHooks {
for _, e := range h(file, exercice, exceptions) {
errs = append(errs, NewFileError(exercice, fname, e))
for _, e := range multierr.Errors(h(file, exercice, exceptions)) {
errs = multierr.Append(errs, NewFileError(exercice, fname, e))
}
}