sync: Handle remote challenge files

admin/sync/exercice_defines.go

@@ -31,6 +31,7 @@ type ExerciceUnlockFile struct {
 // ExerciceFile defines attributes on files.
 type ExerciceFile struct {
 	Filename   string `toml:",omitempty"`
+	URL        string `toml:",omitempty"`
 	Hidden     bool   `toml:",omitempty"`
 	Disclaimer string `toml:",omitempty"`
 }

admin/sync/exercice_files.go

@@ -4,6 +4,7 @@ import (
 	"compress/gzip"
 	"encoding/hex"
 	"fmt"
+	"io"
 	"log"
 	"net/http"
 	"os"
@@ -71,6 +72,25 @@ func BuildFilesListInto(i Importer, exercice *fic.Exercice, into string) (files
 		}
 	}
 
+	// Complete with remote file names
+	if paramsFiles, err := GetExerciceFilesParams(i, exercice); err == nil {
+		for _, pf := range paramsFiles {
+			if pf.URL != "" {
+				found := false
+				for _, file := range files {
+					if file == pf.Filename {
+						found = true
+						break
+					}
+				}
+
+				if !found {
+					files = append(files, pf.Filename)
+				}
+			}
+		}
+	}
+
 	return
 }
 
@@ -79,10 +99,30 @@ func CheckExerciceFilesPresence(i Importer, exercice *fic.Exercice) (files []str
 	flist, digests, berrs := BuildFilesListInto(i, exercice, "files")
 	errs = append(errs, berrs...)
 
+	paramsFiles, _ := GetExerciceFilesParams(i, exercice)
+
 	for _, fname := range flist {
 		if !i.Exists(path.Join(exercice.Path, "files", fname)) && !i.Exists(path.Join(exercice.Path, "files", fname+".00")) {
-			errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("No such file or directory")))
+			// File not found locally, is this a remote file?
-		} else if _, ok := digests[fname]; !ok {
+			if pf, exists := paramsFiles[fname]; !exists || pf.URL == "" {
+				errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("No such file or directory")))
+				continue
+			} else {
+				resp, err := http.Head(pf.URL)
+				if err != nil {
+					errs = append(errs, NewFileError(exercice, fname, err))
+					continue
+				}
+				defer resp.Body.Close()
+
+				if resp.StatusCode >= 300 {
+					errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("Unexpected status code for the HTTP response: %d %s", resp.StatusCode, resp.Status)))
+					continue
+				}
+			}
+		}
+
+		if _, ok := digests[fname]; !ok {
 			errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to import file: No digest given")))
 		} else {
 			files = append(files, fname)
@@ -91,7 +131,9 @@ func CheckExerciceFilesPresence(i Importer, exercice *fic.Exercice) (files []str
 
 	for fname := range digests {
 		if !i.Exists(path.Join(exercice.Path, "files", fname)) && !i.Exists(path.Join(exercice.Path, "files", fname+".gz")) && !i.Exists(path.Join(exercice.Path, "files", fname+".00")) && !i.Exists(path.Join(exercice.Path, "files", fname+".gz.00")) {
-			errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to read file: No such file or directory. Check your DIGESTS.txt for legacy entries.")))
+			if pf, exists := paramsFiles[fname]; !exists || pf.URL == "" {
+				errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to read file: No such file or directory. Check your DIGESTS.txt for legacy entries.")))
+			}
 		}
 	}
 
@@ -163,7 +205,7 @@ func CheckExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExc
 					}
 				}
 
-				file := exercice.NewDummyFile(path.Join(exercice.Path, "files", fname), getDestinationFilePath(path.Join(exercice.Path, "files", fname)), (*hash512).Sum(nil), digest_shown, disclaimer, size)
+				file := exercice.NewDummyFile(path.Join(exercice.Path, "files", fname), getDestinationFilePath(path.Join(exercice.Path, "files", fname), nil), (*hash512).Sum(nil), digest_shown, disclaimer, size)
 
 				// Call checks hooks
 				for _, h := range hooks.fileHooks {
@@ -197,37 +239,86 @@ func SyncExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExce
 
 	// Import standard files
 	for _, fname := range files {
-		if f, err := i.importFile(path.Join(exercice.Path, "files", fname),
+		actionAfterImport := func(filePath string, origin string) (interface{}, error) {
-			func(filePath string, origin string) (interface{}, error) {
+			var digest_shown []byte
-				var digest_shown []byte
+			if strings.HasSuffix(fname, ".gz") {
-				if strings.HasSuffix(fname, ".gz") {
+				if d, exists := digests[strings.TrimSuffix(fname, ".gz")]; exists {
-					if d, exists := digests[strings.TrimSuffix(fname, ".gz")]; exists {
+					digest_shown = d
-						digest_shown = d
+				}
+			}
+
+			published := true
+			disclaimer := ""
+			if f, exists := paramsFiles[fname]; exists {
+				published = !f.Hidden
+
+				// Call checks hooks
+				for _, hk := range hooks.mdTextHooks {
+					for _, err := range hk(f.Disclaimer, exercice.Language, exceptions) {
+						errs = append(errs, NewFileError(exercice, fname, err))
 					}
 				}
 
-				published := true
+				if disclaimer, err = ProcessMarkdown(i, fixnbsp(f.Disclaimer), exercice.Path); err != nil {
-				disclaimer := ""
+					errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("error during markdown formating of disclaimer: %w", err)))
-				if f, exists := paramsFiles[fname]; exists {
+				}
-					published = !f.Hidden
+			}
 
-					// Call checks hooks
+			return exercice.ImportFile(filePath, origin, digests[fname], digest_shown, disclaimer, published)
-					for _, hk := range hooks.mdTextHooks {
+		}
-						for _, err := range hk(f.Disclaimer, exercice.Language, exceptions) {
-							errs = append(errs, NewFileError(exercice, fname, err))
-						}
-					}
 
-					if disclaimer, err = ProcessMarkdown(i, fixnbsp(f.Disclaimer), exercice.Path); err != nil {
+		var f interface{}
-						errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("error during markdown formating of disclaimer: %w", err)))
+
+		if pf, exists := paramsFiles[fname]; exists && pf.URL != "" {
+			dest := getDestinationFilePath(pf.URL, &pf.Filename)
+
+			if _, err := os.Stat(dest); !os.IsNotExist(err) {
+				if d, err := actionAfterImport(dest, pf.URL); err == nil {
+					f = d
+				}
+			}
+
+			if f == nil {
+				var resp *http.Response
+				resp, err = http.Get(pf.URL)
+				if err != nil {
+					errs = append(errs, NewFileError(exercice, fname, err))
+					continue
+				}
+				defer resp.Body.Close()
+
+				if err = os.MkdirAll(path.Dir(dest), 0751); err != nil {
+					errs = append(errs, NewFileError(exercice, fname, err))
+					continue
+				}
+
+				// Write file
+				var fdto *os.File
+				if fdto, err = os.Create(dest); err != nil {
+					errs = append(errs, NewFileError(exercice, fname, err))
+					continue
+				} else {
+					defer fdto.Close()
+
+					_, err = io.Copy(fdto, resp.Body)
+					if err != nil {
+						errs = append(errs, NewFileError(exercice, fname, err))
+						continue
 					}
 				}
 
-				return exercice.ImportFile(filePath, origin, digests[fname], digest_shown, disclaimer, published)
+				f, err = actionAfterImport(dest, pf.URL)
-			}); err != nil {
+			}
+		} else {
+			f, err = i.importFile(path.Join(exercice.Path, "files", fname), actionAfterImport)
+		}
+
+		if err != nil {
 			errs = append(errs, NewFileError(exercice, fname, err))
 			continue
-		} else if f.(*fic.EFile).Size == 0 {
+		}
+
+		if f.(*fic.EFile).Size == 0 {
 			errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("imported file is empty!")))
 		} else {
 			file := f.(*fic.EFile)

admin/sync/file.go

@@ -183,9 +183,13 @@ func GetFileContent(i Importer, URI string) (string, error) {
 // getDestinationFilePath generates the destination path, from the URI.
 // This function permits to obfusce to player the original URI.
 // Theoricaly, changing the import method doesn't change destination URI.
-func getDestinationFilePath(URI string) string {
+func getDestinationFilePath(URI string, filename *string) string {
+	if filename == nil {
+		tmp := path.Base(URI)
+		filename = &tmp
+	}
 	hash := blake2b.Sum512([]byte(URI))
-	return path.Join(fic.FilesDir, strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:])), path.Base(URI))
+	return path.Join(fic.FilesDir, strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:])), *filename)
 }
 
 func importFile(i Importer, URI string, dest string) error {
@@ -214,7 +218,7 @@ func importFile(i Importer, URI string, dest string) error {
 // ImportFile imports the file at the given URI, using helpers of the given Importer.
 // After import, next is called with relative path where the file has been saved and the original URI.
 func ImportFile(i Importer, URI string, next func(string, string) (interface{}, error)) (interface{}, error) {
-	dest := getDestinationFilePath(URI)
+	dest := getDestinationFilePath(URI, nil)
 
 	// If the present file is still valide, don't erase it
 	if _, err := os.Stat(dest); !os.IsNotExist(err) {

admin/sync/importer_localfs.go

@@ -62,7 +62,7 @@ func (i LocalImporter) GetLocalPath(p ...string) string {
 
 func (i LocalImporter) importFile(URI string, next func(string, string) (interface{}, error)) (interface{}, error) {
 	if i.Symlink {
-		dest := getDestinationFilePath(URI)
+		dest := getDestinationFilePath(URI, nil)
 
 		if err := os.MkdirAll(path.Dir(dest), 0751); err != nil {
 			return nil, err

libfic/file.go

@@ -234,7 +234,7 @@ func checkFileHash(filePath string, digest []byte) (dgst []byte, size int64, err
 // ImportFile registers (ou updates if it already exists in database) the file in database.
 func (e *Exercice) ImportFile(filePath string, origin string, digest []byte, digestshown []byte, disclaimer string, published bool) (interface{}, error) {
 	if result512, size, err := checkFileHash(filePath, digest); !OptionalDigest && err != nil {
-		return EFile{}, err
+		return nil, err
 	} else {
 		dPath := strings.TrimPrefix(filePath, FilesDir)
 		if f, err := e.GetFileByPath(dPath); err != nil {