fic/server
Archived
1
0
Fork 0
Code Issues Pull requests 10 Releases Wiki Activity

sync: Handle remote challenge files

Browse source
This commit is contained in:
nemunaire 2023-10-13 16:05:15 +02:00
parent ec3f818c30
commit 1d4b79bf90
5 changed files with 126 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

1
admin/sync/exercice_defines.go
View file

@ -31,6 +31,7 @@ type ExerciceUnlockFile struct {
// ExerciceFile defines attributes on files.
type ExerciceFile struct {
Filename string `toml:",omitempty"`
URL string `toml:",omitempty"`
Hidden bool `toml:",omitempty"`
Disclaimer string `toml:",omitempty"`
}

141
admin/sync/exercice_files.go
View file

@ -4,6 +4,7 @@ import (
"compress/gzip"
"encoding/hex"
"fmt"
"io"
"log"
"net/http"
"os"
@ -71,6 +72,25 @@ func BuildFilesListInto(i Importer, exercice *fic.Exercice, into string) (files
}
}
// Complete with remote file names
if paramsFiles, err := GetExerciceFilesParams(i, exercice); err == nil {
for _, pf := range paramsFiles {
if pf.URL != "" {
found := false
for _, file := range files {
if file == pf.Filename {
found = true
break
}
}
if !found {
files = append(files, pf.Filename)
}
}
}
}
return
}
@ -79,10 +99,30 @@ func CheckExerciceFilesPresence(i Importer, exercice *fic.Exercice) (files []str
flist, digests, berrs := BuildFilesListInto(i, exercice, "files")
errs = append(errs, berrs...)
paramsFiles, _ := GetExerciceFilesParams(i, exercice)
for _, fname := range flist {
if !i.Exists(path.Join(exercice.Path, "files", fname)) && !i.Exists(path.Join(exercice.Path, "files", fname+".00")) {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("No such file or directory")))
} else if _, ok := digests[fname]; !ok {
// File not found locally, is this a remote file?
if pf, exists := paramsFiles[fname]; !exists || pf.URL == "" {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("No such file or directory")))
continue
} else {
resp, err := http.Head(pf.URL)
if err != nil {
errs = append(errs, NewFileError(exercice, fname, err))
continue
}
defer resp.Body.Close()
if resp.StatusCode >= 300 {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("Unexpected status code for the HTTP response: %d %s", resp.StatusCode, resp.Status)))
continue
}
}
}
if _, ok := digests[fname]; !ok {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to import file: No digest given")))
} else {
files = append(files, fname)
@ -91,7 +131,9 @@ func CheckExerciceFilesPresence(i Importer, exercice *fic.Exercice) (files []str
for fname := range digests {
if !i.Exists(path.Join(exercice.Path, "files", fname)) && !i.Exists(path.Join(exercice.Path, "files", fname+".gz")) && !i.Exists(path.Join(exercice.Path, "files", fname+".00")) && !i.Exists(path.Join(exercice.Path, "files", fname+".gz.00")) {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to read file: No such file or directory. Check your DIGESTS.txt for legacy entries.")))
if pf, exists := paramsFiles[fname]; !exists || pf.URL == "" {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("unable to read file: No such file or directory. Check your DIGESTS.txt for legacy entries.")))
}
}
}
@ -163,7 +205,7 @@ func CheckExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExc
}
}
file := exercice.NewDummyFile(path.Join(exercice.Path, "files", fname), getDestinationFilePath(path.Join(exercice.Path, "files", fname)), (*hash512).Sum(nil), digest_shown, disclaimer, size)
file := exercice.NewDummyFile(path.Join(exercice.Path, "files", fname), getDestinationFilePath(path.Join(exercice.Path, "files", fname), nil), (*hash512).Sum(nil), digest_shown, disclaimer, size)
// Call checks hooks
for _, h := range hooks.fileHooks {
@ -197,37 +239,86 @@ func SyncExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExce
// Import standard files
for _, fname := range files {
if f, err := i.importFile(path.Join(exercice.Path, "files", fname),
func(filePath string, origin string) (interface{}, error) {
var digest_shown []byte
if strings.HasSuffix(fname, ".gz") {
if d, exists := digests[strings.TrimSuffix(fname, ".gz")]; exists {
digest_shown = d
actionAfterImport := func(filePath string, origin string) (interface{}, error) {
var digest_shown []byte
if strings.HasSuffix(fname, ".gz") {
if d, exists := digests[strings.TrimSuffix(fname, ".gz")]; exists {
digest_shown = d
}
}
published := true
disclaimer := ""
if f, exists := paramsFiles[fname]; exists {
published = !f.Hidden
// Call checks hooks
for _, hk := range hooks.mdTextHooks {
for _, err := range hk(f.Disclaimer, exercice.Language, exceptions) {
errs = append(errs, NewFileError(exercice, fname, err))
}
}
published := true
disclaimer := ""
if f, exists := paramsFiles[fname]; exists {
published = !f.Hidden
if disclaimer, err = ProcessMarkdown(i, fixnbsp(f.Disclaimer), exercice.Path); err != nil {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("error during markdown formating of disclaimer: %w", err)))
}
}
// Call checks hooks
for _, hk := range hooks.mdTextHooks {
for _, err := range hk(f.Disclaimer, exercice.Language, exceptions) {
errs = append(errs, NewFileError(exercice, fname, err))
}
}
return exercice.ImportFile(filePath, origin, digests[fname], digest_shown, disclaimer, published)
}
if disclaimer, err = ProcessMarkdown(i, fixnbsp(f.Disclaimer), exercice.Path); err != nil {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("error during markdown formating of disclaimer: %w", err)))
var f interface{}
if pf, exists := paramsFiles[fname]; exists && pf.URL != "" {
dest := getDestinationFilePath(pf.URL, &pf.Filename)
if _, err := os.Stat(dest); !os.IsNotExist(err) {
if d, err := actionAfterImport(dest, pf.URL); err == nil {
f = d
}
}
if f == nil {
var resp *http.Response
resp, err = http.Get(pf.URL)
if err != nil {
errs = append(errs, NewFileError(exercice, fname, err))
continue
}
defer resp.Body.Close()
if err = os.MkdirAll(path.Dir(dest), 0751); err != nil {
errs = append(errs, NewFileError(exercice, fname, err))
continue
}
// Write file
var fdto *os.File
if fdto, err = os.Create(dest); err != nil {
errs = append(errs, NewFileError(exercice, fname, err))
continue
} else {
defer fdto.Close()
_, err = io.Copy(fdto, resp.Body)
if err != nil {
errs = append(errs, NewFileError(exercice, fname, err))
continue
}
}
return exercice.ImportFile(filePath, origin, digests[fname], digest_shown, disclaimer, published)
}); err != nil {
f, err = actionAfterImport(dest, pf.URL)
}
} else {
f, err = i.importFile(path.Join(exercice.Path, "files", fname), actionAfterImport)
}
if err != nil {
errs = append(errs, NewFileError(exercice, fname, err))
continue
} else if f.(*fic.EFile).Size == 0 {
}
if f.(*fic.EFile).Size == 0 {
errs = append(errs, NewFileError(exercice, fname, fmt.Errorf("imported file is empty!")))
} else {
file := f.(*fic.EFile)

10
admin/sync/file.go
View file

@ -183,9 +183,13 @@ func GetFileContent(i Importer, URI string) (string, error) {
// getDestinationFilePath generates the destination path, from the URI.
// This function permits to obfusce to player the original URI.
// Theoricaly, changing the import method doesn't change destination URI.
func getDestinationFilePath(URI string) string {
func getDestinationFilePath(URI string, filename *string) string {
if filename == nil {
tmp := path.Base(URI)
filename = &tmp
}
hash := blake2b.Sum512([]byte(URI))
return path.Join(fic.FilesDir, strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:])), path.Base(URI))
return path.Join(fic.FilesDir, strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:])), *filename)
}
func importFile(i Importer, URI string, dest string) error {
@ -214,7 +218,7 @@ func importFile(i Importer, URI string, dest string) error {
// ImportFile imports the file at the given URI, using helpers of the given Importer.
// After import, next is called with relative path where the file has been saved and the original URI.
func ImportFile(i Importer, URI string, next func(string, string) (interface{}, error)) (interface{}, error) {
dest := getDestinationFilePath(URI)
dest := getDestinationFilePath(URI, nil)
// If the present file is still valide, don't erase it
if _, err := os.Stat(dest); !os.IsNotExist(err) {

2
admin/sync/importer_localfs.go
View file

@ -62,7 +62,7 @@ func (i LocalImporter) GetLocalPath(p ...string) string {
func (i LocalImporter) importFile(URI string, next func(string, string) (interface{}, error)) (interface{}, error) {
if i.Symlink {
dest := getDestinationFilePath(URI)
dest := getDestinationFilePath(URI, nil)
if err := os.MkdirAll(path.Dir(dest), 0751); err != nil {
return nil, err

2
libfic/file.go
View file

@ -234,7 +234,7 @@ func checkFileHash(filePath string, digest []byte) (dgst []byte, size int64, err
// ImportFile registers (ou updates if it already exists in database) the file in database.
func (e *Exercice) ImportFile(filePath string, origin string, digest []byte, digestshown []byte, disclaimer string, published bool) (interface{}, error) {
if result512, size, err := checkFileHash(filePath, digest); !OptionalDigest && err != nil {
return EFile{}, err
return nil, err
} else {
dPath := strings.TrimPrefix(filePath, FilesDir)
if f, err := e.GetFileByPath(dPath); err != nil {