fic/server
Archived
1
0
Fork 0
Code Issues Pull requests 10 Releases Wiki Activity

sync: Extract function that import files from importer

Browse source
This commit is contained in:
nemunaire 2019-07-05 19:07:28 +02:00
commit 3f55845374
3 changed files with 131 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

121
admin/sync/exercice_files.go
View file

@ -1,6 +1,7 @@
package sync package sync
import ( import (
"bufio"
"encoding/hex" "encoding/hex"
"fmt" "fmt"
"path" "path"
@ -10,76 +11,104 @@ import (
"srs.epita.fr/fic-server/libfic" "srs.epita.fr/fic-server/libfic"
) )
// SyncExerciceFiles reads the content of files/ directory and import it as EFile for the given challenge. func buildFilesListInto(i Importer, exercice fic.Exercice, into string) (files []string, digests map[string][]byte, errs []string) {
// It takes care of DIGESTS.txt and ensure imported files match.
func SyncExerciceFiles(i Importer, exercice fic.Exercice) (errs []string) {
// If no files directory, don't display error // If no files directory, don't display error
if ! i.exists(path.Join(exercice.Path, "files")) { if !i.exists(path.Join(exercice.Path, into)) {
return return
} }
if digs, err := getFileContent(i, path.Join(exercice.Path, "files", "DIGESTS.txt")); err != nil { // Parse DIGESTS.txt
if digs, err := getFileContent(i, path.Join(exercice.Path, into, "DIGESTS.txt")); err != nil {
errs = append(errs, fmt.Sprintf("%q: unable to read DIGESTS.txt: %s", path.Base(exercice.Path), err)) errs = append(errs, fmt.Sprintf("%q: unable to read DIGESTS.txt: %s", path.Base(exercice.Path), err))
} else if _, err := exercice.WipeFiles(); err != nil {
errs = append(errs, err.Error())
} else if files, err := i.listDir(path.Join(exercice.Path, "files")); err != nil {
errs = append(errs, err.Error())
} else { } else {
// Parse DIGESTS.txt digests = map[string][]byte{}
digests := map[string][]byte{}
for nline, d := range strings.Split(digs, "\n") { for nline, d := range strings.Split(digs, "\n") {
if dsplt := strings.SplitN(d, " ", 2); len(dsplt) < 2 { if dsplt := strings.SplitN(d, " ", 2); len(dsplt) < 2 {
errs = append(errs, fmt.Sprintf("%q: unable to parse DIGESTS.txt line %d: invalid format", path.Base(exercice.Path), nline + 1)) errs = append(errs, fmt.Sprintf("%q: unable to parse DIGESTS.txt line %d: invalid format", path.Base(exercice.Path), nline+1))
continue continue
} else if hash, err := hex.DecodeString(dsplt[0]); err != nil { } else if hash, err := hex.DecodeString(dsplt[0]); err != nil {
errs = append(errs, fmt.Sprintf("%q: unable to parse DIGESTS.txt line %d: %s", path.Base(exercice.Path), nline + 1, err)) errs = append(errs, fmt.Sprintf("%q: unable to parse DIGESTS.txt line %d: %s", path.Base(exercice.Path), nline+1, err))
continue continue
} else { } else {
digests[strings.TrimFunc(dsplt[1], unicode.IsSpace)] = hash digests[strings.TrimFunc(dsplt[1], unicode.IsSpace)] = hash
} }
} }
}
// Import splitted files // Read file list
var splittedFiles []string if flist, err := i.listDir(path.Join(exercice.Path, into)); err != nil {
for _, fname := range files { errs = append(errs, err.Error())
if matched, _ := path.Match("*.00", fname); matched { } else {
fname = fname[:len(fname)-3] for _, fname := range flist {
splittedFiles = append(splittedFiles, fname + ".[0-9][0-9]")
files = append(files, fname)
} else if matched, _ := path.Match("*00", fname); matched {
fname = fname[:len(fname)-2]
splittedFiles = append(splittedFiles, fname + "[0-9][0-9]")
files = append(files, fname)
} else if matched, _ := path.Match("*_MERGED", fname); matched {
splittedFiles = append(splittedFiles, fname)
}
}
// Import standard files
for _, fname := range files {
if fname == "DIGESTS.txt" { if fname == "DIGESTS.txt" {
continue continue
} }
foundSplitted := false if matched, _ := path.Match("*.00", fname); matched {
for _, sf := range splittedFiles { fname = fname[:len(fname)-3]
if matched, _ := path.Match(sf, fname); matched { } else if matched, _ := path.Match("*00", fname); matched {
foundSplitted = true fname = fname[:len(fname)-2]
} } else if matched, _ := path.Match("*_MERGED", fname); matched {
}
if foundSplitted {
continue continue
} }
if f, err := i.importFile(path.Join(exercice.Path, "files", fname), fileFound := false
func(filePath string, origin string) (interface{}, error) { for _, f := range files {
return exercice.ImportFile(filePath, origin, digests[fname]) if fname == f {
}); err != nil { fileFound = true
errs = append(errs, fmt.Sprintf("%q: unable to import file %q: %s", path.Base(exercice.Path), fname, err)) break
continue
} else if f.(fic.EFile).Size == 0 {
errs = append(errs, fmt.Sprintf("%q: WARNING imported file %q is empty!", path.Base(exercice.Path), fname))
} }
}
if !fileFound {
files = append(files, fname)
}
}
}
return
}
// CheckExerciceFiles checks that remote files have the right digest.
func CheckExerciceFiles(i Importer, exercice fic.Exercice) (files []fic.EFile, errs []string) {
flist, digests, berrs := buildFilesListInto(i, exercice, "files")
errs = append(errs, berrs...)
for _, fname := range flist {
w, hash160, hash512 := fic.CreateHashBuffers()
if err := i.getFile(path.Join(exercice.Path, "files", fname), bufio.NewWriter(w)); err != nil {
errs = append(errs, fmt.Sprintf("%q: unable to read file %q: %s", path.Base(exercice.Path), fname, err))
continue
} else if _, err := fic.CheckBufferHash(hash160, hash512, digests[fname]); err != nil {
errs = append(errs, fmt.Sprintf("%q: %s: %s", path.Base(exercice.Path), fname, err))
}
files = append(files, fic.EFile{Name: fname})
}
return
}
// SyncExerciceFiles reads the content of files/ directory and import it as EFile for the given challenge.
// It takes care of DIGESTS.txt and ensure imported files match.
func SyncExerciceFiles(i Importer, exercice fic.Exercice) (errs []string) {
if _, err := exercice.WipeFiles(); err != nil {
errs = append(errs, err.Error())
}
files, digests, berrs := buildFilesListInto(i, exercice, "files")
errs = append(errs, berrs...)
// Import standard files
for _, fname := range files {
if f, err := i.importFile(path.Join(exercice.Path, "files", fname),
func(filePath string, origin string) (interface{}, error) {
return exercice.ImportFile(filePath, origin, digests[fname])
}); err != nil {
errs = append(errs, fmt.Sprintf("%q: unable to import file %q: %s", path.Base(exercice.Path), fname, err))
continue
} else if f.(fic.EFile).Size == 0 {
errs = append(errs, fmt.Sprintf("%q: WARNING imported file %q is empty!", path.Base(exercice.Path), fname))
} }
} }
return return

4
admin/sync/file.go
View file

@ -58,7 +58,7 @@ func getFileSize(i Importer, URI string) (size int64, err error) {
for _, fname := range []string{filename, filename + "."} { for _, fname := range []string{filename, filename + "."} {
found := false found := false
for _, file := range files { for _, file := range files {
if matched, _ := path.Match(fname + "[0-9][0-9]", file); matched { if matched, _ := path.Match(fname+"[0-9][0-9]", file); matched {
found = true found = true
if fi, err := i.stat(path.Join(dirname, file)); err != nil { if fi, err := i.stat(path.Join(dirname, file)); err != nil {
return size, err return size, err
@ -95,7 +95,7 @@ func getFile(i Importer, URI string, writer *bufio.Writer) error {
for _, fname := range []string{filename, filename + "."} { for _, fname := range []string{filename, filename + "."} {
found := false found := false
for _, file := range files { for _, file := range files {
if matched, _ := path.Match(fname + "[0-9][0-9]", file); matched { if matched, _ := path.Match(fname+"[0-9][0-9]", file); matched {
found = true found = true
if err := i.getFile(path.Join(dirname, file), writer); err != nil { if err := i.getFile(path.Join(dirname, file), writer); err != nil {
return err return err

101
libfic/file.go
View file

@ -6,6 +6,7 @@ import (
_ "crypto/sha1" _ "crypto/sha1"
"encoding/hex" "encoding/hex"
"errors" "errors"
"hash"
"io" "io"
"os" "os"
"path" "path"
@ -28,19 +29,19 @@ var PlainDigest bool = false
// EFile represents a challenge file. // EFile represents a challenge file.
type EFile struct { type EFile struct {
Id int64 `json:"id"` Id int64 `json:"id"`
// origin holds the import relative path of the file // origin holds the import relative path of the file
origin string origin string
// Path is the location where the file is stored, relatively to FilesDir // Path is the location where the file is stored, relatively to FilesDir
Path string `json:"path"` Path string `json:"path"`
// IdExercice is the identifier of the underlying challenge // IdExercice is the identifier of the underlying challenge
IdExercice int64 `json:"idExercice"` IdExercice int64 `json:"idExercice"`
// Name is the title displayed to players // Name is the title displayed to players
Name string `json:"name"` Name string `json:"name"`
// Checksum stores the cached hash of the file // Checksum stores the cached hash of the file
Checksum []byte `json:"checksum"` Checksum []byte `json:"checksum"`
// Size contains the cached size of the file // Size contains the cached size of the file
Size int64 `json:"size"` Size int64 `json:"size"`
} }
// GetFiles returns a list of all files living in the database. // GetFiles returns a list of all files living in the database.
@ -88,7 +89,7 @@ func GetFileByPath(path string) (EFile, error) {
func (e Exercice) GetFileByFilename(filename string) (f EFile, err error) { func (e Exercice) GetFileByFilename(filename string) (f EFile, err error) {
filename = path.Base(filename) filename = path.Base(filename)
err = DBQueryRow("SELECT id_file, origin, path, id_exercice, name, cksum, size FROM exercice_files WHERE id_exercice = ? AND origin LIKE ?", e.Id, "%/" + filename).Scan(&f.Id, &f.origin, &f.Path, &f.IdExercice, &f.Name, &f.Checksum, &f.Size) err = DBQueryRow("SELECT id_file, origin, path, id_exercice, name, cksum, size FROM exercice_files WHERE id_exercice = ? AND origin LIKE ?", e.Id, "%/"+filename).Scan(&f.Id, &f.origin, &f.Path, &f.IdExercice, &f.Name, &f.Checksum, &f.Size)
return return
} }
@ -139,50 +140,66 @@ func minifyHash(hash string) string {
} }
} }
// CheckBufferHash checks if the bufio has the given digest.
func CreateHashBuffers() (io.Writer, *hash.Hash, *hash.Hash) {
hash160 := crypto.SHA1.New()
hash512 := crypto.BLAKE2b_512.New()
w := io.MultiWriter(hash160, hash512)
return w, &hash160, &hash512
}
// CheckBufferHash checks if the bufio has the given digest.
func CheckBufferHash(hash160 *hash.Hash, hash512 *hash.Hash, digest []byte) ([]byte, error) {
result160 := (*hash160).Sum(nil)
result512 := (*hash512).Sum(nil)
if len(digest) != len(result512) {
if len(digest) != len(result160) {
return []byte{}, errors.New("Digests doesn't match: calculated: sha1:" + minifyHash(hex.EncodeToString(result160)) + " & blake2b:" + minifyHash(hex.EncodeToString(result512)) + " vs. given: " + hex.EncodeToString(digest))
} else if StrongDigest {
return []byte{}, errors.New("Invalid digests: SHA-1 checksums are no more accepted. Calculated sha1:" + minifyHash(hex.EncodeToString(result160)) + " & blake2b:" + minifyHash(hex.EncodeToString(result512)) + " vs. given: " + hex.EncodeToString(digest))
}
for k := range result160 {
if result160[k] != digest[k] {
return []byte{}, errors.New("Digests doesn't match: calculated: sha1:" + minifyHash(hex.EncodeToString(result160)) + " & blake2b:" + minifyHash(hex.EncodeToString(result512)) + " vs. given: " + hex.EncodeToString(digest))
}
}
} else {
for k := range result512 {
if result512[k] != digest[k] {
return []byte{}, errors.New("Digests doesn't match: calculated: " + minifyHash(hex.EncodeToString(result512)) + " vs. given: " + hex.EncodeToString(digest))
}
}
}
return result512, nil
}
// checkFileHash checks if the file at the given filePath has the given digest. // checkFileHash checks if the file at the given filePath has the given digest.
// It also returns the file's size. // It also returns the file's size.
func checkFileHash(filePath string, digest []byte) ([]byte, int64, error) { func checkFileHash(filePath string, digest []byte) (dgst []byte, size int64, err error) {
if digest == nil { if digest == nil {
return []byte{}, 0, errors.New("No digest given.") return []byte{}, 0, errors.New("No digest given.")
} else if fi, err := os.Stat(filePath); err != nil { } else if fi, errr := os.Stat(filePath); err != nil {
return []byte{}, 0, err return []byte{}, 0, errr
} else if fd, err := os.Open(filePath); err != nil { } else if fd, errr := os.Open(filePath); err != nil {
return []byte{}, fi.Size(), err return []byte{}, fi.Size(), errr
} else { } else {
defer fd.Close() defer fd.Close()
size = fi.Size()
reader := bufio.NewReader(fd) w, hash160, hash512 := CreateHashBuffers()
hash160 := crypto.SHA1.New()
hash512 := crypto.BLAKE2b_512.New()
w := io.MultiWriter(hash160, hash512) if _, err = io.Copy(w, bufio.NewReader(fd)); err != nil {
if _, err := io.Copy(w, reader); err != nil { return
return []byte{}, fi.Size(), err
}
result160 := hash160.Sum(nil)
result512 := hash512.Sum(nil)
if len(digest) != len(result512) {
if len(digest) != len(result160) {
return []byte{}, fi.Size(), errors.New("Digests doesn't match: calculated: sha1:" + minifyHash(hex.EncodeToString(result160)) + " & blake2b:" + minifyHash(hex.EncodeToString(result512)) + " vs. given: " + hex.EncodeToString(digest))
} else if StrongDigest {
return []byte{}, fi.Size(), errors.New("Invalid digests: SHA-1 checksums are no more accepted. Calculated sha1:" + minifyHash(hex.EncodeToString(result160)) + " & blake2b:" + minifyHash(hex.EncodeToString(result512)) + " vs. given: " + hex.EncodeToString(digest))
}
for k := range result160 {
if result160[k] != digest[k] {
return []byte{}, fi.Size(), errors.New("Digests doesn't match: calculated: sha1:" + minifyHash(hex.EncodeToString(result160)) + " & blake2b:" + minifyHash(hex.EncodeToString(result512)) + " vs. given: " + hex.EncodeToString(digest))
}
}
} else {
for k := range result512 {
if result512[k] != digest[k] {
return []byte{}, fi.Size(), errors.New("Digests doesn't match: calculated: " + minifyHash(hex.EncodeToString(result512)) + " vs. given: " + hex.EncodeToString(digest))
}
}
} }
return result512, fi.Size(), nil dgst, err = CheckBufferHash(hash160, hash512, digest)
return
} }
} }