fic/server
Archived
1
0
Fork 0
Code Issues Pull requests 10 Releases Wiki Activity

db: Add cksum_shown field to files in order to store second checksum in case of gziped content

Browse source
This commit is contained in:
nemunaire 2022-10-31 18:29:41 +01:00
parent 91b2daea2e
commit 6b7ed273b7
5 changed files with 40 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

2
admin/api/file.go
View file

@ -178,7 +178,7 @@ func createExerciceFile(c *gin.Context) {
if digest, err := hex.DecodeString(uf.Digest); err != nil { if digest, err := hex.DecodeString(uf.Digest); err != nil {
return nil, err return nil, err
} else { } else {
return exercice.(*fic.Exercice).ImportFile(filePath, origin, digest) return exercice.(*fic.Exercice).ImportFile(filePath, origin, digest, nil)
} }
}) })
if err != nil { if err != nil {

8
admin/sync/exercice_files.go
View file

@ -141,7 +141,13 @@ func SyncExerciceFiles(i Importer, exercice *fic.Exercice, exceptions *CheckExce
for _, fname := range files { for _, fname := range files {
if f, err := i.importFile(path.Join(exercice.Path, "files", fname), if f, err := i.importFile(path.Join(exercice.Path, "files", fname),
func(filePath string, origin string) (interface{}, error) { func(filePath string, origin string) (interface{}, error) {
return exercice.ImportFile(filePath, origin, digests[fname]) var digest_shown []byte
if strings.HasSuffix(fname, ".gz") {
if d, exists := digests[strings.TrimSuffix(fname, ".gz")]; exists {
digest_shown = d
}
}
return exercice.ImportFile(filePath, origin, digests[fname], digest_shown)
}); err != nil { }); err != nil {
errs = append(errs, NewFileError(exercice, fname, err)) errs = append(errs, NewFileError(exercice, fname, err))
continue continue

1
libfic/db.go
View file

@ -168,6 +168,7 @@ CREATE TABLE IF NOT EXISTS exercice_files(
id_exercice INTEGER NOT NULL, id_exercice INTEGER NOT NULL,
name VARCHAR(255) NOT NULL, name VARCHAR(255) NOT NULL,
cksum BINARY(64) NOT NULL, cksum BINARY(64) NOT NULL,
cksum_shown BINARY(64),
size BIGINT UNSIGNED NOT NULL, size BIGINT UNSIGNED NOT NULL,
FOREIGN KEY(id_exercice) REFERENCES exercices(id_exercice) FOREIGN KEY(id_exercice) REFERENCES exercices(id_exercice)
) DEFAULT CHARACTER SET = utf8mb4 COLLATE = utf8mb4_unicode_ci; ) DEFAULT CHARACTER SET = utf8mb4 COLLATE = utf8mb4_unicode_ci;

48
libfic/file.go
View file

@ -40,6 +40,8 @@ type EFile struct {
Name string `json:"name"` Name string `json:"name"`
// Checksum stores the cached hash of the file // Checksum stores the cached hash of the file
Checksum []byte `json:"checksum"` Checksum []byte `json:"checksum"`
// ChecksumShown stores the hash of the downloaded file (in case of gzipped content)
ChecksumShown []byte `json:"checksum_shown"`
// Size contains the cached size of the file // Size contains the cached size of the file
Size int64 `json:"size"` Size int64 `json:"size"`
} }
@ -48,19 +50,20 @@ type EFile struct {
// It is used to check the file validity // It is used to check the file validity
func (e *Exercice) NewDummyFile(origin string, dest string, checksum []byte, size int64) *EFile { func (e *Exercice) NewDummyFile(origin string, dest string, checksum []byte, size int64) *EFile {
return &EFile{ return &EFile{
Id: 0, Id: 0,
origin: origin, origin: origin,
Path: dest, Path: dest,
IdExercice: e.Id, IdExercice: e.Id,
Name: path.Base(origin), Name: path.Base(origin),
Checksum: checksum, Checksum: checksum,
Size: size, ChecksumShown: nil,
Size: size,
} }
} }
// GetFiles returns a list of all files living in the database. // GetFiles returns a list of all files living in the database.
func GetFiles() ([]*EFile, error) { func GetFiles() ([]*EFile, error) {
if rows, err := DBQuery("SELECT id_file, id_exercice, origin, path, name, cksum, size FROM exercice_files"); err != nil { if rows, err := DBQuery("SELECT id_file, id_exercice, origin, path, name, cksum, cksum_shown, size FROM exercice_files"); err != nil {
return nil, err return nil, err
} else { } else {
defer rows.Close() defer rows.Close()
@ -68,7 +71,7 @@ func GetFiles() ([]*EFile, error) {
files := []*EFile{} files := []*EFile{}
for rows.Next() { for rows.Next() {
f := &EFile{} f := &EFile{}
if err := rows.Scan(&f.Id, &f.IdExercice, &f.origin, &f.Path, &f.Name, &f.Checksum, &f.Size); err != nil { if err := rows.Scan(&f.Id, &f.IdExercice, &f.origin, &f.Path, &f.Name, &f.Checksum, &f.ChecksumShown, &f.Size); err != nil {
return nil, err return nil, err
} }
files = append(files, f) files = append(files, f)
@ -84,13 +87,13 @@ func GetFiles() ([]*EFile, error) {
// GetFile retrieves the file with the given id. // GetFile retrieves the file with the given id.
func GetFile(id int64) (f *EFile, err error) { func GetFile(id int64) (f *EFile, err error) {
f = &EFile{} f = &EFile{}
err = DBQueryRow("SELECT id_file, origin, path, name, cksum, size FROM exercice_files WHERE id_file = ?", id).Scan(&f.Id, &f.origin, &f.Path, &f.Name, &f.Checksum, &f.Size) err = DBQueryRow("SELECT id_file, origin, path, name, cksum, cksum_shown, size FROM exercice_files WHERE id_file = ?", id).Scan(&f.Id, &f.origin, &f.Path, &f.Name, &f.Checksum, &f.ChecksumShown, &f.Size)
return return
} }
func (e *Exercice) GetFile(id int64) (f *EFile, err error) { func (e *Exercice) GetFile(id int64) (f *EFile, err error) {
f = &EFile{} f = &EFile{}
err = DBQueryRow("SELECT id_file, origin, path, name, cksum, size FROM exercice_files WHERE id_file = ? AND id_exercice = ?", id, e.Id).Scan(&f.Id, &f.origin, &f.Path, &f.Name, &f.Checksum, &f.Size) err = DBQueryRow("SELECT id_file, origin, path, name, cksum, cksum_shown, size FROM exercice_files WHERE id_file = ? AND id_exercice = ?", id, e.Id).Scan(&f.Id, &f.origin, &f.Path, &f.Name, &f.Checksum, &f.ChecksumShown, &f.Size)
return return
} }
@ -99,7 +102,7 @@ func GetFileByPath(path string) (*EFile, error) {
path = strings.TrimPrefix(path, FilesDir) path = strings.TrimPrefix(path, FilesDir)
f := &EFile{} f := &EFile{}
if err := DBQueryRow("SELECT id_file, origin, path, id_exercice, name, cksum, size FROM exercice_files WHERE path = ?", path).Scan(&f.Id, &f.origin, &f.Path, &f.IdExercice, &f.Name, &f.Checksum, &f.Size); err != nil { if err := DBQueryRow("SELECT id_file, origin, path, id_exercice, name, cksum, cksum_shown, size FROM exercice_files WHERE path = ?", path).Scan(&f.Id, &f.origin, &f.Path, &f.IdExercice, &f.Name, &f.Checksum, &f.ChecksumShown, &f.Size); err != nil {
return f, err return f, err
} }
@ -111,14 +114,14 @@ func (e *Exercice) GetFileByFilename(filename string) (f *EFile, err error) {
filename = path.Base(filename) filename = path.Base(filename)
f = &EFile{} f = &EFile{}
err = DBQueryRow("SELECT id_file, origin, path, id_exercice, name, cksum, size FROM exercice_files WHERE id_exercice = ? AND origin LIKE ?", e.Id, "%/"+filename).Scan(&f.Id, &f.origin, &f.Path, &f.IdExercice, &f.Name, &f.Checksum, &f.Size) err = DBQueryRow("SELECT id_file, origin, path, id_exercice, name, cksum, cksum_shown, size FROM exercice_files WHERE id_exercice = ? AND origin LIKE ?", e.Id, "%/"+filename).Scan(&f.Id, &f.origin, &f.Path, &f.IdExercice, &f.Name, &f.Checksum, &f.ChecksumShown, &f.Size)
return return
} }
// GetFiles returns a list of files coming with the challenge. // GetFiles returns a list of files coming with the challenge.
func (e *Exercice) GetFiles() ([]*EFile, error) { func (e *Exercice) GetFiles() ([]*EFile, error) {
if rows, err := DBQuery("SELECT id_file, origin, path, name, cksum, size FROM exercice_files WHERE id_exercice = ?", e.Id); err != nil { if rows, err := DBQuery("SELECT id_file, origin, path, name, cksum, cksum_shown, size FROM exercice_files WHERE id_exercice = ?", e.Id); err != nil {
return nil, err return nil, err
} else { } else {
defer rows.Close() defer rows.Close()
@ -127,7 +130,7 @@ func (e *Exercice) GetFiles() ([]*EFile, error) {
for rows.Next() { for rows.Next() {
f := &EFile{} f := &EFile{}
f.IdExercice = e.Id f.IdExercice = e.Id
if err := rows.Scan(&f.Id, &f.origin, &f.Path, &f.Name, &f.Checksum, &f.Size); err != nil { if err := rows.Scan(&f.Id, &f.origin, &f.Path, &f.Name, &f.Checksum, &f.ChecksumShown, &f.Size); err != nil {
return nil, err return nil, err
} }
files = append(files, f) files = append(files, f)
@ -145,7 +148,7 @@ func (e *Exercice) GetFileByPath(path string) (*EFile, error) {
path = strings.TrimPrefix(path, FilesDir) path = strings.TrimPrefix(path, FilesDir)
f := &EFile{} f := &EFile{}
if err := DBQueryRow("SELECT id_file, origin, path, name, cksum, size FROM exercice_files WHERE id_exercice = ? AND path = ?", e.Id, path).Scan(&f.Id, &f.origin, &f.Path, &f.Name, &f.Checksum, &f.Size); err != nil { if err := DBQueryRow("SELECT id_file, origin, path, name, cksum, cksum_shown, size FROM exercice_files WHERE id_exercice = ? AND path = ?", e.Id, path).Scan(&f.Id, &f.origin, &f.Path, &f.Name, &f.Checksum, &f.ChecksumShown, &f.Size); err != nil {
return nil, err return nil, err
} }
@ -226,19 +229,20 @@ func checkFileHash(filePath string, digest []byte) (dgst []byte, size int64, err
} }
// ImportFile registers (ou updates if it already exists in database) the file in database. // ImportFile registers (ou updates if it already exists in database) the file in database.
func (e *Exercice) ImportFile(filePath string, origin string, digest []byte) (interface{}, error) { func (e *Exercice) ImportFile(filePath string, origin string, digest []byte, digestshown []byte) (interface{}, error) {
if result512, size, err := checkFileHash(filePath, digest); !OptionalDigest && err != nil { if result512, size, err := checkFileHash(filePath, digest); !OptionalDigest && err != nil {
return EFile{}, err return EFile{}, err
} else { } else {
dPath := strings.TrimPrefix(filePath, FilesDir) dPath := strings.TrimPrefix(filePath, FilesDir)
if f, err := e.GetFileByPath(dPath); err != nil { if f, err := e.GetFileByPath(dPath); err != nil {
return e.AddFile(dPath, origin, path.Base(filePath), result512, size) return e.AddFile(dPath, origin, path.Base(filePath), result512, digestshown, size)
} else { } else {
// Don't need to update Path and Name, because they are related to dPath // Don't need to update Path and Name, because they are related to dPath
f.IdExercice = e.Id f.IdExercice = e.Id
f.origin = origin f.origin = origin
f.Checksum = result512 f.Checksum = result512
f.ChecksumShown = digestshown
f.Size = size f.Size = size
if _, err := f.Update(); err != nil { if _, err := f.Update(); err != nil {
@ -251,19 +255,19 @@ func (e *Exercice) ImportFile(filePath string, origin string, digest []byte) (in
} }
// AddFile creates and fills a new struct File and registers it into the database. // AddFile creates and fills a new struct File and registers it into the database.
func (e *Exercice) AddFile(path string, origin string, name string, checksum []byte, size int64) (*EFile, error) { func (e *Exercice) AddFile(path string, origin string, name string, checksum []byte, checksumshown []byte, size int64) (*EFile, error) {
if res, err := DBExec("INSERT INTO exercice_files (id_exercice, origin, path, name, cksum, size) VALUES (?, ?, ?, ?, ?, ?)", e.Id, origin, path, name, checksum, size); err != nil { if res, err := DBExec("INSERT INTO exercice_files (id_exercice, origin, path, name, cksum, cksum_shown, size) VALUES (?, ?, ?, ?, ?, ?, ?)", e.Id, origin, path, name, checksum, checksumshown, size); err != nil {
return nil, err return nil, err
} else if fid, err := res.LastInsertId(); err != nil { } else if fid, err := res.LastInsertId(); err != nil {
return nil, err return nil, err
} else { } else {
return &EFile{fid, origin, path, e.Id, name, checksum, size}, nil return &EFile{fid, origin, path, e.Id, name, checksum, checksumshown, size}, nil
} }
} }
// Update applies modifications back to the database. // Update applies modifications back to the database.
func (f *EFile) Update() (int64, error) { func (f *EFile) Update() (int64, error) {
if res, err := DBExec("UPDATE exercice_files SET id_exercice = ?, origin = ?, path = ?, name = ?, cksum = ?, size = ? WHERE id_file = ?", f.IdExercice, f.origin, f.Path, f.Name, f.Checksum, f.Size, f.Id); err != nil { if res, err := DBExec("UPDATE exercice_files SET id_exercice = ?, origin = ?, path = ?, name = ?, cksum = ?, cksum_shown = ?, size = ? WHERE id_file = ?", f.IdExercice, f.origin, f.Path, f.Name, f.Checksum, f.ChecksumShown, f.Size, f.Id); err != nil {
return 0, err return 0, err
} else if nb, err := res.RowsAffected(); err != nil { } else if nb, err := res.RowsAffected(); err != nil {
return 0, err return 0, err

6
libfic/team_my.go
View file

@ -174,7 +174,11 @@ func MyJSONTeam(t *Team, started bool) (interface{}, error) {
} else { } else {
for _, f := range files { for _, f := range files {
if t == nil || t.CanDownload(f) { if t == nil || t.CanDownload(f) {
exercice.Files = append(exercice.Files, myTeamFile{path.Join(FilesDir, f.Path), f.Name, hex.EncodeToString(f.Checksum), f.Size}) cksum := f.Checksum
if len(f.ChecksumShown) > 0 {
cksum = f.ChecksumShown
}
exercice.Files = append(exercice.Files, myTeamFile{path.Join(FilesDir, f.Path), f.Name, hex.EncodeToString(cksum), f.Size})
} }
} }
} }